falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
37297 Commits
32 Branches
283MB
Tree: 1b343cedd7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '1b343cedd7'
${ noResults }
FFmpeg/libavcodec/x86/dsputil_mmx.c

1272 lines
64KB
Raw Blame History 

  1. /*

  2.  * MMX optimized DSP utils

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  *

  8.  * This file is part of Libav.

  9.  *

  10.  * Libav is free software; you can redistribute it and/or

  11.  * modify it under the terms of the GNU Lesser General Public

  12.  * License as published by the Free Software Foundation; either

  13.  * version 2.1 of the License, or (at your option) any later version.

  14.  *

  15.  * Libav is distributed in the hope that it will be useful,

  16.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  17.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  18.  * Lesser General Public License for more details.

  19.  *

  20.  * You should have received a copy of the GNU Lesser General Public

  21.  * License along with Libav; if not, write to the Free Software

  22.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  23.  */

  24. 

  25. #include "libavutil/attributes.h"

  26. #include "libavutil/cpu.h"

  27. #include "libavutil/x86/asm.h"

  28. #include "libavcodec/dsputil.h"

  29. #include "libavcodec/h264dsp.h"

  30. #include "libavcodec/mpegvideo.h"

  31. #include "libavcodec/simple_idct.h"

  32. #include "dsputil_mmx.h"

  33. #include "idct_xvid.h"

  34. 

  35. //#undef NDEBUG

  36. //#include <assert.h>

  37. 

  38. /* pixel operations */

  39. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_15)   =   0x000F000F000F000FULL;

  40. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_17)   = { 0x0011001100110011ULL, 0x0011001100110011ULL };

  41. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_20)   =   0x0014001400140014ULL;

  42. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_42)   =   0x002A002A002A002AULL;

  43. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_53)   =   0x0035003500350035ULL;

  44. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_96)   =   0x0060006000600060ULL;

  45. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_128)  =   0x0080008000800080ULL;

  46. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_255)  =   0x00ff00ff00ff00ffULL;

  47. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_512)  = { 0x0200020002000200ULL, 0x0200020002000200ULL };

  48. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };

  49. 

  50. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_FC)   =   0xFCFCFCFCFCFCFCFCULL;

  51. 

  52. DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };

  53. DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };

  54. 

  55. 

  56. void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  57.                               int dstStride, int src1Stride, int h);

  58. void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,

  59.                                      uint8_t *src2, int dstStride,

  60.                                      int src1Stride, int h);

  61. void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  62.                               int dstStride, int src1Stride, int h);

  63. void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  64.                                int dstStride, int src1Stride, int h);

  65. void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  66.                                int dstStride, int src1Stride, int h);

  67. void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  68.                                       int dstStride, int src1Stride, int h);

  69. void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  70.                                          int dstStride, int srcStride, int h);

  71. void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  72.                                          int dstStride, int srcStride, int h);

  73. void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  74.                                                  int dstStride, int srcStride,

  75.                                                  int h);

  76. void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  77.                                         int dstStride, int srcStride, int h);

  78. void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  79.                                         int dstStride, int srcStride, int h);

  80. void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  81.                                                 int dstStride, int srcStride,

  82.                                                 int h);

  83. void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  84.                                          int dstStride, int srcStride);

  85. void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  86.                                          int dstStride, int srcStride);

  87. void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  88.                                                  int dstStride, int srcStride);

  89. void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  90.                                         int dstStride, int srcStride);

  91. void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  92.                                         int dstStride, int srcStride);

  93. void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  94.                                                 int dstStride, int srcStride);

  95. #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext

  96. #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext

  97. 

  98. 

  99. #if HAVE_INLINE_ASM

  100. 

  101. /***********************************/

  102. /* standard MMX */

  103. 

  104. void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  105.                                int line_size)

  106. {

  107.     const int16_t *p;

  108.     uint8_t *pix;

  109. 

  110.     /* read the pixels */

  111.     p   = block;

  112.     pix = pixels;

  113.     /* unrolled loop */

  114.     __asm__ volatile (

  115.         "movq      (%3), %%mm0          \n\t"

  116.         "movq     8(%3), %%mm1          \n\t"

  117.         "movq    16(%3), %%mm2          \n\t"

  118.         "movq    24(%3), %%mm3          \n\t"

  119.         "movq    32(%3), %%mm4          \n\t"

  120.         "movq    40(%3), %%mm5          \n\t"

  121.         "movq    48(%3), %%mm6          \n\t"

  122.         "movq    56(%3), %%mm7          \n\t"

  123.         "packuswb %%mm1, %%mm0          \n\t"

  124.         "packuswb %%mm3, %%mm2          \n\t"

  125.         "packuswb %%mm5, %%mm4          \n\t"

  126.         "packuswb %%mm7, %%mm6          \n\t"

  127.         "movq     %%mm0, (%0)           \n\t"

  128.         "movq     %%mm2, (%0, %1)       \n\t"

  129.         "movq     %%mm4, (%0, %1, 2)    \n\t"

  130.         "movq     %%mm6, (%0, %2)       \n\t"

  131.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),

  132.            "r"(p)

  133.         : "memory");

  134.     pix += line_size * 4;

  135.     p   += 32;

  136. 

  137.     // if here would be an exact copy of the code above

  138.     // compiler would generate some very strange code

  139.     // thus using "r"

  140.     __asm__ volatile (

  141.         "movq       (%3), %%mm0         \n\t"

  142.         "movq      8(%3), %%mm1         \n\t"

  143.         "movq     16(%3), %%mm2         \n\t"

  144.         "movq     24(%3), %%mm3         \n\t"

  145.         "movq     32(%3), %%mm4         \n\t"

  146.         "movq     40(%3), %%mm5         \n\t"

  147.         "movq     48(%3), %%mm6         \n\t"

  148.         "movq     56(%3), %%mm7         \n\t"

  149.         "packuswb  %%mm1, %%mm0         \n\t"

  150.         "packuswb  %%mm3, %%mm2         \n\t"

  151.         "packuswb  %%mm5, %%mm4         \n\t"

  152.         "packuswb  %%mm7, %%mm6         \n\t"

  153.         "movq      %%mm0, (%0)          \n\t"

  154.         "movq      %%mm2, (%0, %1)      \n\t"

  155.         "movq      %%mm4, (%0, %1, 2)   \n\t"

  156.         "movq      %%mm6, (%0, %2)      \n\t"

  157.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)

  158.         : "memory");

  159. }

  160. 

  161. #define put_signed_pixels_clamped_mmx_half(off)             \

  162.     "movq          "#off"(%2), %%mm1        \n\t"           \

  163.     "movq     16 + "#off"(%2), %%mm2        \n\t"           \

  164.     "movq     32 + "#off"(%2), %%mm3        \n\t"           \

  165.     "movq     48 + "#off"(%2), %%mm4        \n\t"           \

  166.     "packsswb  8 + "#off"(%2), %%mm1        \n\t"           \

  167.     "packsswb 24 + "#off"(%2), %%mm2        \n\t"           \

  168.     "packsswb 40 + "#off"(%2), %%mm3        \n\t"           \

  169.     "packsswb 56 + "#off"(%2), %%mm4        \n\t"           \

  170.     "paddb              %%mm0, %%mm1        \n\t"           \

  171.     "paddb              %%mm0, %%mm2        \n\t"           \

  172.     "paddb              %%mm0, %%mm3        \n\t"           \

  173.     "paddb              %%mm0, %%mm4        \n\t"           \

  174.     "movq               %%mm1, (%0)         \n\t"           \

  175.     "movq               %%mm2, (%0, %3)     \n\t"           \

  176.     "movq               %%mm3, (%0, %3, 2)  \n\t"           \

  177.     "movq               %%mm4, (%0, %1)     \n\t"

  178. 

  179. void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  180.                                       int line_size)

  181. {

  182.     x86_reg line_skip = line_size;

  183.     x86_reg line_skip3;

  184. 

  185.     __asm__ volatile (

  186.         "movq "MANGLE(ff_pb_80)", %%mm0     \n\t"

  187.         "lea         (%3, %3, 2), %1        \n\t"

  188.         put_signed_pixels_clamped_mmx_half(0)

  189.         "lea         (%0, %3, 4), %0        \n\t"

  190.         put_signed_pixels_clamped_mmx_half(64)

  191.         : "+&r"(pixels), "=&r"(line_skip3)

  192.         : "r"(block), "r"(line_skip)

  193.         : "memory");

  194. }

  195. 

  196. void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  197.                                int line_size)

  198. {

  199.     const int16_t *p;

  200.     uint8_t *pix;

  201.     int i;

  202. 

  203.     /* read the pixels */

  204.     p   = block;

  205.     pix = pixels;

  206.     MOVQ_ZERO(mm7);

  207.     i = 4;

  208.     do {

  209.         __asm__ volatile (

  210.             "movq        (%2), %%mm0    \n\t"

  211.             "movq       8(%2), %%mm1    \n\t"

  212.             "movq      16(%2), %%mm2    \n\t"

  213.             "movq      24(%2), %%mm3    \n\t"

  214.             "movq          %0, %%mm4    \n\t"

  215.             "movq          %1, %%mm6    \n\t"

  216.             "movq       %%mm4, %%mm5    \n\t"

  217.             "punpcklbw  %%mm7, %%mm4    \n\t"

  218.             "punpckhbw  %%mm7, %%mm5    \n\t"

  219.             "paddsw     %%mm4, %%mm0    \n\t"

  220.             "paddsw     %%mm5, %%mm1    \n\t"

  221.             "movq       %%mm6, %%mm5    \n\t"

  222.             "punpcklbw  %%mm7, %%mm6    \n\t"

  223.             "punpckhbw  %%mm7, %%mm5    \n\t"

  224.             "paddsw     %%mm6, %%mm2    \n\t"

  225.             "paddsw     %%mm5, %%mm3    \n\t"

  226.             "packuswb   %%mm1, %%mm0    \n\t"

  227.             "packuswb   %%mm3, %%mm2    \n\t"

  228.             "movq       %%mm0, %0       \n\t"

  229.             "movq       %%mm2, %1       \n\t"

  230.             : "+m"(*pix), "+m"(*(pix + line_size))

  231.             : "r"(p)

  232.             : "memory");

  233.         pix += line_size * 2;

  234.         p   += 16;

  235.     } while (--i);

  236. }

  237. 

  238. #define CLEAR_BLOCKS(name, n)                           \

  239. static void name(int16_t *blocks)                       \

  240. {                                                       \

  241.     __asm__ volatile (                                  \

  242.         "pxor %%mm7, %%mm7              \n\t"           \

  243.         "mov     %1,        %%"REG_a"   \n\t"           \

  244.         "1:                             \n\t"           \

  245.         "movq %%mm7,   (%0, %%"REG_a")  \n\t"           \

  246.         "movq %%mm7,  8(%0, %%"REG_a")  \n\t"           \

  247.         "movq %%mm7, 16(%0, %%"REG_a")  \n\t"           \

  248.         "movq %%mm7, 24(%0, %%"REG_a")  \n\t"           \

  249.         "add    $32, %%"REG_a"          \n\t"           \

  250.         "js      1b                     \n\t"           \

  251.         :: "r"(((uint8_t *)blocks) + 128 * n),          \

  252.            "i"(-128 * n)                                \

  253.         : "%"REG_a                                      \

  254.         );                                              \

  255. }

  256. CLEAR_BLOCKS(clear_blocks_mmx, 6)

  257. CLEAR_BLOCKS(clear_block_mmx, 1)

  258. 

  259. static void clear_block_sse(int16_t *block)

  260. {

  261.     __asm__ volatile (

  262.         "xorps  %%xmm0, %%xmm0          \n"

  263.         "movaps %%xmm0,    (%0)         \n"

  264.         "movaps %%xmm0,  16(%0)         \n"

  265.         "movaps %%xmm0,  32(%0)         \n"

  266.         "movaps %%xmm0,  48(%0)         \n"

  267.         "movaps %%xmm0,  64(%0)         \n"

  268.         "movaps %%xmm0,  80(%0)         \n"

  269.         "movaps %%xmm0,  96(%0)         \n"

  270.         "movaps %%xmm0, 112(%0)         \n"

  271.         :: "r"(block)

  272.         : "memory"

  273.     );

  274. }

  275. 

  276. static void clear_blocks_sse(int16_t *blocks)

  277. {

  278.     __asm__ volatile (

  279.         "xorps  %%xmm0, %%xmm0              \n"

  280.         "mov        %1,         %%"REG_a"   \n"

  281.         "1:                                 \n"

  282.         "movaps %%xmm0,    (%0, %%"REG_a")  \n"

  283.         "movaps %%xmm0,  16(%0, %%"REG_a")  \n"

  284.         "movaps %%xmm0,  32(%0, %%"REG_a")  \n"

  285.         "movaps %%xmm0,  48(%0, %%"REG_a")  \n"

  286.         "movaps %%xmm0,  64(%0, %%"REG_a")  \n"

  287.         "movaps %%xmm0,  80(%0, %%"REG_a")  \n"

  288.         "movaps %%xmm0,  96(%0, %%"REG_a")  \n"

  289.         "movaps %%xmm0, 112(%0, %%"REG_a")  \n"

  290.         "add      $128,         %%"REG_a"   \n"

  291.         "js         1b                      \n"

  292.         :: "r"(((uint8_t *)blocks) + 128 * 6),

  293.            "i"(-128 * 6)

  294.         : "%"REG_a

  295.     );

  296. }

  297. 

  298. static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)

  299. {

  300.     x86_reg i = 0;

  301.     __asm__ volatile (

  302.         "jmp          2f                \n\t"

  303.         "1:                             \n\t"

  304.         "movq   (%1, %0), %%mm0         \n\t"

  305.         "movq   (%2, %0), %%mm1         \n\t"

  306.         "paddb     %%mm0, %%mm1         \n\t"

  307.         "movq      %%mm1, (%2, %0)      \n\t"

  308.         "movq  8(%1, %0), %%mm0         \n\t"

  309.         "movq  8(%2, %0), %%mm1         \n\t"

  310.         "paddb     %%mm0, %%mm1         \n\t"

  311.         "movq      %%mm1, 8(%2, %0)     \n\t"

  312.         "add         $16, %0            \n\t"

  313.         "2:                             \n\t"

  314.         "cmp          %3, %0            \n\t"

  315.         "js           1b                \n\t"

  316.         : "+r"(i)

  317.         : "r"(src), "r"(dst), "r"((x86_reg)w - 15)

  318.     );

  319.     for ( ; i < w; i++)

  320.         dst[i + 0] += src[i + 0];

  321. }

  322. 

  323. #if HAVE_7REGS

  324. static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,

  325.                                             const uint8_t *diff, int w,

  326.                                             int *left, int *left_top)

  327. {

  328.     x86_reg w2 = -w;

  329.     x86_reg x;

  330.     int l  = *left     & 0xff;

  331.     int tl = *left_top & 0xff;

  332.     int t;

  333.     __asm__ volatile (

  334.         "mov          %7, %3            \n"

  335.         "1:                             \n"

  336.         "movzbl (%3, %4), %2            \n"

  337.         "mov          %2, %k3           \n"

  338.         "sub         %b1, %b3           \n"

  339.         "add         %b0, %b3           \n"

  340.         "mov          %2, %1            \n"

  341.         "cmp          %0, %2            \n"

  342.         "cmovg        %0, %2            \n"

  343.         "cmovg        %1, %0            \n"

  344.         "cmp         %k3, %0            \n"

  345.         "cmovg       %k3, %0            \n"

  346.         "mov          %7, %3            \n"

  347.         "cmp          %2, %0            \n"

  348.         "cmovl        %2, %0            \n"

  349.         "add    (%6, %4), %b0           \n"

  350.         "mov         %b0, (%5, %4)      \n"

  351.         "inc          %4                \n"

  352.         "jl           1b                \n"

  353.         : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)

  354.         : "r"(dst + w), "r"(diff + w), "rm"(top + w)

  355.     );

  356.     *left     = l;

  357.     *left_top = tl;

  358. }

  359. #endif

  360. 

  361. /* Draw the edges of width 'w' of an image of size width, height

  362.  * this MMX version can only handle w == 8 || w == 16. */

  363. static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,

  364.                            int w, int h, int sides)

  365. {

  366.     uint8_t *ptr, *last_line;

  367.     int i;

  368. 

  369.     last_line = buf + (height - 1) * wrap;

  370.     /* left and right */

  371.     ptr = buf;

  372.     if (w == 8) {

  373.         __asm__ volatile (

  374.             "1:                             \n\t"

  375.             "movd            (%0), %%mm0    \n\t"

  376.             "punpcklbw      %%mm0, %%mm0    \n\t"

  377.             "punpcklwd      %%mm0, %%mm0    \n\t"

  378.             "punpckldq      %%mm0, %%mm0    \n\t"

  379.             "movq           %%mm0, -8(%0)   \n\t"

  380.             "movq      -8(%0, %2), %%mm1    \n\t"

  381.             "punpckhbw      %%mm1, %%mm1    \n\t"

  382.             "punpckhwd      %%mm1, %%mm1    \n\t"

  383.             "punpckhdq      %%mm1, %%mm1    \n\t"

  384.             "movq           %%mm1, (%0, %2) \n\t"

  385.             "add               %1, %0       \n\t"

  386.             "cmp               %3, %0       \n\t"

  387.             "jb                1b           \n\t"

  388.             : "+r"(ptr)

  389.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  390.             );

  391.     } else {

  392.         __asm__ volatile (

  393.             "1:                                 \n\t"

  394.             "movd            (%0), %%mm0        \n\t"

  395.             "punpcklbw      %%mm0, %%mm0        \n\t"

  396.             "punpcklwd      %%mm0, %%mm0        \n\t"

  397.             "punpckldq      %%mm0, %%mm0        \n\t"

  398.             "movq           %%mm0, -8(%0)       \n\t"

  399.             "movq           %%mm0, -16(%0)      \n\t"

  400.             "movq      -8(%0, %2), %%mm1        \n\t"

  401.             "punpckhbw      %%mm1, %%mm1        \n\t"

  402.             "punpckhwd      %%mm1, %%mm1        \n\t"

  403.             "punpckhdq      %%mm1, %%mm1        \n\t"

  404.             "movq           %%mm1,  (%0, %2)    \n\t"

  405.             "movq           %%mm1, 8(%0, %2)    \n\t"

  406.             "add               %1, %0           \n\t"

  407.             "cmp               %3, %0           \n\t"

  408.             "jb                1b               \n\t"

  409.             : "+r"(ptr)

  410.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  411.             );

  412.     }

  413. 

  414.     /* top and bottom (and hopefully also the corners) */

  415.     if (sides & EDGE_TOP) {

  416.         for (i = 0; i < h; i += 4) {

  417.             ptr = buf - (i + 1) * wrap - w;

  418.             __asm__ volatile (

  419.                 "1:                             \n\t"

  420.                 "movq (%1, %0), %%mm0           \n\t"

  421.                 "movq    %%mm0, (%0)            \n\t"

  422.                 "movq    %%mm0, (%0, %2)        \n\t"

  423.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  424.                 "movq    %%mm0, (%0, %3)        \n\t"

  425.                 "add        $8, %0              \n\t"

  426.                 "cmp        %4, %0              \n\t"

  427.                 "jb         1b                  \n\t"

  428.                 : "+r"(ptr)

  429.                 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),

  430.                   "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)

  431.                 );

  432.         }

  433.     }

  434. 

  435.     if (sides & EDGE_BOTTOM) {

  436.         for (i = 0; i < h; i += 4) {

  437.             ptr = last_line + (i + 1) * wrap - w;

  438.             __asm__ volatile (

  439.                 "1:                             \n\t"

  440.                 "movq (%1, %0), %%mm0           \n\t"

  441.                 "movq    %%mm0, (%0)            \n\t"

  442.                 "movq    %%mm0, (%0, %2)        \n\t"

  443.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  444.                 "movq    %%mm0, (%0, %3)        \n\t"

  445.                 "add        $8, %0              \n\t"

  446.                 "cmp        %4, %0              \n\t"

  447.                 "jb         1b                  \n\t"

  448.                 : "+r"(ptr)

  449.                 : "r"((x86_reg)last_line - (x86_reg)ptr - w),

  450.                   "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),

  451.                   "r"(ptr + width + 2 * w)

  452.                 );

  453.         }

  454.     }

  455. }

  456. #endif /* HAVE_INLINE_ASM */

  457. 

  458. 

  459. #if HAVE_YASM

  460. static void ff_avg_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,

  461.                                    int line_size, int h)

  462. {

  463.     ff_avg_pixels8_mmxext(block,     pixels,     line_size, h);

  464.     ff_avg_pixels8_mmxext(block + 8, pixels + 8, line_size, h);

  465. }

  466. 

  467. static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,

  468.                                    ptrdiff_t line_size, int h)

  469. {

  470.     ff_put_pixels8_mmxext(block,     pixels,     line_size, h);

  471.     ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);

  472. }

  473. 

  474. #define QPEL_OP(OPNAME, ROUNDER, RND, MMX)                              \

  475. static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \

  476.                                           ptrdiff_t stride)             \

  477. {                                                                       \

  478.     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \

  479. }                                                                       \

  480.                                                                         \

  481. static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \

  482.                                          ptrdiff_t stride)              \

  483. {                                                                       \

  484.     uint64_t temp[8];                                                   \

  485.     uint8_t * const half = (uint8_t*)temp;                              \

  486.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  487.                                                    stride, 8);          \

  488.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  489.                                         stride, stride, 8);             \

  490. }                                                                       \

  491.                                                                         \

  492. static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \

  493.                                          ptrdiff_t stride)              \

  494. {                                                                       \

  495.     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \

  496.                                                    stride, 8);          \

  497. }                                                                       \

  498.                                                                         \

  499. static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \

  500.                                          ptrdiff_t stride)              \

  501. {                                                                       \

  502.     uint64_t temp[8];                                                   \

  503.     uint8_t * const half = (uint8_t*)temp;                              \

  504.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  505.                                                    stride, 8);          \

  506.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \

  507.                                         stride, 8);                     \

  508. }                                                                       \

  509.                                                                         \

  510. static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \

  511.                                          ptrdiff_t stride)              \

  512. {                                                                       \

  513.     uint64_t temp[8];                                                   \

  514.     uint8_t * const half = (uint8_t*)temp;                              \

  515.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  516.                                                    8, stride);          \

  517.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  518.                                         stride, stride, 8);             \

  519. }                                                                       \

  520.                                                                         \

  521. static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \

  522.                                          ptrdiff_t stride)              \

  523. {                                                                       \

  524.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \

  525.                                                    stride, stride);     \

  526. }                                                                       \

  527.                                                                         \

  528. static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \

  529.                                          ptrdiff_t stride)              \

  530. {                                                                       \

  531.     uint64_t temp[8];                                                   \

  532.     uint8_t * const half = (uint8_t*)temp;                              \

  533.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  534.                                                    8, stride);          \

  535.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\

  536.                                         stride, 8);                     \

  537. }                                                                       \

  538.                                                                         \

  539. static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \

  540.                                          ptrdiff_t stride)              \

  541. {                                                                       \

  542.     uint64_t half[8 + 9];                                               \

  543.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  544.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  545.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  546.                                                    stride, 9);          \

  547.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  548.                                         stride, 9);                     \

  549.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  550.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  551.                                         stride, 8, 8);                  \

  552. }                                                                       \

  553.                                                                         \

  554. static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \

  555.                                          ptrdiff_t stride)              \

  556. {                                                                       \

  557.     uint64_t half[8 + 9];                                               \

  558.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  559.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  560.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  561.                                                    stride, 9);          \

  562.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  563.                                         stride, 9);                     \

  564.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  565.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  566.                                         stride, 8, 8);                  \

  567. }                                                                       \

  568.                                                                         \

  569. static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \

  570.                                          ptrdiff_t stride)              \

  571. {                                                                       \

  572.     uint64_t half[8 + 9];                                               \

  573.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  574.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  575.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  576.                                                    stride, 9);          \

  577.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  578.                                         stride, 9);                     \

  579.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  580.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  581.                                         stride, 8, 8);                  \

  582. }                                                                       \

  583.                                                                         \

  584. static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \

  585.                                          ptrdiff_t stride)              \

  586. {                                                                       \

  587.     uint64_t half[8 + 9];                                               \

  588.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  589.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  590.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  591.                                                    stride, 9);          \

  592.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  593.                                         stride, 9);                     \

  594.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  595.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  596.                                         stride, 8, 8);                  \

  597. }                                                                       \

  598.                                                                         \

  599. static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \

  600.                                          ptrdiff_t stride)              \

  601. {                                                                       \

  602.     uint64_t half[8 + 9];                                               \

  603.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  604.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  605.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  606.                                                    stride, 9);          \

  607.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  608.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  609.                                         stride, 8, 8);                  \

  610. }                                                                       \

  611.                                                                         \

  612. static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \

  613.                                          ptrdiff_t stride)              \

  614. {                                                                       \

  615.     uint64_t half[8 + 9];                                               \

  616.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  617.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  618.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  619.                                                    stride, 9);          \

  620.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  621.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  622.                                         stride, 8, 8);                  \

  623. }                                                                       \

  624.                                                                         \

  625. static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \

  626.                                          ptrdiff_t stride)              \

  627. {                                                                       \

  628.     uint64_t half[8 + 9];                                               \

  629.     uint8_t * const halfH = ((uint8_t*)half);                           \

  630.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  631.                                                    stride, 9);          \

  632.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \

  633.                                         8, stride, 9);                  \

  634.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  635.                                                    stride, 8);          \

  636. }                                                                       \

  637.                                                                         \

  638. static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \

  639.                                          ptrdiff_t stride)              \

  640. {                                                                       \

  641.     uint64_t half[8 + 9];                                               \

  642.     uint8_t * const halfH = ((uint8_t*)half);                           \

  643.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  644.                                                    stride, 9);          \

  645.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  646.                                         stride, 9);                     \

  647.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  648.                                                    stride, 8);          \

  649. }                                                                       \

  650.                                                                         \

  651. static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \

  652.                                          ptrdiff_t stride)              \

  653. {                                                                       \

  654.     uint64_t half[9];                                                   \

  655.     uint8_t * const halfH = ((uint8_t*)half);                           \

  656.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  657.                                                    stride, 9);          \

  658.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  659.                                                    stride, 8);          \

  660. }                                                                       \

  661.                                                                         \

  662. static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \

  663.                                            ptrdiff_t stride)            \

  664. {                                                                       \

  665.     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \

  666. }                                                                       \

  667.                                                                         \

  668. static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \

  669.                                           ptrdiff_t stride)             \

  670. {                                                                       \

  671.     uint64_t temp[32];                                                  \

  672.     uint8_t * const half = (uint8_t*)temp;                              \

  673.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  674.                                                     stride, 16);        \

  675.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  676.                                          stride, 16);                   \

  677. }                                                                       \

  678.                                                                         \

  679. static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \

  680.                                           ptrdiff_t stride)             \

  681. {                                                                       \

  682.     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \

  683.                                                     stride, stride, 16);\

  684. }                                                                       \

  685.                                                                         \

  686. static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \

  687.                                           ptrdiff_t stride)             \

  688. {                                                                       \

  689.     uint64_t temp[32];                                                  \

  690.     uint8_t * const half = (uint8_t*)temp;                              \

  691.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  692.                                                     stride, 16);        \

  693.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \

  694.                                          stride, stride, 16);           \

  695. }                                                                       \

  696.                                                                         \

  697. static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \

  698.                                           ptrdiff_t stride)             \

  699. {                                                                       \

  700.     uint64_t temp[32];                                                  \

  701.     uint8_t * const half = (uint8_t*)temp;                              \

  702.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  703.                                                     stride);            \

  704.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  705.                                          stride, 16);                   \

  706. }                                                                       \

  707.                                                                         \

  708. static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \

  709.                                           ptrdiff_t stride)             \

  710. {                                                                       \

  711.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \

  712.                                                     stride, stride);    \

  713. }                                                                       \

  714.                                                                         \

  715. static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \

  716.                                           ptrdiff_t stride)             \

  717. {                                                                       \

  718.     uint64_t temp[32];                                                  \

  719.     uint8_t * const half = (uint8_t*)temp;                              \

  720.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  721.                                                     stride);            \

  722.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \

  723.                                          stride, stride, 16);           \

  724. }                                                                       \

  725.                                                                         \

  726. static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \

  727.                                           ptrdiff_t stride)             \

  728. {                                                                       \

  729.     uint64_t half[16 * 2 + 17 * 2];                                     \

  730.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  731.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  732.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  733.                                                     stride, 17);        \

  734.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  735.                                          stride, 17);                   \

  736.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  737.                                                     16, 16);            \

  738.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  739.                                          stride, 16, 16);               \

  740. }                                                                       \

  741.                                                                         \

  742. static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \

  743.                                           ptrdiff_t stride)             \

  744. {                                                                       \

  745.     uint64_t half[16 * 2 + 17 * 2];                                     \

  746.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  747.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  748.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  749.                                                     stride, 17);        \

  750.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  751.                                          stride, 17);                   \

  752.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  753.                                                     16, 16);            \

  754.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  755.                                          stride, 16, 16);               \

  756. }                                                                       \

  757.                                                                         \

  758. static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \

  759.                                           ptrdiff_t stride)             \

  760. {                                                                       \

  761.     uint64_t half[16 * 2 + 17 * 2];                                     \

  762.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  763.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  764.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  765.                                                     stride, 17);        \

  766.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  767.                                          stride, 17);                   \

  768.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  769.                                                     16, 16);            \

  770.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  771.                                          stride, 16, 16);               \

  772. }                                                                       \

  773.                                                                         \

  774. static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \

  775.                                           ptrdiff_t stride)             \

  776. {                                                                       \

  777.     uint64_t half[16 * 2 + 17 * 2];                                     \

  778.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  779.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  780.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  781.                                                     stride, 17);        \

  782.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  783.                                          stride, 17);                   \

  784.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  785.                                                     16, 16);            \

  786.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  787.                                          stride, 16, 16);               \

  788. }                                                                       \

  789.                                                                         \

  790. static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \

  791.                                           ptrdiff_t stride)             \

  792. {                                                                       \

  793.     uint64_t half[16 * 2 + 17 * 2];                                     \

  794.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  795.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  796.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  797.                                                     stride, 17);        \

  798.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  799.                                                     16, 16);            \

  800.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  801.                                          stride, 16, 16);               \

  802. }                                                                       \

  803.                                                                         \

  804. static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \

  805.                                           ptrdiff_t stride)             \

  806. {                                                                       \

  807.     uint64_t half[16 * 2 + 17 * 2];                                     \

  808.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  809.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  810.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  811.                                                     stride, 17);        \

  812.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  813.                                                     16, 16);            \

  814.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  815.                                          stride, 16, 16);               \

  816. }                                                                       \

  817.                                                                         \

  818. static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \

  819.                                           ptrdiff_t stride)             \

  820. {                                                                       \

  821.     uint64_t half[17 * 2];                                              \

  822.     uint8_t * const halfH = ((uint8_t*)half);                           \

  823.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  824.                                                     stride, 17);        \

  825.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  826.                                          stride, 17);                   \

  827.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  828.                                                     stride, 16);        \

  829. }                                                                       \

  830.                                                                         \

  831. static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \

  832.                                           ptrdiff_t stride)             \

  833. {                                                                       \

  834.     uint64_t half[17 * 2];                                              \

  835.     uint8_t * const halfH = ((uint8_t*)half);                           \

  836.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  837.                                                     stride, 17);        \

  838.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  839.                                          stride, 17);                   \

  840.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  841.                                                     stride, 16);        \

  842. }                                                                       \

  843.                                                                         \

  844. static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \

  845.                                           ptrdiff_t stride)             \

  846. {                                                                       \

  847.     uint64_t half[17 * 2];                                              \

  848.     uint8_t * const halfH = ((uint8_t*)half);                           \

  849.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  850.                                                     stride, 17);        \

  851.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  852.                                                     stride, 16);        \

  853. }

  854. 

  855. QPEL_OP(put_,          ff_pw_16, _,        mmxext)

  856. QPEL_OP(avg_,          ff_pw_16, _,        mmxext)

  857. QPEL_OP(put_no_rnd_,   ff_pw_15, _no_rnd_, mmxext)

  858. #endif /* HAVE_YASM */

  859. 

  860. 

  861. #if HAVE_INLINE_ASM

  862. static void gmc_mmx(uint8_t *dst, uint8_t *src,

  863.                     int stride, int h, int ox, int oy,

  864.                     int dxx, int dxy, int dyx, int dyy,

  865.                     int shift, int r, int width, int height)

  866. {

  867.     const int w    = 8;

  868.     const int ix   = ox  >> (16 + shift);

  869.     const int iy   = oy  >> (16 + shift);

  870.     const int oxs  = ox  >> 4;

  871.     const int oys  = oy  >> 4;

  872.     const int dxxs = dxx >> 4;

  873.     const int dxys = dxy >> 4;

  874.     const int dyxs = dyx >> 4;

  875.     const int dyys = dyy >> 4;

  876.     const uint16_t r4[4]   = { r, r, r, r };

  877.     const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };

  878.     const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };

  879.     const uint64_t shift2 = 2 * shift;

  880.     int x, y;

  881. 

  882.     const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);

  883.     const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);

  884.     const int dxh = dxy * (h - 1);

  885.     const int dyw = dyx * (w - 1);

  886.     if ( // non-constant fullpel offset (3% of blocks)

  887.         ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |

  888.          (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)

  889.         // uses more than 16 bits of subpel mv (only at huge resolution)

  890.         || (dxx | dxy | dyx | dyy) & 15 ||

  891.         (unsigned)ix >= width  - w ||

  892.         (unsigned)iy >= height - h) {

  893.         // FIXME could still use mmx for some of the rows

  894.         ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,

  895.                  shift, r, width, height);

  896.         return;

  897.     }

  898. 

  899.     src += ix + iy * stride;

  900. 

  901.     __asm__ volatile (

  902.         "movd         %0, %%mm6         \n\t"

  903.         "pxor      %%mm7, %%mm7         \n\t"

  904.         "punpcklwd %%mm6, %%mm6         \n\t"

  905.         "punpcklwd %%mm6, %%mm6         \n\t"

  906.         :: "r"(1<<shift)

  907.     );

  908. 

  909.     for (x = 0; x < w; x += 4) {

  910.         uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),

  911.                             oxs - dxys + dxxs * (x + 1),

  912.                             oxs - dxys + dxxs * (x + 2),

  913.                             oxs - dxys + dxxs * (x + 3) };

  914.         uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),

  915.                             oys - dyys + dyxs * (x + 1),

  916.                             oys - dyys + dyxs * (x + 2),

  917.                             oys - dyys + dyxs * (x + 3) };

  918. 

  919.         for (y = 0; y < h; y++) {

  920.             __asm__ volatile (

  921.                 "movq      %0, %%mm4    \n\t"

  922.                 "movq      %1, %%mm5    \n\t"

  923.                 "paddw     %2, %%mm4    \n\t"

  924.                 "paddw     %3, %%mm5    \n\t"

  925.                 "movq   %%mm4, %0       \n\t"

  926.                 "movq   %%mm5, %1       \n\t"

  927.                 "psrlw    $12, %%mm4    \n\t"

  928.                 "psrlw    $12, %%mm5    \n\t"

  929.                 : "+m"(*dx4), "+m"(*dy4)

  930.                 : "m"(*dxy4), "m"(*dyy4)

  931.             );

  932. 

  933.             __asm__ volatile (

  934.                 "movq      %%mm6, %%mm2 \n\t"

  935.                 "movq      %%mm6, %%mm1 \n\t"

  936.                 "psubw     %%mm4, %%mm2 \n\t"

  937.                 "psubw     %%mm5, %%mm1 \n\t"

  938.                 "movq      %%mm2, %%mm0 \n\t"

  939.                 "movq      %%mm4, %%mm3 \n\t"

  940.                 "pmullw    %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)

  941.                 "pmullw    %%mm5, %%mm3 \n\t" // dx * dy

  942.                 "pmullw    %%mm5, %%mm2 \n\t" // (s - dx) * dy

  943.                 "pmullw    %%mm4, %%mm1 \n\t" // dx * (s - dy)

  944. 

  945.                 "movd         %4, %%mm5 \n\t"

  946.                 "movd         %3, %%mm4 \n\t"

  947.                 "punpcklbw %%mm7, %%mm5 \n\t"

  948.                 "punpcklbw %%mm7, %%mm4 \n\t"

  949.                 "pmullw    %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy

  950.                 "pmullw    %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy

  951. 

  952.                 "movd         %2, %%mm5 \n\t"

  953.                 "movd         %1, %%mm4 \n\t"

  954.                 "punpcklbw %%mm7, %%mm5 \n\t"

  955.                 "punpcklbw %%mm7, %%mm4 \n\t"

  956.                 "pmullw    %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)

  957.                 "pmullw    %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)

  958.                 "paddw        %5, %%mm1 \n\t"

  959.                 "paddw     %%mm3, %%mm2 \n\t"

  960.                 "paddw     %%mm1, %%mm0 \n\t"

  961.                 "paddw     %%mm2, %%mm0 \n\t"

  962. 

  963.                 "psrlw        %6, %%mm0 \n\t"

  964.                 "packuswb  %%mm0, %%mm0 \n\t"

  965.                 "movd      %%mm0, %0    \n\t"

  966. 

  967.                 : "=m"(dst[x + y * stride])

  968.                 : "m"(src[0]), "m"(src[1]),

  969.                   "m"(src[stride]), "m"(src[stride + 1]),

  970.                   "m"(*r4), "m"(shift2)

  971.             );

  972.             src += stride;

  973.         }

  974.         src += 4 - h * stride;

  975.     }

  976. }

  977. 

  978. static void vector_clipf_sse(float *dst, const float *src,

  979.                              float min, float max, int len)

  980. {

  981.     x86_reg i = (len - 16) * 4;

  982.     __asm__ volatile (

  983.         "movss          %3, %%xmm4      \n\t"

  984.         "movss          %4, %%xmm5      \n\t"

  985.         "shufps $0, %%xmm4, %%xmm4      \n\t"

  986.         "shufps $0, %%xmm5, %%xmm5      \n\t"

  987.         "1:                             \n\t"

  988.         "movaps   (%2, %0), %%xmm0      \n\t" // 3/1 on intel

  989.         "movaps 16(%2, %0), %%xmm1      \n\t"

  990.         "movaps 32(%2, %0), %%xmm2      \n\t"

  991.         "movaps 48(%2, %0), %%xmm3      \n\t"

  992.         "maxps      %%xmm4, %%xmm0      \n\t"

  993.         "maxps      %%xmm4, %%xmm1      \n\t"

  994.         "maxps      %%xmm4, %%xmm2      \n\t"

  995.         "maxps      %%xmm4, %%xmm3      \n\t"

  996.         "minps      %%xmm5, %%xmm0      \n\t"

  997.         "minps      %%xmm5, %%xmm1      \n\t"

  998.         "minps      %%xmm5, %%xmm2      \n\t"

  999.         "minps      %%xmm5, %%xmm3      \n\t"

  1000.         "movaps     %%xmm0,   (%1, %0)  \n\t"

  1001.         "movaps     %%xmm1, 16(%1, %0)  \n\t"

  1002.         "movaps     %%xmm2, 32(%1, %0)  \n\t"

  1003.         "movaps     %%xmm3, 48(%1, %0)  \n\t"

  1004.         "sub           $64, %0          \n\t"

  1005.         "jge            1b              \n\t"

  1006.         : "+&r"(i)

  1007.         : "r"(dst), "r"(src), "m"(min), "m"(max)

  1008.         : "memory"

  1009.     );

  1010. }

  1011. 

  1012. #endif /* HAVE_INLINE_ASM */

  1013. 

  1014. void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  1015. void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  1016. 

  1017. int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,

  1018.                                       int order);

  1019. int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,

  1020.                                     int order);

  1021. int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,

  1022.                                                const int16_t *v3,

  1023.                                                int order, int mul);

  1024. int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,

  1025.                                              const int16_t *v3,

  1026.                                              int order, int mul);

  1027. int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,

  1028.                                               const int16_t *v3,

  1029.                                               int order, int mul);

  1030. 

  1031. void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,

  1032.                                         const int16_t *window, unsigned int len);

  1033. void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,

  1034.                                       const int16_t *window, unsigned int len);

  1035. void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,

  1036.                                   const int16_t *window, unsigned int len);

  1037. void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,

  1038.                                 const int16_t *window, unsigned int len);

  1039. void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,

  1040.                                  const int16_t *window, unsigned int len);

  1041. void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,

  1042.                                       const int16_t *window, unsigned int len);

  1043. 

  1044. void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);

  1045. void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);

  1046. 

  1047. void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,

  1048.                                           const uint8_t *diff, int w,

  1049.                                           int *left, int *left_top);

  1050. int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,

  1051.                                        int w, int left);

  1052. int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,

  1053.                                       int w, int left);

  1054. 

  1055. void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,

  1056.                                    int32_t min, int32_t max, unsigned int len);

  1057. void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,

  1058.                                    int32_t min, int32_t max, unsigned int len);

  1059. void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,

  1060.                                    int32_t min, int32_t max, unsigned int len);

  1061. void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,

  1062.                                    int32_t min, int32_t max, unsigned int len);

  1063. 

  1064. #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \

  1065.     do {                                                                     \

  1066.     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \

  1067.     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \

  1068.     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \

  1069.     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \

  1070.     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \

  1071.     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \

  1072.     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \

  1073.     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \

  1074.     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \

  1075.     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \

  1076.     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \

  1077.     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \

  1078.     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \

  1079.     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \

  1080.     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \

  1081.     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \

  1082.     } while (0)

  1083. 

  1084. static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,

  1085.                                      int mm_flags)

  1086. {

  1087. #if HAVE_MMX_INLINE

  1088.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1089. 

  1090.     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;

  1091.     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;

  1092.     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;

  1093. 

  1094.     if (!high_bit_depth) {

  1095.         c->clear_block  = clear_block_mmx;

  1096.         c->clear_blocks = clear_blocks_mmx;

  1097.         c->draw_edges   = draw_edges_mmx;

  1098. 

  1099.         switch (avctx->idct_algo) {

  1100.         case FF_IDCT_AUTO:

  1101.         case FF_IDCT_SIMPLEMMX:

  1102.             c->idct_put              = ff_simple_idct_put_mmx;

  1103.             c->idct_add              = ff_simple_idct_add_mmx;

  1104.             c->idct                  = ff_simple_idct_mmx;

  1105.             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;

  1106.             break;

  1107.         case FF_IDCT_XVIDMMX:

  1108.             c->idct_put              = ff_idct_xvid_mmx_put;

  1109.             c->idct_add              = ff_idct_xvid_mmx_add;

  1110.             c->idct                  = ff_idct_xvid_mmx;

  1111.             break;

  1112.         }

  1113.     }

  1114. 

  1115.     c->gmc = gmc_mmx;

  1116. 

  1117.     c->add_bytes = add_bytes_mmx;

  1118. #endif /* HAVE_MMX_INLINE */

  1119. 

  1120. #if HAVE_MMX_EXTERNAL

  1121.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  1122.         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;

  1123.         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;

  1124.     }

  1125. 

  1126.     c->vector_clip_int32 = ff_vector_clip_int32_mmx;

  1127. #endif /* HAVE_MMX_EXTERNAL */

  1128. }

  1129. 

  1130. static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,

  1131.                                         int mm_flags)

  1132. {

  1133. #if HAVE_MMXEXT_INLINE

  1134.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1135. 

  1136.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1137.         c->idct_put = ff_idct_xvid_mmxext_put;

  1138.         c->idct_add = ff_idct_xvid_mmxext_add;

  1139.         c->idct     = ff_idct_xvid_mmxext;

  1140.     }

  1141. #endif /* HAVE_MMXEXT_INLINE */

  1142. 

  1143. #if HAVE_MMXEXT_EXTERNAL

  1144.     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );

  1145.     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );

  1146. 

  1147.     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );

  1148.     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );

  1149.     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );

  1150.     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );

  1151. 

  1152.     /* slower than cmov version on AMD */

  1153.     if (!(mm_flags & AV_CPU_FLAG_3DNOW))

  1154.         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;

  1155. 

  1156.     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;

  1157.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;

  1158. 

  1159.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1160.         c->apply_window_int16 = ff_apply_window_int16_mmxext;

  1161.     } else {

  1162.         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;

  1163.     }

  1164. #endif /* HAVE_MMXEXT_EXTERNAL */

  1165. }

  1166. 

  1167. static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,

  1168.                                      int mm_flags)

  1169. {

  1170. #if HAVE_SSE_INLINE

  1171.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1172. 

  1173.     if (!high_bit_depth) {

  1174.         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {

  1175.             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */

  1176.             c->clear_block  = clear_block_sse;

  1177.             c->clear_blocks = clear_blocks_sse;

  1178.         }

  1179.     }

  1180. 

  1181.     c->vector_clipf = vector_clipf_sse;

  1182. #endif /* HAVE_SSE_INLINE */

  1183. }

  1184. 

  1185. static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,

  1186.                                       int mm_flags)

  1187. {

  1188. #if HAVE_SSE2_INLINE

  1189.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1190. 

  1191.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1192.         c->idct_put              = ff_idct_xvid_sse2_put;

  1193.         c->idct_add              = ff_idct_xvid_sse2_add;

  1194.         c->idct                  = ff_idct_xvid_sse2;

  1195.         c->idct_permutation_type = FF_SSE2_IDCT_PERM;

  1196.     }

  1197. #endif /* HAVE_SSE2_INLINE */

  1198. 

  1199. #if HAVE_SSE2_EXTERNAL

  1200.     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;

  1201.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;

  1202.     if (mm_flags & AV_CPU_FLAG_ATOM) {

  1203.         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;

  1204.     } else {

  1205.         c->vector_clip_int32 = ff_vector_clip_int32_sse2;

  1206.     }

  1207.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1208.         c->apply_window_int16 = ff_apply_window_int16_sse2;

  1209.     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  1210.         c->apply_window_int16 = ff_apply_window_int16_round_sse2;

  1211.     }

  1212.     c->bswap_buf = ff_bswap32_buf_sse2;

  1213. #endif /* HAVE_SSE2_EXTERNAL */

  1214. }

  1215. 

  1216. static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,

  1217.                                        int mm_flags)

  1218. {

  1219. #if HAVE_SSSE3_EXTERNAL

  1220.     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;

  1221.     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe

  1222.         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;

  1223. 

  1224.     if (mm_flags & AV_CPU_FLAG_ATOM)

  1225.         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;

  1226.     else

  1227.         c->apply_window_int16 = ff_apply_window_int16_ssse3;

  1228.     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit

  1229.         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;

  1230.     c->bswap_buf = ff_bswap32_buf_ssse3;

  1231. #endif /* HAVE_SSSE3_EXTERNAL */

  1232. }

  1233. 

  1234. static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,

  1235.                                       int mm_flags)

  1236. {

  1237. #if HAVE_SSE4_EXTERNAL

  1238.     c->vector_clip_int32 = ff_vector_clip_int32_sse4;

  1239. #endif /* HAVE_SSE4_EXTERNAL */

  1240. }

  1241. 

  1242. av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)

  1243. {

  1244.     int mm_flags = av_get_cpu_flags();

  1245. 

  1246. #if HAVE_7REGS && HAVE_INLINE_ASM

  1247.     if (mm_flags & AV_CPU_FLAG_CMOV)

  1248.         c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;

  1249. #endif

  1250. 

  1251.     if (mm_flags & AV_CPU_FLAG_MMX)

  1252.         dsputil_init_mmx(c, avctx, mm_flags);

  1253. 

  1254.     if (mm_flags & AV_CPU_FLAG_MMXEXT)

  1255.         dsputil_init_mmxext(c, avctx, mm_flags);

  1256. 

  1257.     if (mm_flags & AV_CPU_FLAG_SSE)

  1258.         dsputil_init_sse(c, avctx, mm_flags);

  1259. 

  1260.     if (mm_flags & AV_CPU_FLAG_SSE2)

  1261.         dsputil_init_sse2(c, avctx, mm_flags);

  1262. 

  1263.     if (mm_flags & AV_CPU_FLAG_SSSE3)

  1264.         dsputil_init_ssse3(c, avctx, mm_flags);

  1265. 

  1266.     if (mm_flags & AV_CPU_FLAG_SSE4)

  1267.         dsputil_init_sse4(c, avctx, mm_flags);

  1268. 

  1269.     if (CONFIG_ENCODERS)

  1270.         ff_dsputilenc_init_mmx(c, avctx);

  1271. }