falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36408 Commits
32 Branches
303MB
Tree: ded3673d77
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'ded3673d77'
${ noResults }
FFmpeg/libavcodec/x86/dsputil_mmx.c

2230 lines
114KB
Raw Blame History 

  1. /*

  2.  * MMX optimized DSP utils

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  *

  8.  * This file is part of Libav.

  9.  *

  10.  * Libav is free software; you can redistribute it and/or

  11.  * modify it under the terms of the GNU Lesser General Public

  12.  * License as published by the Free Software Foundation; either

  13.  * version 2.1 of the License, or (at your option) any later version.

  14.  *

  15.  * Libav is distributed in the hope that it will be useful,

  16.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  17.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  18.  * Lesser General Public License for more details.

  19.  *

  20.  * You should have received a copy of the GNU Lesser General Public

  21.  * License along with Libav; if not, write to the Free Software

  22.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  23.  */

  24. 

  25. #include "libavutil/cpu.h"

  26. #include "libavutil/x86/asm.h"

  27. #include "libavcodec/dsputil.h"

  28. #include "libavcodec/h264dsp.h"

  29. #include "libavcodec/mpegvideo.h"

  30. #include "libavcodec/simple_idct.h"

  31. #include "dsputil_mmx.h"

  32. #include "idct_xvid.h"

  33. 

  34. //#undef NDEBUG

  35. //#include <assert.h>

  36. 

  37. /* pixel operations */

  38. DECLARE_ALIGNED(8,  const uint64_t, ff_bone) = 0x0101010101010101ULL;

  39. DECLARE_ALIGNED(8,  const uint64_t, ff_wtwo) = 0x0002000200020002ULL;

  40. 

  41. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1)    = { 0x0001000100010001ULL, 0x0001000100010001ULL };

  42. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_2)    = { 0x0002000200020002ULL, 0x0002000200020002ULL };

  43. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_3)    = { 0x0003000300030003ULL, 0x0003000300030003ULL };

  44. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_4)    = { 0x0004000400040004ULL, 0x0004000400040004ULL };

  45. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_5)    = { 0x0005000500050005ULL, 0x0005000500050005ULL };

  46. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_8)    = { 0x0008000800080008ULL, 0x0008000800080008ULL };

  47. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_9)    = { 0x0009000900090009ULL, 0x0009000900090009ULL };

  48. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_15)   =   0x000F000F000F000FULL;

  49. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_16)   = { 0x0010001000100010ULL, 0x0010001000100010ULL };

  50. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_17)   = { 0x0011001100110011ULL, 0x0011001100110011ULL };

  51. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_18)   = { 0x0012001200120012ULL, 0x0012001200120012ULL };

  52. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_20)   =   0x0014001400140014ULL;

  53. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_27)   = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };

  54. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_28)   = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };

  55. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_32)   = { 0x0020002000200020ULL, 0x0020002000200020ULL };

  56. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_42)   =   0x002A002A002A002AULL;

  57. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_53)   =   0x0035003500350035ULL;

  58. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_63)   = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };

  59. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_64)   = { 0x0040004000400040ULL, 0x0040004000400040ULL };

  60. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_96)   =   0x0060006000600060ULL;

  61. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_128)  =   0x0080008000800080ULL;

  62. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_255)  =   0x00ff00ff00ff00ffULL;

  63. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_512)  = { 0x0200020002000200ULL, 0x0200020002000200ULL };

  64. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };

  65. 

  66. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_0)    = { 0x0000000000000000ULL, 0x0000000000000000ULL };

  67. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_1)    = { 0x0101010101010101ULL, 0x0101010101010101ULL };

  68. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_3)    = { 0x0303030303030303ULL, 0x0303030303030303ULL };

  69. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_4)    = { 0x0404040404040404ULL, 0x0404040404040404ULL };

  70. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_7)    =   0x0707070707070707ULL;

  71. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_1F)   =   0x1F1F1F1F1F1F1F1FULL;

  72. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_3F)   =   0x3F3F3F3F3F3F3F3FULL;

  73. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_80)   = { 0x8080808080808080ULL, 0x8080808080808080ULL };

  74. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_81)   =   0x8181818181818181ULL;

  75. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_A1)   = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };

  76. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_F8)   = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };

  77. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_FC)   =   0xFCFCFCFCFCFCFCFCULL;

  78. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_FE)   = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };

  79. 

  80. DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };

  81. DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };

  82. 

  83. #if HAVE_INLINE_ASM

  84. 

  85. #define JUMPALIGN()     __asm__ volatile (".p2align 3"::)

  86. #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)

  87. 

  88. #define MOVQ_BFE(regd)                                  \

  89.     __asm__ volatile (                                  \

  90.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  91.         "paddb   %%"#regd", %%"#regd"   \n\t" ::)

  92. 

  93. #ifndef PIC

  94. #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))

  95. #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))

  96. #else

  97. // for shared library it's better to use this way for accessing constants

  98. // pcmpeqd -> -1

  99. #define MOVQ_BONE(regd)                                 \

  100.     __asm__ volatile (                                  \

  101.         "pcmpeqd  %%"#regd", %%"#regd"  \n\t"           \

  102.         "psrlw          $15, %%"#regd"  \n\t"           \

  103.         "packuswb %%"#regd", %%"#regd"  \n\t" ::)

  104. 

  105. #define MOVQ_WTWO(regd)                                 \

  106.     __asm__ volatile (                                  \

  107.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  108.         "psrlw         $15, %%"#regd"   \n\t"           \

  109.         "psllw          $1, %%"#regd"   \n\t"::)

  110. 

  111. #endif

  112. 

  113. // using regr as temporary and for the output result

  114. // first argument is unmodifed and second is trashed

  115. // regfe is supposed to contain 0xfefefefefefefefe

  116. #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe)                \

  117.     "movq   "#rega", "#regr"            \n\t"                    \

  118.     "pand   "#regb", "#regr"            \n\t"                    \

  119.     "pxor   "#rega", "#regb"            \n\t"                    \

  120.     "pand  "#regfe", "#regb"            \n\t"                    \

  121.     "psrlq       $1, "#regb"            \n\t"                    \

  122.     "paddb  "#regb", "#regr"            \n\t"

  123. 

  124. #define PAVGB_MMX(rega, regb, regr, regfe)                       \

  125.     "movq   "#rega", "#regr"            \n\t"                    \

  126.     "por    "#regb", "#regr"            \n\t"                    \

  127.     "pxor   "#rega", "#regb"            \n\t"                    \

  128.     "pand  "#regfe", "#regb"            \n\t"                    \

  129.     "psrlq       $1, "#regb"            \n\t"                    \

  130.     "psubb  "#regb", "#regr"            \n\t"

  131. 

  132. // mm6 is supposed to contain 0xfefefefefefefefe

  133. #define PAVGBP_MMX_NO_RND(rega, regb, regr,  regc, regd, regp)   \

  134.     "movq  "#rega", "#regr"             \n\t"                    \

  135.     "movq  "#regc", "#regp"             \n\t"                    \

  136.     "pand  "#regb", "#regr"             \n\t"                    \

  137.     "pand  "#regd", "#regp"             \n\t"                    \

  138.     "pxor  "#rega", "#regb"             \n\t"                    \

  139.     "pxor  "#regc", "#regd"             \n\t"                    \

  140.     "pand    %%mm6, "#regb"             \n\t"                    \

  141.     "pand    %%mm6, "#regd"             \n\t"                    \

  142.     "psrlq      $1, "#regb"             \n\t"                    \

  143.     "psrlq      $1, "#regd"             \n\t"                    \

  144.     "paddb "#regb", "#regr"             \n\t"                    \

  145.     "paddb "#regd", "#regp"             \n\t"

  146. 

  147. #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp)           \

  148.     "movq  "#rega", "#regr"             \n\t"                    \

  149.     "movq  "#regc", "#regp"             \n\t"                    \

  150.     "por   "#regb", "#regr"             \n\t"                    \

  151.     "por   "#regd", "#regp"             \n\t"                    \

  152.     "pxor  "#rega", "#regb"             \n\t"                    \

  153.     "pxor  "#regc", "#regd"             \n\t"                    \

  154.     "pand    %%mm6, "#regb"             \n\t"                    \

  155.     "pand    %%mm6, "#regd"             \n\t"                    \

  156.     "psrlq      $1, "#regd"             \n\t"                    \

  157.     "psrlq      $1, "#regb"             \n\t"                    \

  158.     "psubb "#regb", "#regr"             \n\t"                    \

  159.     "psubb "#regd", "#regp"             \n\t"

  160. 

  161. /***********************************/

  162. /* MMX no rounding */

  163. #define NO_RND 1

  164. #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx

  165. #define SET_RND  MOVQ_WONE

  166. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX_NO_RND(a, b, c, d, e, f)

  167. #define PAVGB(a, b, c, e)               PAVGB_MMX_NO_RND(a, b, c, e)

  168. #define OP_AVG(a, b, c, e)              PAVGB_MMX(a, b, c, e)

  169. 

  170. #include "dsputil_rnd_template.c"

  171. 

  172. #undef DEF

  173. #undef SET_RND

  174. #undef PAVGBP

  175. #undef PAVGB

  176. #undef NO_RND

  177. /***********************************/

  178. /* MMX rounding */

  179. 

  180. #define DEF(x, y) x ## _ ## y ## _mmx

  181. #define SET_RND  MOVQ_WTWO

  182. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX(a, b, c, d, e, f)

  183. #define PAVGB(a, b, c, e)               PAVGB_MMX(a, b, c, e)

  184. 

  185. #include "dsputil_rnd_template.c"

  186. 

  187. #undef DEF

  188. #undef SET_RND

  189. #undef PAVGBP

  190. #undef PAVGB

  191. #undef OP_AVG

  192. 

  193. /***********************************/

  194. /* 3Dnow specific */

  195. 

  196. #define DEF(x) x ## _3dnow

  197. #define PAVGB "pavgusb"

  198. #define SKIP_FOR_3DNOW

  199. 

  200. #include "dsputil_avg_template.c"

  201. 

  202. #undef DEF

  203. #undef PAVGB

  204. #undef SKIP_FOR_3DNOW

  205. 

  206. /***********************************/

  207. /* MMXEXT specific */

  208. 

  209. #define DEF(x) x ## _mmxext

  210. 

  211. /* Introduced only in MMXEXT set */

  212. #define PAVGB "pavgb"

  213. 

  214. #include "dsputil_avg_template.c"

  215. 

  216. #undef DEF

  217. #undef PAVGB

  218. 

  219. #define put_no_rnd_pixels16_mmx put_pixels16_mmx

  220. #define put_no_rnd_pixels8_mmx put_pixels8_mmx

  221. #define put_pixels16_mmxext put_pixels16_mmx

  222. #define put_pixels8_mmxext put_pixels8_mmx

  223. #define put_pixels4_mmxext put_pixels4_mmx

  224. #define put_no_rnd_pixels16_mmxext put_no_rnd_pixels16_mmx

  225. #define put_no_rnd_pixels8_mmxext put_no_rnd_pixels8_mmx

  226. 

  227. /***********************************/

  228. /* standard MMX */

  229. 

  230. void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  231.                                int line_size)

  232. {

  233.     const int16_t *p;

  234.     uint8_t *pix;

  235. 

  236.     /* read the pixels */

  237.     p   = block;

  238.     pix = pixels;

  239.     /* unrolled loop */

  240.     __asm__ volatile (

  241.         "movq      (%3), %%mm0          \n\t"

  242.         "movq     8(%3), %%mm1          \n\t"

  243.         "movq    16(%3), %%mm2          \n\t"

  244.         "movq    24(%3), %%mm3          \n\t"

  245.         "movq    32(%3), %%mm4          \n\t"

  246.         "movq    40(%3), %%mm5          \n\t"

  247.         "movq    48(%3), %%mm6          \n\t"

  248.         "movq    56(%3), %%mm7          \n\t"

  249.         "packuswb %%mm1, %%mm0          \n\t"

  250.         "packuswb %%mm3, %%mm2          \n\t"

  251.         "packuswb %%mm5, %%mm4          \n\t"

  252.         "packuswb %%mm7, %%mm6          \n\t"

  253.         "movq     %%mm0, (%0)           \n\t"

  254.         "movq     %%mm2, (%0, %1)       \n\t"

  255.         "movq     %%mm4, (%0, %1, 2)    \n\t"

  256.         "movq     %%mm6, (%0, %2)       \n\t"

  257.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),

  258.            "r"(p)

  259.         : "memory");

  260.     pix += line_size * 4;

  261.     p   += 32;

  262. 

  263.     // if here would be an exact copy of the code above

  264.     // compiler would generate some very strange code

  265.     // thus using "r"

  266.     __asm__ volatile (

  267.         "movq       (%3), %%mm0         \n\t"

  268.         "movq      8(%3), %%mm1         \n\t"

  269.         "movq     16(%3), %%mm2         \n\t"

  270.         "movq     24(%3), %%mm3         \n\t"

  271.         "movq     32(%3), %%mm4         \n\t"

  272.         "movq     40(%3), %%mm5         \n\t"

  273.         "movq     48(%3), %%mm6         \n\t"

  274.         "movq     56(%3), %%mm7         \n\t"

  275.         "packuswb  %%mm1, %%mm0         \n\t"

  276.         "packuswb  %%mm3, %%mm2         \n\t"

  277.         "packuswb  %%mm5, %%mm4         \n\t"

  278.         "packuswb  %%mm7, %%mm6         \n\t"

  279.         "movq      %%mm0, (%0)          \n\t"

  280.         "movq      %%mm2, (%0, %1)      \n\t"

  281.         "movq      %%mm4, (%0, %1, 2)   \n\t"

  282.         "movq      %%mm6, (%0, %2)      \n\t"

  283.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)

  284.         : "memory");

  285. }

  286. 

  287. #define put_signed_pixels_clamped_mmx_half(off)             \

  288.     "movq          "#off"(%2), %%mm1        \n\t"           \

  289.     "movq     16 + "#off"(%2), %%mm2        \n\t"           \

  290.     "movq     32 + "#off"(%2), %%mm3        \n\t"           \

  291.     "movq     48 + "#off"(%2), %%mm4        \n\t"           \

  292.     "packsswb  8 + "#off"(%2), %%mm1        \n\t"           \

  293.     "packsswb 24 + "#off"(%2), %%mm2        \n\t"           \

  294.     "packsswb 40 + "#off"(%2), %%mm3        \n\t"           \

  295.     "packsswb 56 + "#off"(%2), %%mm4        \n\t"           \

  296.     "paddb              %%mm0, %%mm1        \n\t"           \

  297.     "paddb              %%mm0, %%mm2        \n\t"           \

  298.     "paddb              %%mm0, %%mm3        \n\t"           \

  299.     "paddb              %%mm0, %%mm4        \n\t"           \

  300.     "movq               %%mm1, (%0)         \n\t"           \

  301.     "movq               %%mm2, (%0, %3)     \n\t"           \

  302.     "movq               %%mm3, (%0, %3, 2)  \n\t"           \

  303.     "movq               %%mm4, (%0, %1)     \n\t"

  304. 

  305. void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  306.                                       int line_size)

  307. {

  308.     x86_reg line_skip = line_size;

  309.     x86_reg line_skip3;

  310. 

  311.     __asm__ volatile (

  312.         "movq "MANGLE(ff_pb_80)", %%mm0     \n\t"

  313.         "lea         (%3, %3, 2), %1        \n\t"

  314.         put_signed_pixels_clamped_mmx_half(0)

  315.         "lea         (%0, %3, 4), %0        \n\t"

  316.         put_signed_pixels_clamped_mmx_half(64)

  317.         : "+&r"(pixels), "=&r"(line_skip3)

  318.         : "r"(block), "r"(line_skip)

  319.         : "memory");

  320. }

  321. 

  322. void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  323.                                int line_size)

  324. {

  325.     const int16_t *p;

  326.     uint8_t *pix;

  327.     int i;

  328. 

  329.     /* read the pixels */

  330.     p   = block;

  331.     pix = pixels;

  332.     MOVQ_ZERO(mm7);

  333.     i = 4;

  334.     do {

  335.         __asm__ volatile (

  336.             "movq        (%2), %%mm0    \n\t"

  337.             "movq       8(%2), %%mm1    \n\t"

  338.             "movq      16(%2), %%mm2    \n\t"

  339.             "movq      24(%2), %%mm3    \n\t"

  340.             "movq          %0, %%mm4    \n\t"

  341.             "movq          %1, %%mm6    \n\t"

  342.             "movq       %%mm4, %%mm5    \n\t"

  343.             "punpcklbw  %%mm7, %%mm4    \n\t"

  344.             "punpckhbw  %%mm7, %%mm5    \n\t"

  345.             "paddsw     %%mm4, %%mm0    \n\t"

  346.             "paddsw     %%mm5, %%mm1    \n\t"

  347.             "movq       %%mm6, %%mm5    \n\t"

  348.             "punpcklbw  %%mm7, %%mm6    \n\t"

  349.             "punpckhbw  %%mm7, %%mm5    \n\t"

  350.             "paddsw     %%mm6, %%mm2    \n\t"

  351.             "paddsw     %%mm5, %%mm3    \n\t"

  352.             "packuswb   %%mm1, %%mm0    \n\t"

  353.             "packuswb   %%mm3, %%mm2    \n\t"

  354.             "movq       %%mm0, %0       \n\t"

  355.             "movq       %%mm2, %1       \n\t"

  356.             : "+m"(*pix), "+m"(*(pix + line_size))

  357.             : "r"(p)

  358.             : "memory");

  359.         pix += line_size * 2;

  360.         p   += 16;

  361.     } while (--i);

  362. }

  363. 

  364. static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,

  365.                             int line_size, int h)

  366. {

  367.     __asm__ volatile (

  368.         "lea   (%3, %3), %%"REG_a"      \n\t"

  369.         ".p2align     3                 \n\t"

  370.         "1:                             \n\t"

  371.         "movq  (%1    ), %%mm0          \n\t"

  372.         "movq  (%1, %3), %%mm1          \n\t"

  373.         "movq     %%mm0, (%2)           \n\t"

  374.         "movq     %%mm1, (%2, %3)       \n\t"

  375.         "add  %%"REG_a", %1             \n\t"

  376.         "add  %%"REG_a", %2             \n\t"

  377.         "movq  (%1    ), %%mm0          \n\t"

  378.         "movq  (%1, %3), %%mm1          \n\t"

  379.         "movq     %%mm0, (%2)           \n\t"

  380.         "movq     %%mm1, (%2, %3)       \n\t"

  381.         "add  %%"REG_a", %1             \n\t"

  382.         "add  %%"REG_a", %2             \n\t"

  383.         "subl        $4, %0             \n\t"

  384.         "jnz         1b                 \n\t"

  385.         : "+g"(h), "+r"(pixels),  "+r"(block)

  386.         : "r"((x86_reg)line_size)

  387.         : "%"REG_a, "memory"

  388.         );

  389. }

  390. 

  391. static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,

  392.                              int line_size, int h)

  393. {

  394.     __asm__ volatile (

  395.         "lea   (%3, %3), %%"REG_a"      \n\t"

  396.         ".p2align     3                 \n\t"

  397.         "1:                             \n\t"

  398.         "movq  (%1    ), %%mm0          \n\t"

  399.         "movq 8(%1    ), %%mm4          \n\t"

  400.         "movq  (%1, %3), %%mm1          \n\t"

  401.         "movq 8(%1, %3), %%mm5          \n\t"

  402.         "movq     %%mm0,  (%2)          \n\t"

  403.         "movq     %%mm4, 8(%2)          \n\t"

  404.         "movq     %%mm1,  (%2, %3)      \n\t"

  405.         "movq     %%mm5, 8(%2, %3)      \n\t"

  406.         "add  %%"REG_a", %1             \n\t"

  407.         "add  %%"REG_a", %2             \n\t"

  408.         "movq  (%1    ), %%mm0          \n\t"

  409.         "movq 8(%1    ), %%mm4          \n\t"

  410.         "movq  (%1, %3), %%mm1          \n\t"

  411.         "movq 8(%1, %3), %%mm5          \n\t"

  412.         "movq     %%mm0,  (%2)          \n\t"

  413.         "movq     %%mm4, 8(%2)          \n\t"

  414.         "movq     %%mm1,  (%2, %3)      \n\t"

  415.         "movq     %%mm5, 8(%2, %3)      \n\t"

  416.         "add  %%"REG_a", %1             \n\t"

  417.         "add  %%"REG_a", %2             \n\t"

  418.         "subl        $4, %0             \n\t"

  419.         "jnz         1b                 \n\t"

  420.         : "+g"(h), "+r"(pixels),  "+r"(block)

  421.         : "r"((x86_reg)line_size)

  422.         : "%"REG_a, "memory"

  423.         );

  424. }

  425. 

  426. #define CLEAR_BLOCKS(name, n)                           \

  427. static void name(int16_t *blocks)                       \

  428. {                                                       \

  429.     __asm__ volatile (                                  \

  430.         "pxor %%mm7, %%mm7              \n\t"           \

  431.         "mov     %1,        %%"REG_a"   \n\t"           \

  432.         "1:                             \n\t"           \

  433.         "movq %%mm7,   (%0, %%"REG_a")  \n\t"           \

  434.         "movq %%mm7,  8(%0, %%"REG_a")  \n\t"           \

  435.         "movq %%mm7, 16(%0, %%"REG_a")  \n\t"           \

  436.         "movq %%mm7, 24(%0, %%"REG_a")  \n\t"           \

  437.         "add    $32, %%"REG_a"          \n\t"           \

  438.         "js      1b                     \n\t"           \

  439.         :: "r"(((uint8_t *)blocks) + 128 * n),          \

  440.            "i"(-128 * n)                                \

  441.         : "%"REG_a                                      \

  442.         );                                              \

  443. }

  444. CLEAR_BLOCKS(clear_blocks_mmx, 6)

  445. CLEAR_BLOCKS(clear_block_mmx, 1)

  446. 

  447. static void clear_block_sse(int16_t *block)

  448. {

  449.     __asm__ volatile (

  450.         "xorps  %%xmm0, %%xmm0          \n"

  451.         "movaps %%xmm0,    (%0)         \n"

  452.         "movaps %%xmm0,  16(%0)         \n"

  453.         "movaps %%xmm0,  32(%0)         \n"

  454.         "movaps %%xmm0,  48(%0)         \n"

  455.         "movaps %%xmm0,  64(%0)         \n"

  456.         "movaps %%xmm0,  80(%0)         \n"

  457.         "movaps %%xmm0,  96(%0)         \n"

  458.         "movaps %%xmm0, 112(%0)         \n"

  459.         :: "r"(block)

  460.         : "memory"

  461.     );

  462. }

  463. 

  464. static void clear_blocks_sse(int16_t *blocks)

  465. {

  466.     __asm__ volatile (

  467.         "xorps  %%xmm0, %%xmm0              \n"

  468.         "mov        %1,         %%"REG_a"   \n"

  469.         "1:                                 \n"

  470.         "movaps %%xmm0,    (%0, %%"REG_a")  \n"

  471.         "movaps %%xmm0,  16(%0, %%"REG_a")  \n"

  472.         "movaps %%xmm0,  32(%0, %%"REG_a")  \n"

  473.         "movaps %%xmm0,  48(%0, %%"REG_a")  \n"

  474.         "movaps %%xmm0,  64(%0, %%"REG_a")  \n"

  475.         "movaps %%xmm0,  80(%0, %%"REG_a")  \n"

  476.         "movaps %%xmm0,  96(%0, %%"REG_a")  \n"

  477.         "movaps %%xmm0, 112(%0, %%"REG_a")  \n"

  478.         "add      $128,         %%"REG_a"   \n"

  479.         "js         1b                      \n"

  480.         :: "r"(((uint8_t *)blocks) + 128 * 6),

  481.            "i"(-128 * 6)

  482.         : "%"REG_a

  483.     );

  484. }

  485. 

  486. static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)

  487. {

  488.     x86_reg i = 0;

  489.     __asm__ volatile (

  490.         "jmp          2f                \n\t"

  491.         "1:                             \n\t"

  492.         "movq   (%1, %0), %%mm0         \n\t"

  493.         "movq   (%2, %0), %%mm1         \n\t"

  494.         "paddb     %%mm0, %%mm1         \n\t"

  495.         "movq      %%mm1, (%2, %0)      \n\t"

  496.         "movq  8(%1, %0), %%mm0         \n\t"

  497.         "movq  8(%2, %0), %%mm1         \n\t"

  498.         "paddb     %%mm0, %%mm1         \n\t"

  499.         "movq      %%mm1, 8(%2, %0)     \n\t"

  500.         "add         $16, %0            \n\t"

  501.         "2:                             \n\t"

  502.         "cmp          %3, %0            \n\t"

  503.         "js           1b                \n\t"

  504.         : "+r"(i)

  505.         : "r"(src), "r"(dst), "r"((x86_reg)w - 15)

  506.     );

  507.     for ( ; i < w; i++)

  508.         dst[i + 0] += src[i + 0];

  509. }

  510. 

  511. #if HAVE_7REGS

  512. static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,

  513.                                             const uint8_t *diff, int w,

  514.                                             int *left, int *left_top)

  515. {

  516.     x86_reg w2 = -w;

  517.     x86_reg x;

  518.     int l  = *left     & 0xff;

  519.     int tl = *left_top & 0xff;

  520.     int t;

  521.     __asm__ volatile (

  522.         "mov          %7, %3            \n"

  523.         "1:                             \n"

  524.         "movzbl (%3, %4), %2            \n"

  525.         "mov          %2, %k3           \n"

  526.         "sub         %b1, %b3           \n"

  527.         "add         %b0, %b3           \n"

  528.         "mov          %2, %1            \n"

  529.         "cmp          %0, %2            \n"

  530.         "cmovg        %0, %2            \n"

  531.         "cmovg        %1, %0            \n"

  532.         "cmp         %k3, %0            \n"

  533.         "cmovg       %k3, %0            \n"

  534.         "mov          %7, %3            \n"

  535.         "cmp          %2, %0            \n"

  536.         "cmovl        %2, %0            \n"

  537.         "add    (%6, %4), %b0           \n"

  538.         "mov         %b0, (%5, %4)      \n"

  539.         "inc          %4                \n"

  540.         "jl           1b                \n"

  541.         : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)

  542.         : "r"(dst + w), "r"(diff + w), "rm"(top + w)

  543.     );

  544.     *left     = l;

  545.     *left_top = tl;

  546. }

  547. #endif

  548. 

  549. static inline void transpose4x4(uint8_t *dst, uint8_t *src, x86_reg dst_stride, x86_reg src_stride){

  550.     __asm__ volatile( //FIXME could save 1 instruction if done as 8x4 ...

  551.         "movd  (%1), %%mm0              \n\t"

  552.         "add   %3, %1                   \n\t"

  553.         "movd  (%1), %%mm1              \n\t"

  554.         "movd  (%1,%3,1), %%mm2         \n\t"

  555.         "movd  (%1,%3,2), %%mm3         \n\t"

  556.         "punpcklbw %%mm1, %%mm0         \n\t"

  557.         "punpcklbw %%mm3, %%mm2         \n\t"

  558.         "movq %%mm0, %%mm1              \n\t"

  559.         "punpcklwd %%mm2, %%mm0         \n\t"

  560.         "punpckhwd %%mm2, %%mm1         \n\t"

  561.         "movd  %%mm0, (%0)              \n\t"

  562.         "add   %2, %0                   \n\t"

  563.         "punpckhdq %%mm0, %%mm0         \n\t"

  564.         "movd  %%mm0, (%0)              \n\t"

  565.         "movd  %%mm1, (%0,%2,1)         \n\t"

  566.         "punpckhdq %%mm1, %%mm1         \n\t"

  567.         "movd  %%mm1, (%0,%2,2)         \n\t"

  568. 

  569.         :  "+&r" (dst),

  570.            "+&r" (src)

  571.         :  "r" (dst_stride),

  572.            "r" (src_stride)

  573.         :  "memory"

  574.     );

  575. }

  576. 

  577. #define H263_LOOP_FILTER                        \

  578.     "pxor      %%mm7, %%mm7             \n\t"   \

  579.     "movq         %0, %%mm0             \n\t"   \

  580.     "movq         %0, %%mm1             \n\t"   \

  581.     "movq         %3, %%mm2             \n\t"   \

  582.     "movq         %3, %%mm3             \n\t"   \

  583.     "punpcklbw %%mm7, %%mm0             \n\t"   \

  584.     "punpckhbw %%mm7, %%mm1             \n\t"   \

  585.     "punpcklbw %%mm7, %%mm2             \n\t"   \

  586.     "punpckhbw %%mm7, %%mm3             \n\t"   \

  587.     "psubw     %%mm2, %%mm0             \n\t"   \

  588.     "psubw     %%mm3, %%mm1             \n\t"   \

  589.     "movq         %1, %%mm2             \n\t"   \

  590.     "movq         %1, %%mm3             \n\t"   \

  591.     "movq         %2, %%mm4             \n\t"   \

  592.     "movq         %2, %%mm5             \n\t"   \

  593.     "punpcklbw %%mm7, %%mm2             \n\t"   \

  594.     "punpckhbw %%mm7, %%mm3             \n\t"   \

  595.     "punpcklbw %%mm7, %%mm4             \n\t"   \

  596.     "punpckhbw %%mm7, %%mm5             \n\t"   \

  597.     "psubw     %%mm2, %%mm4             \n\t"   \

  598.     "psubw     %%mm3, %%mm5             \n\t"   \

  599.     "psllw        $2, %%mm4             \n\t"   \

  600.     "psllw        $2, %%mm5             \n\t"   \

  601.     "paddw     %%mm0, %%mm4             \n\t"   \

  602.     "paddw     %%mm1, %%mm5             \n\t"   \

  603.     "pxor      %%mm6, %%mm6             \n\t"   \

  604.     "pcmpgtw   %%mm4, %%mm6             \n\t"   \

  605.     "pcmpgtw   %%mm5, %%mm7             \n\t"   \

  606.     "pxor      %%mm6, %%mm4             \n\t"   \

  607.     "pxor      %%mm7, %%mm5             \n\t"   \

  608.     "psubw     %%mm6, %%mm4             \n\t"   \

  609.     "psubw     %%mm7, %%mm5             \n\t"   \

  610.     "psrlw        $3, %%mm4             \n\t"   \

  611.     "psrlw        $3, %%mm5             \n\t"   \

  612.     "packuswb  %%mm5, %%mm4             \n\t"   \

  613.     "packsswb  %%mm7, %%mm6             \n\t"   \

  614.     "pxor      %%mm7, %%mm7             \n\t"   \

  615.     "movd         %4, %%mm2             \n\t"   \

  616.     "punpcklbw %%mm2, %%mm2             \n\t"   \

  617.     "punpcklbw %%mm2, %%mm2             \n\t"   \

  618.     "punpcklbw %%mm2, %%mm2             \n\t"   \

  619.     "psubusb   %%mm4, %%mm2             \n\t"   \

  620.     "movq      %%mm2, %%mm3             \n\t"   \

  621.     "psubusb   %%mm4, %%mm3             \n\t"   \

  622.     "psubb     %%mm3, %%mm2             \n\t"   \

  623.     "movq         %1, %%mm3             \n\t"   \

  624.     "movq         %2, %%mm4             \n\t"   \

  625.     "pxor      %%mm6, %%mm3             \n\t"   \

  626.     "pxor      %%mm6, %%mm4             \n\t"   \

  627.     "paddusb   %%mm2, %%mm3             \n\t"   \

  628.     "psubusb   %%mm2, %%mm4             \n\t"   \

  629.     "pxor      %%mm6, %%mm3             \n\t"   \

  630.     "pxor      %%mm6, %%mm4             \n\t"   \

  631.     "paddusb   %%mm2, %%mm2             \n\t"   \

  632.     "packsswb  %%mm1, %%mm0             \n\t"   \

  633.     "pcmpgtb   %%mm0, %%mm7             \n\t"   \

  634.     "pxor      %%mm7, %%mm0             \n\t"   \

  635.     "psubb     %%mm7, %%mm0             \n\t"   \

  636.     "movq      %%mm0, %%mm1             \n\t"   \

  637.     "psubusb   %%mm2, %%mm0             \n\t"   \

  638.     "psubb     %%mm0, %%mm1             \n\t"   \

  639.     "pand         %5, %%mm1             \n\t"   \

  640.     "psrlw        $2, %%mm1             \n\t"   \

  641.     "pxor      %%mm7, %%mm1             \n\t"   \

  642.     "psubb     %%mm7, %%mm1             \n\t"   \

  643.     "movq         %0, %%mm5             \n\t"   \

  644.     "movq         %3, %%mm6             \n\t"   \

  645.     "psubb     %%mm1, %%mm5             \n\t"   \

  646.     "paddb     %%mm1, %%mm6             \n\t"

  647. 

  648. static void h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale)

  649. {

  650.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  651.         const int strength = ff_h263_loop_filter_strength[qscale];

  652. 

  653.         __asm__ volatile (

  654.             H263_LOOP_FILTER

  655. 

  656.             "movq %%mm3, %1             \n\t"

  657.             "movq %%mm4, %2             \n\t"

  658.             "movq %%mm5, %0             \n\t"

  659.             "movq %%mm6, %3             \n\t"

  660.             : "+m"(*(uint64_t*)(src - 2 * stride)),

  661.               "+m"(*(uint64_t*)(src - 1 * stride)),

  662.               "+m"(*(uint64_t*)(src + 0 * stride)),

  663.               "+m"(*(uint64_t*)(src + 1 * stride))

  664.             : "g"(2 * strength), "m"(ff_pb_FC)

  665.             );

  666.     }

  667. }

  668. 

  669. static void h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale)

  670. {

  671.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  672.         const int strength = ff_h263_loop_filter_strength[qscale];

  673.         DECLARE_ALIGNED(8, uint64_t, temp)[4];

  674.         uint8_t *btemp = (uint8_t*)temp;

  675. 

  676.         src -= 2;

  677. 

  678.         transpose4x4(btemp,     src,              8, stride);

  679.         transpose4x4(btemp + 4, src + 4 * stride, 8, stride);

  680.         __asm__ volatile (

  681.             H263_LOOP_FILTER // 5 3 4 6

  682. 

  683.             : "+m"(temp[0]),

  684.               "+m"(temp[1]),

  685.               "+m"(temp[2]),

  686.               "+m"(temp[3])

  687.             : "g"(2 * strength), "m"(ff_pb_FC)

  688.             );

  689. 

  690.         __asm__ volatile (

  691.             "movq      %%mm5, %%mm1         \n\t"

  692.             "movq      %%mm4, %%mm0         \n\t"

  693.             "punpcklbw %%mm3, %%mm5         \n\t"

  694.             "punpcklbw %%mm6, %%mm4         \n\t"

  695.             "punpckhbw %%mm3, %%mm1         \n\t"

  696.             "punpckhbw %%mm6, %%mm0         \n\t"

  697.             "movq      %%mm5, %%mm3         \n\t"

  698.             "movq      %%mm1, %%mm6         \n\t"

  699.             "punpcklwd %%mm4, %%mm5         \n\t"

  700.             "punpcklwd %%mm0, %%mm1         \n\t"

  701.             "punpckhwd %%mm4, %%mm3         \n\t"

  702.             "punpckhwd %%mm0, %%mm6         \n\t"

  703.             "movd      %%mm5, (%0)          \n\t"

  704.             "punpckhdq %%mm5, %%mm5         \n\t"

  705.             "movd      %%mm5, (%0, %2)      \n\t"

  706.             "movd      %%mm3, (%0, %2, 2)   \n\t"

  707.             "punpckhdq %%mm3, %%mm3         \n\t"

  708.             "movd      %%mm3, (%0, %3)      \n\t"

  709.             "movd      %%mm1, (%1)          \n\t"

  710.             "punpckhdq %%mm1, %%mm1         \n\t"

  711.             "movd      %%mm1, (%1, %2)      \n\t"

  712.             "movd      %%mm6, (%1, %2, 2)   \n\t"

  713.             "punpckhdq %%mm6, %%mm6         \n\t"

  714.             "movd      %%mm6, (%1, %3)      \n\t"

  715.             :: "r"(src),

  716.                "r"(src + 4 * stride),

  717.                "r"((x86_reg)stride),

  718.                "r"((x86_reg)(3 * stride))

  719.             );

  720.     }

  721. }

  722. 

  723. /* Draw the edges of width 'w' of an image of size width, height

  724.  * this MMX version can only handle w == 8 || w == 16. */

  725. static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,

  726.                            int w, int h, int sides)

  727. {

  728.     uint8_t *ptr, *last_line;

  729.     int i;

  730. 

  731.     last_line = buf + (height - 1) * wrap;

  732.     /* left and right */

  733.     ptr = buf;

  734.     if (w == 8) {

  735.         __asm__ volatile (

  736.             "1:                             \n\t"

  737.             "movd            (%0), %%mm0    \n\t"

  738.             "punpcklbw      %%mm0, %%mm0    \n\t"

  739.             "punpcklwd      %%mm0, %%mm0    \n\t"

  740.             "punpckldq      %%mm0, %%mm0    \n\t"

  741.             "movq           %%mm0, -8(%0)   \n\t"

  742.             "movq      -8(%0, %2), %%mm1    \n\t"

  743.             "punpckhbw      %%mm1, %%mm1    \n\t"

  744.             "punpckhwd      %%mm1, %%mm1    \n\t"

  745.             "punpckhdq      %%mm1, %%mm1    \n\t"

  746.             "movq           %%mm1, (%0, %2) \n\t"

  747.             "add               %1, %0       \n\t"

  748.             "cmp               %3, %0       \n\t"

  749.             "jb                1b           \n\t"

  750.             : "+r"(ptr)

  751.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  752.             );

  753.     } else {

  754.         __asm__ volatile (

  755.             "1:                                 \n\t"

  756.             "movd            (%0), %%mm0        \n\t"

  757.             "punpcklbw      %%mm0, %%mm0        \n\t"

  758.             "punpcklwd      %%mm0, %%mm0        \n\t"

  759.             "punpckldq      %%mm0, %%mm0        \n\t"

  760.             "movq           %%mm0, -8(%0)       \n\t"

  761.             "movq           %%mm0, -16(%0)      \n\t"

  762.             "movq      -8(%0, %2), %%mm1        \n\t"

  763.             "punpckhbw      %%mm1, %%mm1        \n\t"

  764.             "punpckhwd      %%mm1, %%mm1        \n\t"

  765.             "punpckhdq      %%mm1, %%mm1        \n\t"

  766.             "movq           %%mm1,  (%0, %2)    \n\t"

  767.             "movq           %%mm1, 8(%0, %2)    \n\t"

  768.             "add               %1, %0           \n\t"

  769.             "cmp               %3, %0           \n\t"

  770.             "jb                1b               \n\t"

  771.             : "+r"(ptr)

  772.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  773.             );

  774.     }

  775. 

  776.     /* top and bottom (and hopefully also the corners) */

  777.     if (sides & EDGE_TOP) {

  778.         for (i = 0; i < h; i += 4) {

  779.             ptr = buf - (i + 1) * wrap - w;

  780.             __asm__ volatile (

  781.                 "1:                             \n\t"

  782.                 "movq (%1, %0), %%mm0           \n\t"

  783.                 "movq    %%mm0, (%0)            \n\t"

  784.                 "movq    %%mm0, (%0, %2)        \n\t"

  785.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  786.                 "movq    %%mm0, (%0, %3)        \n\t"

  787.                 "add        $8, %0              \n\t"

  788.                 "cmp        %4, %0              \n\t"

  789.                 "jb         1b                  \n\t"

  790.                 : "+r"(ptr)

  791.                 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),

  792.                   "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)

  793.                 );

  794.         }

  795.     }

  796. 

  797.     if (sides & EDGE_BOTTOM) {

  798.         for (i = 0; i < h; i += 4) {

  799.             ptr = last_line + (i + 1) * wrap - w;

  800.             __asm__ volatile (

  801.                 "1:                             \n\t"

  802.                 "movq (%1, %0), %%mm0           \n\t"

  803.                 "movq    %%mm0, (%0)            \n\t"

  804.                 "movq    %%mm0, (%0, %2)        \n\t"

  805.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  806.                 "movq    %%mm0, (%0, %3)        \n\t"

  807.                 "add        $8, %0              \n\t"

  808.                 "cmp        %4, %0              \n\t"

  809.                 "jb         1b                  \n\t"

  810.                 : "+r"(ptr)

  811.                 : "r"((x86_reg)last_line - (x86_reg)ptr - w),

  812.                   "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),

  813.                   "r"(ptr + width + 2 * w)

  814.                 );

  815.         }

  816.     }

  817. }

  818. 

  819. #define QPEL_V_LOW(m3, m4, m5, m6, pw_20, pw_3, rnd,                      \

  820.                    in0, in1, in2, in7, out, OP)                           \

  821.     "paddw               "#m4", "#m3"   \n\t" /* x1 */                    \

  822.     "movq   "MANGLE(ff_pw_20)", %%mm4   \n\t" /* 20 */                    \

  823.     "pmullw              "#m3", %%mm4   \n\t" /* 20x1 */                  \

  824.     "movq               "#in7", "#m3"   \n\t" /* d */                     \

  825.     "movq               "#in0", %%mm5   \n\t" /* D */                     \

  826.     "paddw               "#m3", %%mm5   \n\t" /* x4 */                    \

  827.     "psubw               %%mm5, %%mm4   \n\t" /* 20x1 - x4 */             \

  828.     "movq               "#in1", %%mm5   \n\t" /* C */                     \

  829.     "movq               "#in2", %%mm6   \n\t" /* B */                     \

  830.     "paddw               "#m6", %%mm5   \n\t" /* x3 */                    \

  831.     "paddw               "#m5", %%mm6   \n\t" /* x2 */                    \

  832.     "paddw               %%mm6, %%mm6   \n\t" /* 2x2 */                   \

  833.     "psubw               %%mm6, %%mm5   \n\t" /* -2x2 + x3 */             \

  834.     "pmullw  "MANGLE(ff_pw_3)", %%mm5   \n\t" /* -6x2 + 3x3 */            \

  835.     "paddw              "#rnd", %%mm4   \n\t" /* x2 */                    \

  836.     "paddw               %%mm4, %%mm5   \n\t" /* 20x1 - 6x2 + 3x3 - x4 */ \

  837.     "psraw                  $5, %%mm5   \n\t"                             \

  838.     "packuswb            %%mm5, %%mm5   \n\t"                             \

  839.     OP(%%mm5, out, %%mm7, d)

  840. 

  841. #define QPEL_BASE(OPNAME, ROUNDER, RND, OP_MMXEXT)                        \

  842. static void OPNAME ## mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst,         \

  843.                                                     uint8_t *src,         \

  844.                                                     int dstStride,        \

  845.                                                     int srcStride,        \

  846.                                                     int h)                \

  847. {                                                                         \

  848.     uint64_t temp;                                                        \

  849.                                                                           \

  850.     __asm__ volatile (                                                    \

  851.         "pxor      %%mm7, %%mm7             \n\t"                         \

  852.         "1:                                 \n\t"                         \

  853.         "movq       (%0), %%mm0             \n\t" /* ABCDEFGH */          \

  854.         "movq      %%mm0, %%mm1             \n\t" /* ABCDEFGH */          \

  855.         "movq      %%mm0, %%mm2             \n\t" /* ABCDEFGH */          \

  856.         "punpcklbw %%mm7, %%mm0             \n\t" /* 0A0B0C0D */          \

  857.         "punpckhbw %%mm7, %%mm1             \n\t" /* 0E0F0G0H */          \

  858.         "pshufw    $0x90, %%mm0, %%mm5      \n\t" /* 0A0A0B0C */          \

  859.         "pshufw    $0x41, %%mm0, %%mm6      \n\t" /* 0B0A0A0B */          \

  860.         "movq      %%mm2, %%mm3             \n\t" /* ABCDEFGH */          \

  861.         "movq      %%mm2, %%mm4             \n\t" /* ABCDEFGH */          \

  862.         "psllq        $8, %%mm2             \n\t" /* 0ABCDEFG */          \

  863.         "psllq       $16, %%mm3             \n\t" /* 00ABCDEF */          \

  864.         "psllq       $24, %%mm4             \n\t" /* 000ABCDE */          \

  865.         "punpckhbw %%mm7, %%mm2             \n\t" /* 0D0E0F0G */          \

  866.         "punpckhbw %%mm7, %%mm3             \n\t" /* 0C0D0E0F */          \

  867.         "punpckhbw %%mm7, %%mm4             \n\t" /* 0B0C0D0E */          \

  868.         "paddw     %%mm3, %%mm5             \n\t" /* b */                 \

  869.         "paddw     %%mm2, %%mm6             \n\t" /* c */                 \

  870.         "paddw     %%mm5, %%mm5             \n\t" /* 2b */                \

  871.         "psubw     %%mm5, %%mm6             \n\t" /* c - 2b */            \

  872.         "pshufw    $0x06, %%mm0, %%mm5      \n\t" /* 0C0B0A0A */          \

  873.         "pmullw "MANGLE(ff_pw_3)", %%mm6    \n\t" /* 3c - 6b */           \

  874.         "paddw     %%mm4, %%mm0             \n\t" /* a */                 \

  875.         "paddw     %%mm1, %%mm5             \n\t" /* d */                 \

  876.         "pmullw "MANGLE(ff_pw_20)", %%mm0   \n\t" /* 20a */               \

  877.         "psubw     %%mm5, %%mm0             \n\t" /* 20a - d */           \

  878.         "paddw        %6, %%mm6             \n\t"                         \

  879.         "paddw     %%mm6, %%mm0             \n\t" /* 20a - 6b + 3c - d */ \

  880.         "psraw        $5, %%mm0             \n\t"                         \

  881.         "movq      %%mm0, %5                \n\t"                         \

  882.         /* mm1 = EFGH, mm2 = DEFG, mm3 = CDEF, mm4 = BCDE, mm7 = 0 */     \

  883.                                                                           \

  884.         "movq      5(%0), %%mm0             \n\t" /* FGHIJKLM */          \

  885.         "movq      %%mm0, %%mm5             \n\t" /* FGHIJKLM */          \

  886.         "movq      %%mm0, %%mm6             \n\t" /* FGHIJKLM */          \

  887.         "psrlq        $8, %%mm0             \n\t" /* GHIJKLM0 */          \

  888.         "psrlq       $16, %%mm5             \n\t" /* HIJKLM00 */          \

  889.         "punpcklbw %%mm7, %%mm0             \n\t" /* 0G0H0I0J */          \

  890.         "punpcklbw %%mm7, %%mm5             \n\t" /* 0H0I0J0K */          \

  891.         "paddw     %%mm0, %%mm2             \n\t" /* b */                 \

  892.         "paddw     %%mm5, %%mm3             \n\t" /* c */                 \

  893.         "paddw     %%mm2, %%mm2             \n\t" /* 2b */                \

  894.         "psubw     %%mm2, %%mm3             \n\t" /* c - 2b */            \

  895.         "movq      %%mm6, %%mm2             \n\t" /* FGHIJKLM */          \

  896.         "psrlq       $24, %%mm6             \n\t" /* IJKLM000 */          \

  897.         "punpcklbw %%mm7, %%mm2             \n\t" /* 0F0G0H0I */          \

  898.         "punpcklbw %%mm7, %%mm6             \n\t" /* 0I0J0K0L */          \

  899.         "pmullw "MANGLE(ff_pw_3)", %%mm3    \n\t" /* 3c - 6b */           \

  900.         "paddw     %%mm2, %%mm1             \n\t" /* a */                 \

  901.         "paddw     %%mm6, %%mm4             \n\t" /* d */                 \

  902.         "pmullw "MANGLE(ff_pw_20)", %%mm1   \n\t" /* 20a */               \

  903.         "psubw     %%mm4, %%mm3             \n\t" /* - 6b +3c - d */      \

  904.         "paddw        %6, %%mm1             \n\t"                         \

  905.         "paddw     %%mm1, %%mm3             \n\t" /* 20a - 6b +3c - d */  \

  906.         "psraw        $5, %%mm3             \n\t"                         \

  907.         "movq         %5, %%mm1             \n\t"                         \

  908.         "packuswb  %%mm3, %%mm1             \n\t"                         \

  909.         OP_MMXEXT(%%mm1, (%1), %%mm4, q)                                  \

  910.         /* mm0 = GHIJ, mm2 = FGHI, mm5 = HIJK, mm6 = IJKL, mm7 = 0 */     \

  911.                                                                           \

  912.         "movq      9(%0), %%mm1             \n\t" /* JKLMNOPQ */          \

  913.         "movq      %%mm1, %%mm4             \n\t" /* JKLMNOPQ */          \

  914.         "movq      %%mm1, %%mm3             \n\t" /* JKLMNOPQ */          \

  915.         "psrlq        $8, %%mm1             \n\t" /* KLMNOPQ0 */          \

  916.         "psrlq       $16, %%mm4             \n\t" /* LMNOPQ00 */          \

  917.         "punpcklbw %%mm7, %%mm1             \n\t" /* 0K0L0M0N */          \

  918.         "punpcklbw %%mm7, %%mm4             \n\t" /* 0L0M0N0O */          \

  919.         "paddw     %%mm1, %%mm5             \n\t" /* b */                 \

  920.         "paddw     %%mm4, %%mm0             \n\t" /* c */                 \

  921.         "paddw     %%mm5, %%mm5             \n\t" /* 2b */                \

  922.         "psubw     %%mm5, %%mm0             \n\t" /* c - 2b */            \

  923.         "movq      %%mm3, %%mm5             \n\t" /* JKLMNOPQ */          \

  924.         "psrlq       $24, %%mm3             \n\t" /* MNOPQ000 */          \

  925.         "pmullw "MANGLE(ff_pw_3)", %%mm0    \n\t" /* 3c - 6b */           \

  926.         "punpcklbw %%mm7, %%mm3             \n\t" /* 0M0N0O0P */          \

  927.         "paddw     %%mm3, %%mm2             \n\t" /* d */                 \

  928.         "psubw     %%mm2, %%mm0             \n\t" /* -6b + 3c - d */      \

  929.         "movq      %%mm5, %%mm2             \n\t" /* JKLMNOPQ */          \

  930.         "punpcklbw %%mm7, %%mm2             \n\t" /* 0J0K0L0M */          \

  931.         "punpckhbw %%mm7, %%mm5             \n\t" /* 0N0O0P0Q */          \

  932.         "paddw     %%mm2, %%mm6             \n\t" /* a */                 \

  933.         "pmullw "MANGLE(ff_pw_20)", %%mm6   \n\t" /* 20a */               \

  934.         "paddw        %6, %%mm0             \n\t"                         \

  935.         "paddw     %%mm6, %%mm0             \n\t" /* 20a - 6b + 3c - d */ \

  936.         "psraw        $5, %%mm0             \n\t"                         \

  937.         /* mm1 = KLMN, mm2 = JKLM, mm3 = MNOP, */                         \

  938.         /* mm4 = LMNO, mm5 = NOPQ mm7 = 0 */                              \

  939.                                                                           \

  940.         "paddw    %%mm5, %%mm3              \n\t" /* a */                 \

  941.         "pshufw   $0xF9, %%mm5, %%mm6       \n\t" /* 0O0P0Q0Q */          \

  942.         "paddw    %%mm4, %%mm6              \n\t" /* b */                 \

  943.         "pshufw   $0xBE, %%mm5, %%mm4       \n\t" /* 0P0Q0Q0P */          \

  944.         "pshufw   $0x6F, %%mm5, %%mm5       \n\t" /* 0Q0Q0P0O */          \

  945.         "paddw    %%mm1, %%mm4              \n\t" /* c */                 \

  946.         "paddw    %%mm2, %%mm5              \n\t" /* d */                 \

  947.         "paddw    %%mm6, %%mm6              \n\t" /* 2b */                \

  948.         "psubw    %%mm6, %%mm4              \n\t" /* c - 2b */            \

  949.         "pmullw "MANGLE(ff_pw_20)", %%mm3   \n\t" /* 20a */               \

  950.         "pmullw  "MANGLE(ff_pw_3)", %%mm4   \n\t" /* 3c - 6b */           \

  951.         "psubw    %%mm5, %%mm3              \n\t" /* -6b + 3c - d */      \

  952.         "paddw       %6, %%mm4              \n\t"                         \

  953.         "paddw    %%mm3, %%mm4              \n\t" /* 20a - 6b + 3c - d */ \

  954.         "psraw       $5, %%mm4              \n\t"                         \

  955.         "packuswb %%mm4, %%mm0              \n\t"                         \

  956.         OP_MMXEXT(%%mm0, 8(%1), %%mm4, q)                                 \

  957.                                                                           \

  958.         "add         %3, %0                 \n\t"                         \

  959.         "add         %4, %1                 \n\t"                         \

  960.         "decl        %2                     \n\t"                         \

  961.         "jnz         1b                     \n\t"                         \

  962.         : "+a"(src), "+c"(dst), "+D"(h)                                   \

  963.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride),               \

  964.           /* "m"(ff_pw_20), "m"(ff_pw_3), */ "m"(temp), "m"(ROUNDER)      \

  965.         : "memory"                                                        \

  966.         );                                                                \

  967. }                                                                         \

  968.                                                                           \

  969. static void OPNAME ## mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst,          \

  970.                                                    uint8_t *src,          \

  971.                                                    int dstStride,         \

  972.                                                    int srcStride,         \

  973.                                                    int h)                 \

  974. {                                                                         \

  975.     __asm__ volatile (                                                    \

  976.         "pxor      %%mm7, %%mm7             \n\t"                         \

  977.         "1:                                 \n\t"                         \

  978.         "movq       (%0), %%mm0             \n\t" /* ABCDEFGH */          \

  979.         "movq      %%mm0, %%mm1             \n\t" /* ABCDEFGH */          \

  980.         "movq      %%mm0, %%mm2             \n\t" /* ABCDEFGH */          \

  981.         "punpcklbw %%mm7, %%mm0             \n\t" /* 0A0B0C0D */          \

  982.         "punpckhbw %%mm7, %%mm1             \n\t" /* 0E0F0G0H */          \

  983.         "pshufw    $0x90, %%mm0, %%mm5      \n\t" /* 0A0A0B0C */          \

  984.         "pshufw    $0x41, %%mm0, %%mm6      \n\t" /* 0B0A0A0B */          \

  985.         "movq      %%mm2, %%mm3             \n\t" /* ABCDEFGH */          \

  986.         "movq      %%mm2, %%mm4             \n\t" /* ABCDEFGH */          \

  987.         "psllq        $8, %%mm2             \n\t" /* 0ABCDEFG */          \

  988.         "psllq       $16, %%mm3             \n\t" /* 00ABCDEF */          \

  989.         "psllq       $24, %%mm4             \n\t" /* 000ABCDE */          \

  990.         "punpckhbw %%mm7, %%mm2             \n\t" /* 0D0E0F0G */          \

  991.         "punpckhbw %%mm7, %%mm3             \n\t" /* 0C0D0E0F */          \

  992.         "punpckhbw %%mm7, %%mm4             \n\t" /* 0B0C0D0E */          \

  993.         "paddw     %%mm3, %%mm5             \n\t" /* b */                 \

  994.         "paddw     %%mm2, %%mm6             \n\t" /* c */                 \

  995.         "paddw     %%mm5, %%mm5             \n\t" /* 2b */                \

  996.         "psubw     %%mm5, %%mm6             \n\t" /* c - 2b */            \

  997.         "pshufw    $0x06, %%mm0, %%mm5      \n\t" /* 0C0B0A0A */          \

  998.         "pmullw "MANGLE(ff_pw_3)", %%mm6    \n\t" /* 3c - 6b */           \

  999.         "paddw     %%mm4, %%mm0             \n\t" /* a */                 \

  1000.         "paddw     %%mm1, %%mm5             \n\t" /* d */                 \

  1001.         "pmullw "MANGLE(ff_pw_20)", %%mm0   \n\t" /* 20a */               \

  1002.         "psubw     %%mm5, %%mm0             \n\t" /* 20a - d */           \

  1003.         "paddw        %5, %%mm6             \n\t"                         \

  1004.         "paddw     %%mm6, %%mm0             \n\t" /* 20a - 6b + 3c - d */ \

  1005.         "psraw        $5, %%mm0             \n\t"                         \

  1006.         /* mm1 = EFGH, mm2 = DEFG, mm3 = CDEF, mm4 = BCDE, mm7 = 0 */     \

  1007.                                                                           \

  1008.         "movd      5(%0), %%mm5             \n\t" /* FGHI */              \

  1009.         "punpcklbw %%mm7, %%mm5             \n\t" /* 0F0G0H0I */          \

  1010.         "pshufw    $0xF9, %%mm5, %%mm6      \n\t" /* 0G0H0I0I */          \

  1011.         "paddw     %%mm5, %%mm1             \n\t" /* a */                 \

  1012.         "paddw     %%mm6, %%mm2             \n\t" /* b */                 \

  1013.         "pshufw    $0xBE, %%mm5, %%mm6      \n\t" /* 0H0I0I0H */          \

  1014.         "pshufw    $0x6F, %%mm5, %%mm5      \n\t" /* 0I0I0H0G */          \

  1015.         "paddw     %%mm6, %%mm3             \n\t" /* c */                 \

  1016.         "paddw     %%mm5, %%mm4             \n\t" /* d */                 \

  1017.         "paddw     %%mm2, %%mm2             \n\t" /* 2b */                \

  1018.         "psubw     %%mm2, %%mm3             \n\t" /* c - 2b */            \

  1019.         "pmullw "MANGLE(ff_pw_20)", %%mm1   \n\t" /* 20a */               \

  1020.         "pmullw  "MANGLE(ff_pw_3)", %%mm3   \n\t" /* 3c - 6b */           \

  1021.         "psubw     %%mm4, %%mm3             \n\t" /* -6b + 3c - d */      \

  1022.         "paddw        %5, %%mm1             \n\t"                         \

  1023.         "paddw     %%mm1, %%mm3             \n\t" /* 20a - 6b + 3c - d */ \

  1024.         "psraw        $5, %%mm3             \n\t"                         \

  1025.         "packuswb  %%mm3, %%mm0             \n\t"                         \

  1026.         OP_MMXEXT(%%mm0, (%1), %%mm4, q)                                  \

  1027.                                                                           \

  1028.         "add          %3, %0                \n\t"                         \

  1029.         "add          %4, %1                \n\t"                         \

  1030.         "decl         %2                    \n\t"                         \

  1031.         "jnz          1b                    \n\t"                         \

  1032.         : "+a"(src), "+c"(dst), "+d"(h)                                   \

  1033.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride),               \

  1034.           /* "m"(ff_pw_20), "m"(ff_pw_3), */ "m"(ROUNDER)                 \

  1035.         : "memory"                                                        \

  1036.         );                                                                \

  1037. }

  1038. 

  1039. #define QPEL_OP(OPNAME, ROUNDER, RND, OP, MMX)                          \

  1040. static void OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(uint8_t *dst,      \

  1041.                                                      uint8_t *src,      \

  1042.                                                      int dstStride,     \

  1043.                                                      int srcStride)     \

  1044. {                                                                       \

  1045.     uint64_t temp[17 * 4];                                              \

  1046.     uint64_t *temp_ptr = temp;                                          \

  1047.     int count = 17;                                                     \

  1048.                                                                         \

  1049.     /* FIXME unroll */                                                  \

  1050.     __asm__ volatile (                                                  \

  1051.         "pxor      %%mm7, %%mm7             \n\t"                       \

  1052.         "1:                                 \n\t"                       \

  1053.         "movq       (%0), %%mm0             \n\t"                       \

  1054.         "movq       (%0), %%mm1             \n\t"                       \

  1055.         "movq      8(%0), %%mm2             \n\t"                       \

  1056.         "movq      8(%0), %%mm3             \n\t"                       \

  1057.         "punpcklbw %%mm7, %%mm0             \n\t"                       \

  1058.         "punpckhbw %%mm7, %%mm1             \n\t"                       \

  1059.         "punpcklbw %%mm7, %%mm2             \n\t"                       \

  1060.         "punpckhbw %%mm7, %%mm3             \n\t"                       \

  1061.         "movq      %%mm0, (%1)              \n\t"                       \

  1062.         "movq      %%mm1, 17 * 8(%1)        \n\t"                       \

  1063.         "movq      %%mm2, 2 * 17 * 8(%1)    \n\t"                       \

  1064.         "movq      %%mm3, 3 * 17 * 8(%1)    \n\t"                       \

  1065.         "add          $8, %1                \n\t"                       \

  1066.         "add          %3, %0                \n\t"                       \

  1067.         "decl         %2                    \n\t"                       \

  1068.         "jnz          1b                    \n\t"                       \

  1069.         : "+r"(src), "+r"(temp_ptr), "+r"(count)                        \

  1070.         : "r"((x86_reg)srcStride)                                       \

  1071.         : "memory"                                                      \

  1072.         );                                                              \

  1073.                                                                         \

  1074.     temp_ptr = temp;                                                    \

  1075.     count    = 4;                                                       \

  1076.                                                                         \

  1077.     /* FIXME reorder for speed */                                       \

  1078.     __asm__ volatile (                                                  \

  1079.         /* "pxor  %%mm7, %%mm7            \n\t" */                      \

  1080.         "1:                             \n\t"                           \

  1081.         "movq    (%0), %%mm0            \n\t"                           \

  1082.         "movq   8(%0), %%mm1            \n\t"                           \

  1083.         "movq  16(%0), %%mm2            \n\t"                           \

  1084.         "movq  24(%0), %%mm3            \n\t"                           \

  1085.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5, 16(%0),   8(%0),    (%0),  32(%0), (%1),     OP) \

  1086.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5,  8(%0),    (%0),    (%0),  40(%0), (%1, %3), OP) \

  1087.         "add       %4, %1               \n\t"                           \

  1088.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5,   (%0),    (%0),   8(%0),  48(%0), (%1),     OP) \

  1089.                                                                         \

  1090.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5,   (%0),   8(%0),  16(%0),  56(%0), (%1, %3), OP) \

  1091.         "add       %4, %1               \n\t"                           \

  1092.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5,  8(%0),  16(%0),  24(%0),  64(%0), (%1),     OP) \

  1093.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5, 16(%0),  24(%0),  32(%0),  72(%0), (%1, %3), OP) \

  1094.         "add       %4, %1               \n\t"                           \

  1095.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5, 24(%0),  32(%0),  40(%0),  80(%0), (%1),     OP) \

  1096.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5, 32(%0),  40(%0),  48(%0),  88(%0), (%1, %3), OP) \

  1097.         "add       %4, %1               \n\t"                           \

  1098.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5, 40(%0),  48(%0),  56(%0),  96(%0), (%1),     OP) \

  1099.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5, 48(%0),  56(%0),  64(%0), 104(%0), (%1, %3), OP) \

  1100.         "add       %4, %1               \n\t"                           \

  1101.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5, 56(%0),  64(%0),  72(%0), 112(%0), (%1),     OP) \

  1102.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5, 64(%0),  72(%0),  80(%0), 120(%0), (%1, %3), OP) \

  1103.         "add       %4, %1               \n\t"                           \

  1104.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5, 72(%0),  80(%0),  88(%0), 128(%0), (%1),     OP) \

  1105.                                                                         \

  1106.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5, 80(%0),  88(%0),  96(%0), 128(%0), (%1, %3), OP) \

  1107.         "add       %4, %1               \n\t"                           \

  1108.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5, 88(%0),  96(%0), 104(%0), 120(%0), (%1),     OP) \

  1109.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5, 96(%0), 104(%0), 112(%0), 112(%0), (%1, %3), OP) \

  1110.                                                                         \

  1111.         "add     $136, %0               \n\t"                           \

  1112.         "add       %6, %1               \n\t"                           \

  1113.         "decl      %2                   \n\t"                           \

  1114.         "jnz       1b                   \n\t"                           \

  1115.                                                                         \

  1116.         : "+r"(temp_ptr), "+r"(dst), "+g"(count)                        \

  1117.         : "r"((x86_reg)dstStride), "r"(2 * (x86_reg)dstStride),         \

  1118.           /* "m"(ff_pw_20), "m"(ff_pw_3), */ "m"(ROUNDER),              \

  1119.           "g"(4 - 14 * (x86_reg)dstStride)                              \

  1120.         : "memory"                                                      \

  1121.         );                                                              \

  1122. }                                                                       \

  1123.                                                                         \

  1124. static void OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(uint8_t *dst,       \

  1125.                                                     uint8_t *src,       \

  1126.                                                     int dstStride,      \

  1127.                                                     int srcStride)      \

  1128. {                                                                       \

  1129.     uint64_t temp[9 * 2];                                               \

  1130.     uint64_t *temp_ptr = temp;                                          \

  1131.     int count = 9;                                                      \

  1132.                                                                         \

  1133.     /* FIXME unroll */                                                  \

  1134.     __asm__ volatile (                                                  \

  1135.         "pxor      %%mm7, %%mm7         \n\t"                           \

  1136.         "1:                             \n\t"                           \

  1137.         "movq       (%0), %%mm0         \n\t"                           \

  1138.         "movq       (%0), %%mm1         \n\t"                           \

  1139.         "punpcklbw %%mm7, %%mm0         \n\t"                           \

  1140.         "punpckhbw %%mm7, %%mm1         \n\t"                           \

  1141.         "movq      %%mm0, (%1)          \n\t"                           \

  1142.         "movq      %%mm1, 9*8(%1)       \n\t"                           \

  1143.         "add          $8, %1            \n\t"                           \

  1144.         "add          %3, %0            \n\t"                           \

  1145.         "decl         %2                \n\t"                           \

  1146.         "jnz          1b                \n\t"                           \

  1147.         : "+r"(src), "+r"(temp_ptr), "+r"(count)                        \

  1148.         : "r"((x86_reg)srcStride)                                       \

  1149.         : "memory"                                                      \

  1150.         );                                                              \

  1151.                                                                         \

  1152.     temp_ptr = temp;                                                    \

  1153.     count    = 2;                                                       \

  1154.                                                                         \

  1155.     /* FIXME reorder for speed */                                       \

  1156.     __asm__ volatile (                                                  \

  1157.         /* "pxor  %%mm7, %%mm7            \n\t" */                      \

  1158.         "1:                             \n\t"                           \

  1159.         "movq    (%0), %%mm0            \n\t"                           \

  1160.         "movq   8(%0), %%mm1            \n\t"                           \

  1161.         "movq  16(%0), %%mm2            \n\t"                           \

  1162.         "movq  24(%0), %%mm3            \n\t"                           \

  1163.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5, 16(%0),  8(%0),   (%0), 32(%0), (%1), OP)     \

  1164.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5,  8(%0),   (%0),   (%0), 40(%0), (%1, %3), OP) \

  1165.         "add       %4, %1               \n\t"                           \

  1166.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5,   (%0),   (%0),  8(%0), 48(%0), (%1), OP)     \

  1167.                                                                         \

  1168.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5,   (%0),  8(%0), 16(%0), 56(%0), (%1, %3), OP) \

  1169.         "add       %4, %1               \n\t"                           \

  1170.         QPEL_V_LOW(%%mm0, %%mm1, %%mm2, %%mm3, %5, %6, %5,  8(%0), 16(%0), 24(%0), 64(%0), (%1), OP)     \

  1171.                                                                         \

  1172.         QPEL_V_LOW(%%mm1, %%mm2, %%mm3, %%mm0, %5, %6, %5, 16(%0), 24(%0), 32(%0), 64(%0), (%1, %3), OP) \

  1173.         "add       %4, %1               \n\t"                           \

  1174.         QPEL_V_LOW(%%mm2, %%mm3, %%mm0, %%mm1, %5, %6, %5, 24(%0), 32(%0), 40(%0), 56(%0), (%1), OP)     \

  1175.         QPEL_V_LOW(%%mm3, %%mm0, %%mm1, %%mm2, %5, %6, %5, 32(%0), 40(%0), 48(%0), 48(%0), (%1, %3), OP) \

  1176.                                                                         \

  1177.         "add      $72, %0               \n\t"                           \

  1178.         "add       %6, %1               \n\t"                           \

  1179.         "decl      %2                   \n\t"                           \

  1180.         "jnz       1b                   \n\t"                           \

  1181.                                                                         \

  1182.         : "+r"(temp_ptr), "+r"(dst), "+g"(count)                        \

  1183.         : "r"((x86_reg)dstStride), "r"(2 * (x86_reg)dstStride),         \

  1184.           /* "m"(ff_pw_20), "m"(ff_pw_3), */ "m"(ROUNDER),              \

  1185.           "g"(4 - 6 * (x86_reg)dstStride)                               \

  1186.         : "memory"                                                      \

  1187.         );                                                              \

  1188. }                                                                       \

  1189.                                                                         \

  1190. static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \

  1191.                                           int stride)                   \

  1192. {                                                                       \

  1193.     OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);                     \

  1194. }                                                                       \

  1195.                                                                         \

  1196. static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1197.                                          int stride)                    \

  1198. {                                                                       \

  1199.     uint64_t temp[8];                                                   \

  1200.     uint8_t * const half = (uint8_t*)temp;                              \

  1201.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,           \

  1202.                                                 stride, 8);             \

  1203.     OPNAME ## pixels8_l2_ ## MMX(dst, src, half, stride, stride, 8);    \

  1204. }                                                                       \

  1205.                                                                         \

  1206. static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1207.                                          int stride)                    \

  1208. {                                                                       \

  1209.     OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,           \

  1210.                                             stride, 8);                 \

  1211. }                                                                       \

  1212.                                                                         \

  1213. static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1214.                                          int stride)                    \

  1215. {                                                                       \

  1216.     uint64_t temp[8];                                                   \

  1217.     uint8_t * const half = (uint8_t*)temp;                              \

  1218.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,           \

  1219.                                                 stride, 8);             \

  1220.     OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,            \

  1221.                                  stride, 8);                            \

  1222. }                                                                       \

  1223.                                                                         \

  1224. static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1225.                                          int stride)                    \

  1226. {                                                                       \

  1227.     uint64_t temp[8];                                                   \

  1228.     uint8_t * const half = (uint8_t*)temp;                              \

  1229.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, 8, stride);  \

  1230.     OPNAME ## pixels8_l2_ ## MMX(dst, src, half, stride, stride, 8);    \

  1231. }                                                                       \

  1232.                                                                         \

  1233. static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1234.                                          int stride)                    \

  1235. {                                                                       \

  1236.     OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src, stride, stride);  \

  1237. }                                                                       \

  1238.                                                                         \

  1239. static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1240.                                          int stride)                    \

  1241. {                                                                       \

  1242.     uint64_t temp[8];                                                   \

  1243.     uint8_t * const half = (uint8_t*)temp;                              \

  1244.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, 8, stride);  \

  1245.     OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,       \

  1246.                                  stride, 8);                            \

  1247. }                                                                       \

  1248.                                                                         \

  1249. static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1250.                                          int stride)                    \

  1251. {                                                                       \

  1252.     uint64_t half[8 + 9];                                               \

  1253.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1254.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1255.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1256.                                                 stride, 9);             \

  1257.     put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, stride, 9);  \

  1258.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1259.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, stride, 8, 8);     \

  1260. }                                                                       \

  1261.                                                                         \

  1262. static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1263.                                          int stride)                    \

  1264. {                                                                       \

  1265.     uint64_t half[8 + 9];                                               \

  1266.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1267.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1268.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1269.                                                 stride, 9);             \

  1270.     put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,          \

  1271.                                      stride, 9);                        \

  1272.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1273.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, stride, 8, 8);     \

  1274. }                                                                       \

  1275.                                                                         \

  1276. static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1277.                                          int stride)                    \

  1278. {                                                                       \

  1279.     uint64_t half[8 + 9];                                               \

  1280.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1281.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1282.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1283.                                                 stride, 9);             \

  1284.     put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, stride, 9);  \

  1285.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1286.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, stride, 8, 8); \

  1287. }                                                                       \

  1288.                                                                         \

  1289. static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1290.                                          int stride)                    \

  1291. {                                                                       \

  1292.     uint64_t half[8 + 9];                                               \

  1293.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1294.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1295.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1296.                                                 stride, 9);             \

  1297.     put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,          \

  1298.                                      stride, 9);                        \

  1299.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1300.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, stride, 8, 8); \

  1301. }                                                                       \

  1302.                                                                         \

  1303. static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1304.                                          int stride)                    \

  1305. {                                                                       \

  1306.     uint64_t half[8 + 9];                                               \

  1307.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1308.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1309.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1310.                                                 stride, 9);             \

  1311.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1312.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, stride, 8, 8);     \

  1313. }                                                                       \

  1314.                                                                         \

  1315. static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1316.                                          int stride)                    \

  1317. {                                                                       \

  1318.     uint64_t half[8 + 9];                                               \

  1319.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  1320.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1321.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1322.                                                 stride, 9);             \

  1323.     put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);   \

  1324.     OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, stride, 8, 8); \

  1325. }                                                                       \

  1326.                                                                         \

  1327. static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1328.                                          int stride)                    \

  1329. {                                                                       \

  1330.     uint64_t half[8 + 9];                                               \

  1331.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1332.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1333.                                                 stride, 9);             \

  1334.     put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, stride, 9);  \

  1335.     OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, stride, 8);     \

  1336. }                                                                       \

  1337.                                                                         \

  1338. static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1339.                                          int stride)                    \

  1340. {                                                                       \

  1341.     uint64_t half[8 + 9];                                               \

  1342.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1343.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1344.                                                 stride, 9);             \

  1345.     put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,          \

  1346.                                      stride, 9);                        \

  1347.     OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, stride, 8);     \

  1348. }                                                                       \

  1349.                                                                         \

  1350. static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \

  1351.                                          int stride)                    \

  1352. {                                                                       \

  1353.     uint64_t half[9];                                                   \

  1354.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1355.     put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,          \

  1356.                                                 stride, 9);             \

  1357.     OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, stride, 8);     \

  1358. }                                                                       \

  1359.                                                                         \

  1360. static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \

  1361.                                            int stride)                  \

  1362. {                                                                       \

  1363.     OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);                   \

  1364. }                                                                       \

  1365.                                                                         \

  1366. static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1367.                                           int stride)                   \

  1368. {                                                                       \

  1369.     uint64_t temp[32];                                                  \

  1370.     uint8_t * const half = (uint8_t*)temp;                              \

  1371.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,         \

  1372.                                                  stride, 16);           \

  1373.     OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, stride, 16);  \

  1374. }                                                                       \

  1375.                                                                         \

  1376. static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1377.                                           int stride)                   \

  1378. {                                                                       \

  1379.     OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,                  \

  1380.                                              stride, stride, 16);       \

  1381. }                                                                       \

  1382.                                                                         \

  1383. static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1384.                                           int stride)                   \

  1385. {                                                                       \

  1386.     uint64_t temp[32];                                                  \

  1387.     uint8_t * const half = (uint8_t*)temp;                              \

  1388.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,         \

  1389.                                                  stride, 16);           \

  1390.     OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,                   \

  1391.                                   stride, stride, 16);                  \

  1392. }                                                                       \

  1393.                                                                         \

  1394. static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1395.                                           int stride)                   \

  1396. {                                                                       \

  1397.     uint64_t temp[32];                                                  \

  1398.     uint8_t * const half = (uint8_t*)temp;                              \

  1399.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,         \

  1400.                                                  stride);               \

  1401.     OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, stride, 16);  \

  1402. }                                                                       \

  1403.                                                                         \

  1404. static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1405.                                           int stride)                   \

  1406. {                                                                       \

  1407.     OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src, stride, stride); \

  1408. }                                                                       \

  1409.                                                                         \

  1410. static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1411.                                           int stride)                   \

  1412. {                                                                       \

  1413.     uint64_t temp[32];                                                  \

  1414.     uint8_t * const half = (uint8_t*)temp;                              \

  1415.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,         \

  1416.                                                  stride);               \

  1417.     OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,                \

  1418.                                   stride, stride, 16);                  \

  1419. }                                                                       \

  1420.                                                                         \

  1421. static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1422.                                           int stride)                   \

  1423. {                                                                       \

  1424.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1425.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1426.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1427.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1428.                                                  stride, 17);           \

  1429.     put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,            \

  1430.                                       stride, 17);                      \

  1431.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1432.                                                  16, 16);               \

  1433.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, stride, 16, 16);  \

  1434. }                                                                       \

  1435.                                                                         \

  1436. static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1437.                                           int stride)                   \

  1438. {                                                                       \

  1439.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1440.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1441.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1442.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1443.                                                  stride, 17);           \

  1444.     put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,        \

  1445.                                       stride, 17);                      \

  1446.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1447.                                                  16, 16);               \

  1448.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, stride, 16, 16);  \

  1449. }                                                                       \

  1450.                                                                         \

  1451. static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1452.                                           int stride)                   \

  1453. {                                                                       \

  1454.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1455.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1456.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1457.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1458.                                                  stride, 17);           \

  1459.     put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,            \

  1460.                                       stride, 17);                      \

  1461.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1462.                                                  16, 16);               \

  1463.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, stride,      \

  1464.                                   16, 16);                              \

  1465. }                                                                       \

  1466.                                                                         \

  1467. static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1468.                                           int stride)                   \

  1469. {                                                                       \

  1470.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1471.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1472.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1473.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1474.                                                  stride, 17);           \

  1475.     put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,        \

  1476.                                       stride, 17);                      \

  1477.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1478.                                                  16, 16);               \

  1479.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, stride,      \

  1480.                                   16, 16);                              \

  1481. }                                                                       \

  1482.                                                                         \

  1483. static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1484.                                           int stride)                   \

  1485. {                                                                       \

  1486.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1487.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1488.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1489.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1490.                                                  stride, 17);           \

  1491.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1492.                                                  16, 16);               \

  1493.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, stride, 16, 16);  \

  1494. }                                                                       \

  1495.                                                                         \

  1496. static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1497.                                           int stride)                   \

  1498. {                                                                       \

  1499.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1500.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1501.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1502.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1503.                                                  stride, 17);           \

  1504.     put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,         \

  1505.                                                  16, 16);               \

  1506.     OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, stride,      \

  1507.                                   16, 16);                              \

  1508. }                                                                       \

  1509.                                                                         \

  1510. static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1511.                                           int stride)                   \

  1512. {                                                                       \

  1513.     uint64_t half[17 * 2];                                              \

  1514.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1515.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1516.                                                  stride, 17);           \

  1517.     put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,            \

  1518.                                       stride, 17);                      \

  1519.     OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, stride, 16);   \

  1520. }                                                                       \

  1521.                                                                         \

  1522. static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1523.                                           int stride)                   \

  1524. {                                                                       \

  1525.     uint64_t half[17 * 2];                                              \

  1526.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1527.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1528.                                                  stride, 17);           \

  1529.     put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,        \

  1530.                                       stride, 17);                      \

  1531.     OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, stride, 16);   \

  1532. }                                                                       \

  1533.                                                                         \

  1534. static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1535.                                           int stride)                   \

  1536. {                                                                       \

  1537.     uint64_t half[17 * 2];                                              \

  1538.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1539.     put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,        \

  1540.                                                  stride, 17);           \

  1541.     OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, stride, 16);   \

  1542. }

  1543. 

  1544. #define PUT_OP(a, b, temp, size)                \

  1545.     "mov"#size"        "#a", "#b"       \n\t"

  1546. 

  1547. #define AVG_MMXEXT_OP(a, b, temp, size)         \

  1548.     "mov"#size"        "#b", "#temp"    \n\t"   \

  1549.     "pavgb          "#temp", "#a"       \n\t"   \

  1550.     "mov"#size"        "#a", "#b"       \n\t"

  1551. 

  1552. QPEL_BASE(put_,        ff_pw_16, _,        PUT_OP)

  1553. QPEL_BASE(avg_,        ff_pw_16, _,        AVG_MMXEXT_OP)

  1554. QPEL_BASE(put_no_rnd_, ff_pw_15, _no_rnd_, PUT_OP)

  1555. QPEL_OP(put_,          ff_pw_16, _,        PUT_OP,        mmxext)

  1556. QPEL_OP(avg_,          ff_pw_16, _,        AVG_MMXEXT_OP, mmxext)

  1557. QPEL_OP(put_no_rnd_,   ff_pw_15, _no_rnd_, PUT_OP,        mmxext)

  1558. 

  1559. void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1560. {

  1561.   put_pixels8_xy2_mmx(dst, src, stride, 8);

  1562. }

  1563. void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1564. {

  1565.   put_pixels16_xy2_mmx(dst, src, stride, 16);

  1566. }

  1567. void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1568. {

  1569.   avg_pixels8_xy2_mmx(dst, src, stride, 8);

  1570. }

  1571. void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1572. {

  1573.   avg_pixels16_xy2_mmx(dst, src, stride, 16);

  1574. }

  1575. 

  1576. static void gmc_mmx(uint8_t *dst, uint8_t *src,

  1577.                     int stride, int h, int ox, int oy,

  1578.                     int dxx, int dxy, int dyx, int dyy,

  1579.                     int shift, int r, int width, int height)

  1580. {

  1581.     const int w    = 8;

  1582.     const int ix   = ox  >> (16 + shift);

  1583.     const int iy   = oy  >> (16 + shift);

  1584.     const int oxs  = ox  >> 4;

  1585.     const int oys  = oy  >> 4;

  1586.     const int dxxs = dxx >> 4;

  1587.     const int dxys = dxy >> 4;

  1588.     const int dyxs = dyx >> 4;

  1589.     const int dyys = dyy >> 4;

  1590.     const uint16_t r4[4]   = { r, r, r, r };

  1591.     const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };

  1592.     const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };

  1593.     const uint64_t shift2 = 2 * shift;

  1594.     int x, y;

  1595. 

  1596.     const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);

  1597.     const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);

  1598.     const int dxh = dxy * (h - 1);

  1599.     const int dyw = dyx * (w - 1);

  1600.     if ( // non-constant fullpel offset (3% of blocks)

  1601.         ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |

  1602.          (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)

  1603.         // uses more than 16 bits of subpel mv (only at huge resolution)

  1604.         || (dxx | dxy | dyx | dyy) & 15 ||

  1605.         (unsigned)ix >= width  - w ||

  1606.         (unsigned)iy >= height - h) {

  1607.         // FIXME could still use mmx for some of the rows

  1608.         ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,

  1609.                  shift, r, width, height);

  1610.         return;

  1611.     }

  1612. 

  1613.     src += ix + iy * stride;

  1614. 

  1615.     __asm__ volatile (

  1616.         "movd         %0, %%mm6         \n\t"

  1617.         "pxor      %%mm7, %%mm7         \n\t"

  1618.         "punpcklwd %%mm6, %%mm6         \n\t"

  1619.         "punpcklwd %%mm6, %%mm6         \n\t"

  1620.         :: "r"(1<<shift)

  1621.     );

  1622. 

  1623.     for (x = 0; x < w; x += 4) {

  1624.         uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),

  1625.                             oxs - dxys + dxxs * (x + 1),

  1626.                             oxs - dxys + dxxs * (x + 2),

  1627.                             oxs - dxys + dxxs * (x + 3) };

  1628.         uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),

  1629.                             oys - dyys + dyxs * (x + 1),

  1630.                             oys - dyys + dyxs * (x + 2),

  1631.                             oys - dyys + dyxs * (x + 3) };

  1632. 

  1633.         for (y = 0; y < h; y++) {

  1634.             __asm__ volatile (

  1635.                 "movq      %0, %%mm4    \n\t"

  1636.                 "movq      %1, %%mm5    \n\t"

  1637.                 "paddw     %2, %%mm4    \n\t"

  1638.                 "paddw     %3, %%mm5    \n\t"

  1639.                 "movq   %%mm4, %0       \n\t"

  1640.                 "movq   %%mm5, %1       \n\t"

  1641.                 "psrlw    $12, %%mm4    \n\t"

  1642.                 "psrlw    $12, %%mm5    \n\t"

  1643.                 : "+m"(*dx4), "+m"(*dy4)

  1644.                 : "m"(*dxy4), "m"(*dyy4)

  1645.             );

  1646. 

  1647.             __asm__ volatile (

  1648.                 "movq      %%mm6, %%mm2 \n\t"

  1649.                 "movq      %%mm6, %%mm1 \n\t"

  1650.                 "psubw     %%mm4, %%mm2 \n\t"

  1651.                 "psubw     %%mm5, %%mm1 \n\t"

  1652.                 "movq      %%mm2, %%mm0 \n\t"

  1653.                 "movq      %%mm4, %%mm3 \n\t"

  1654.                 "pmullw    %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)

  1655.                 "pmullw    %%mm5, %%mm3 \n\t" // dx * dy

  1656.                 "pmullw    %%mm5, %%mm2 \n\t" // (s - dx) * dy

  1657.                 "pmullw    %%mm4, %%mm1 \n\t" // dx * (s - dy)

  1658. 

  1659.                 "movd         %4, %%mm5 \n\t"

  1660.                 "movd         %3, %%mm4 \n\t"

  1661.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1662.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1663.                 "pmullw    %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy

  1664.                 "pmullw    %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy

  1665. 

  1666.                 "movd         %2, %%mm5 \n\t"

  1667.                 "movd         %1, %%mm4 \n\t"

  1668.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1669.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1670.                 "pmullw    %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)

  1671.                 "pmullw    %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)

  1672.                 "paddw        %5, %%mm1 \n\t"

  1673.                 "paddw     %%mm3, %%mm2 \n\t"

  1674.                 "paddw     %%mm1, %%mm0 \n\t"

  1675.                 "paddw     %%mm2, %%mm0 \n\t"

  1676. 

  1677.                 "psrlw        %6, %%mm0 \n\t"

  1678.                 "packuswb  %%mm0, %%mm0 \n\t"

  1679.                 "movd      %%mm0, %0    \n\t"

  1680. 

  1681.                 : "=m"(dst[x + y * stride])

  1682.                 : "m"(src[0]), "m"(src[1]),

  1683.                   "m"(src[stride]), "m"(src[stride + 1]),

  1684.                   "m"(*r4), "m"(shift2)

  1685.             );

  1686.             src += stride;

  1687.         }

  1688.         src += 4 - h * stride;

  1689.     }

  1690. }

  1691. #endif /* HAVE_INLINE_ASM */

  1692. 

  1693. void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1694.                           int line_size, int h);

  1695. void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1696.                           int line_size, int h);

  1697. 

  1698. void ff_put_h264_chroma_mc8_rnd_mmx  (uint8_t *dst, uint8_t *src,

  1699.                                       int stride, int h, int x, int y);

  1700. void ff_avg_h264_chroma_mc8_rnd_mmxext(uint8_t *dst, uint8_t *src,

  1701.                                        int stride, int h, int x, int y);

  1702. void ff_avg_h264_chroma_mc8_rnd_3dnow(uint8_t *dst, uint8_t *src,

  1703.                                       int stride, int h, int x, int y);

  1704. 

  1705. void ff_put_h264_chroma_mc4_mmx      (uint8_t *dst, uint8_t *src,

  1706.                                       int stride, int h, int x, int y);

  1707. void ff_avg_h264_chroma_mc4_mmxext   (uint8_t *dst, uint8_t *src,

  1708.                                       int stride, int h, int x, int y);

  1709. void ff_avg_h264_chroma_mc4_3dnow    (uint8_t *dst, uint8_t *src,

  1710.                                       int stride, int h, int x, int y);

  1711. 

  1712. void ff_put_h264_chroma_mc2_mmxext   (uint8_t *dst, uint8_t *src,

  1713.                                       int stride, int h, int x, int y);

  1714. void ff_avg_h264_chroma_mc2_mmxext   (uint8_t *dst, uint8_t *src,

  1715.                                       int stride, int h, int x, int y);

  1716. 

  1717. void ff_put_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,

  1718.                                       int stride, int h, int x, int y);

  1719. void ff_put_h264_chroma_mc4_ssse3    (uint8_t *dst, uint8_t *src,

  1720.                                       int stride, int h, int x, int y);

  1721. 

  1722. void ff_avg_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,

  1723.                                       int stride, int h, int x, int y);

  1724. void ff_avg_h264_chroma_mc4_ssse3    (uint8_t *dst, uint8_t *src,

  1725.                                       int stride, int h, int x, int y);

  1726. 

  1727. #define CHROMA_MC(OP, NUM, DEPTH, OPT)                                  \

  1728. void ff_ ## OP ## _h264_chroma_mc ## NUM ## _ ## DEPTH ## _ ## OPT      \

  1729.                                       (uint8_t *dst, uint8_t *src,      \

  1730.                                        int stride, int h, int x, int y);

  1731. 

  1732. CHROMA_MC(put, 2, 10, mmxext)

  1733. CHROMA_MC(avg, 2, 10, mmxext)

  1734. CHROMA_MC(put, 4, 10, mmxext)

  1735. CHROMA_MC(avg, 4, 10, mmxext)

  1736. CHROMA_MC(put, 8, 10, sse2)

  1737. CHROMA_MC(avg, 8, 10, sse2)

  1738. CHROMA_MC(put, 8, 10, avx)

  1739. CHROMA_MC(avg, 8, 10, avx)

  1740. 

  1741. #if HAVE_INLINE_ASM

  1742. 

  1743. /* CAVS-specific */

  1744. void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1745. {

  1746.     put_pixels8_mmx(dst, src, stride, 8);

  1747. }

  1748. 

  1749. void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1750. {

  1751.     avg_pixels8_mmx(dst, src, stride, 8);

  1752. }

  1753. 

  1754. void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1755. {

  1756.     put_pixels16_mmx(dst, src, stride, 16);

  1757. }

  1758. 

  1759. void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1760. {

  1761.     avg_pixels16_mmx(dst, src, stride, 16);

  1762. }

  1763. 

  1764. /* VC-1-specific */

  1765. void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,

  1766.                                int stride, int rnd)

  1767. {

  1768.     put_pixels8_mmx(dst, src, stride, 8);

  1769. }

  1770. 

  1771. void ff_avg_vc1_mspel_mc00_mmxext(uint8_t *dst, const uint8_t *src,

  1772.                                   int stride, int rnd)

  1773. {

  1774.     avg_pixels8_mmxext(dst, src, stride, 8);

  1775. }

  1776. 

  1777. static void vector_clipf_sse(float *dst, const float *src,

  1778.                              float min, float max, int len)

  1779. {

  1780.     x86_reg i = (len - 16) * 4;

  1781.     __asm__ volatile (

  1782.         "movss          %3, %%xmm4      \n\t"

  1783.         "movss          %4, %%xmm5      \n\t"

  1784.         "shufps $0, %%xmm4, %%xmm4      \n\t"

  1785.         "shufps $0, %%xmm5, %%xmm5      \n\t"

  1786.         "1:                             \n\t"

  1787.         "movaps   (%2, %0), %%xmm0      \n\t" // 3/1 on intel

  1788.         "movaps 16(%2, %0), %%xmm1      \n\t"

  1789.         "movaps 32(%2, %0), %%xmm2      \n\t"

  1790.         "movaps 48(%2, %0), %%xmm3      \n\t"

  1791.         "maxps      %%xmm4, %%xmm0      \n\t"

  1792.         "maxps      %%xmm4, %%xmm1      \n\t"

  1793.         "maxps      %%xmm4, %%xmm2      \n\t"

  1794.         "maxps      %%xmm4, %%xmm3      \n\t"

  1795.         "minps      %%xmm5, %%xmm0      \n\t"

  1796.         "minps      %%xmm5, %%xmm1      \n\t"

  1797.         "minps      %%xmm5, %%xmm2      \n\t"

  1798.         "minps      %%xmm5, %%xmm3      \n\t"

  1799.         "movaps     %%xmm0,   (%1, %0)  \n\t"

  1800.         "movaps     %%xmm1, 16(%1, %0)  \n\t"

  1801.         "movaps     %%xmm2, 32(%1, %0)  \n\t"

  1802.         "movaps     %%xmm3, 48(%1, %0)  \n\t"

  1803.         "sub           $64, %0          \n\t"

  1804.         "jge            1b              \n\t"

  1805.         : "+&r"(i)

  1806.         : "r"(dst), "r"(src), "m"(min), "m"(max)

  1807.         : "memory"

  1808.     );

  1809. }

  1810. 

  1811. #endif /* HAVE_INLINE_ASM */

  1812. 

  1813. int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,

  1814.                                       int order);

  1815. int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,

  1816.                                     int order);

  1817. int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,

  1818.                                                const int16_t *v3,

  1819.                                                int order, int mul);

  1820. int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,

  1821.                                              const int16_t *v3,

  1822.                                              int order, int mul);

  1823. int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,

  1824.                                               const int16_t *v3,

  1825.                                               int order, int mul);

  1826. 

  1827. void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,

  1828.                                         const int16_t *window, unsigned int len);

  1829. void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,

  1830.                                       const int16_t *window, unsigned int len);

  1831. void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,

  1832.                                   const int16_t *window, unsigned int len);

  1833. void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,

  1834.                                 const int16_t *window, unsigned int len);

  1835. void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,

  1836.                                  const int16_t *window, unsigned int len);

  1837. void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,

  1838.                                       const int16_t *window, unsigned int len);

  1839. 

  1840. void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);

  1841. void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);

  1842. 

  1843. void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,

  1844.                                           const uint8_t *diff, int w,

  1845.                                           int *left, int *left_top);

  1846. int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,

  1847.                                        int w, int left);

  1848. int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,

  1849.                                       int w, int left);

  1850. 

  1851. void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,

  1852.                                    int32_t min, int32_t max, unsigned int len);

  1853. void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,

  1854.                                    int32_t min, int32_t max, unsigned int len);

  1855. void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,

  1856.                                    int32_t min, int32_t max, unsigned int len);

  1857. void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,

  1858.                                    int32_t min, int32_t max, unsigned int len);

  1859. 

  1860. #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \

  1861.     do {                                                                     \

  1862.     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \

  1863.     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \

  1864.     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \

  1865.     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \

  1866.     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \

  1867.     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \

  1868.     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \

  1869.     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \

  1870.     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \

  1871.     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \

  1872.     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \

  1873.     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \

  1874.     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \

  1875.     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \

  1876.     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \

  1877.     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \

  1878.     } while (0)

  1879. 

  1880. #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU)                                     \

  1881.     do {                                                                        \

  1882.         c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _     ## CPU; \

  1883.         c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_  ## CPU; \

  1884.         c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_  ## CPU; \

  1885.         c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \

  1886.     } while (0)

  1887. 

  1888. static void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx, int mm_flags)

  1889. {

  1890.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1891. 

  1892. #if HAVE_INLINE_ASM

  1893.     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;

  1894.     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;

  1895.     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;

  1896. 

  1897.     if (!high_bit_depth) {

  1898.         c->clear_block  = clear_block_mmx;

  1899.         c->clear_blocks = clear_blocks_mmx;

  1900.         c->draw_edges   = draw_edges_mmx;

  1901. 

  1902.         SET_HPEL_FUNCS(put,        [0], 16, mmx);

  1903.         SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);

  1904.         SET_HPEL_FUNCS(avg,        [0], 16, mmx);

  1905.         SET_HPEL_FUNCS(avg_no_rnd,    , 16, mmx);

  1906.         SET_HPEL_FUNCS(put,        [1],  8, mmx);

  1907.         SET_HPEL_FUNCS(put_no_rnd, [1],  8, mmx);

  1908.         SET_HPEL_FUNCS(avg,        [1],  8, mmx);

  1909. 

  1910.         switch (avctx->idct_algo) {

  1911.         case FF_IDCT_AUTO:

  1912.         case FF_IDCT_SIMPLEMMX:

  1913.             c->idct_put              = ff_simple_idct_put_mmx;

  1914.             c->idct_add              = ff_simple_idct_add_mmx;

  1915.             c->idct                  = ff_simple_idct_mmx;

  1916.             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;

  1917.             break;

  1918.         case FF_IDCT_XVIDMMX:

  1919.             c->idct_put              = ff_idct_xvid_mmx_put;

  1920.             c->idct_add              = ff_idct_xvid_mmx_add;

  1921.             c->idct                  = ff_idct_xvid_mmx;

  1922.             break;

  1923.         }

  1924.     }

  1925. 

  1926.     c->gmc = gmc_mmx;

  1927. 

  1928.     c->add_bytes = add_bytes_mmx;

  1929. 

  1930.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  1931.         c->h263_v_loop_filter = h263_v_loop_filter_mmx;

  1932.         c->h263_h_loop_filter = h263_h_loop_filter_mmx;

  1933.     }

  1934. #endif /* HAVE_INLINE_ASM */

  1935. 

  1936. #if HAVE_YASM

  1937.     if (!high_bit_depth && CONFIG_H264CHROMA) {

  1938.         c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_mmx;

  1939.         c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_mmx;

  1940.     }

  1941. 

  1942.     c->vector_clip_int32 = ff_vector_clip_int32_mmx;

  1943. #endif

  1944. 

  1945. }

  1946. 

  1947. static void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,

  1948.                                 int mm_flags)

  1949. {

  1950.     const int bit_depth      = avctx->bits_per_raw_sample;

  1951.     const int high_bit_depth = bit_depth > 8;

  1952. 

  1953. #if HAVE_INLINE_ASM

  1954.     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );

  1955.     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );

  1956. 

  1957.     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );

  1958.     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );

  1959.     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );

  1960.     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );

  1961. 

  1962.     if (!high_bit_depth) {

  1963.         c->put_pixels_tab[0][1] = put_pixels16_x2_mmxext;

  1964.         c->put_pixels_tab[0][2] = put_pixels16_y2_mmxext;

  1965. 

  1966.         c->avg_pixels_tab[0][0] = avg_pixels16_mmxext;

  1967.         c->avg_pixels_tab[0][1] = avg_pixels16_x2_mmxext;

  1968.         c->avg_pixels_tab[0][2] = avg_pixels16_y2_mmxext;

  1969. 

  1970.         c->put_pixels_tab[1][1] = put_pixels8_x2_mmxext;

  1971.         c->put_pixels_tab[1][2] = put_pixels8_y2_mmxext;

  1972. 

  1973.         c->avg_pixels_tab[1][0] = avg_pixels8_mmxext;

  1974.         c->avg_pixels_tab[1][1] = avg_pixels8_x2_mmxext;

  1975.         c->avg_pixels_tab[1][2] = avg_pixels8_y2_mmxext;

  1976.     }

  1977. 

  1978.     if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  1979.         if (!high_bit_depth) {

  1980.             c->put_no_rnd_pixels_tab[0][1] = put_no_rnd_pixels16_x2_mmxext;

  1981.             c->put_no_rnd_pixels_tab[0][2] = put_no_rnd_pixels16_y2_mmxext;

  1982.             c->put_no_rnd_pixels_tab[1][1] = put_no_rnd_pixels8_x2_mmxext;

  1983.             c->put_no_rnd_pixels_tab[1][2] = put_no_rnd_pixels8_y2_mmxext;

  1984. 

  1985.             c->avg_pixels_tab[0][3] = avg_pixels16_xy2_mmxext;

  1986.             c->avg_pixels_tab[1][3] = avg_pixels8_xy2_mmxext;

  1987.         }

  1988.     }

  1989. 

  1990.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1991.         c->idct_put = ff_idct_xvid_mmxext_put;

  1992.         c->idct_add = ff_idct_xvid_mmxext_add;

  1993.         c->idct     = ff_idct_xvid_mmxext;

  1994.     }

  1995. 

  1996.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  1997.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  1998.         c->put_no_rnd_pixels_tab[1][1] = put_no_rnd_pixels8_x2_exact_mmxext;

  1999.         c->put_no_rnd_pixels_tab[1][2] = put_no_rnd_pixels8_y2_exact_mmxext;

  2000.     }

  2001. #endif /* HAVE_INLINE_ASM */

  2002. 

  2003. #if HAVE_MMXEXT_EXTERNAL

  2004.     if (!high_bit_depth && CONFIG_H264CHROMA) {

  2005.         c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_mmxext;

  2006.         c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_mmxext;

  2007.         c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_mmxext;

  2008.         c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_mmxext;

  2009.     }

  2010.     if (bit_depth == 10 && CONFIG_H264CHROMA) {

  2011.         c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_10_mmxext;

  2012.         c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_10_mmxext;

  2013.         c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_10_mmxext;

  2014.         c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_10_mmxext;

  2015.     }

  2016. 

  2017.     /* slower than cmov version on AMD */

  2018.     if (!(mm_flags & AV_CPU_FLAG_3DNOW))

  2019.         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;

  2020. 

  2021.     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;

  2022.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;

  2023. 

  2024.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  2025.         c->apply_window_int16 = ff_apply_window_int16_mmxext;

  2026.     } else {

  2027.         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;

  2028.     }

  2029. #endif /* HAVE_MMXEXT_EXTERNAL */

  2030. }

  2031. 

  2032. static void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,

  2033.                                int mm_flags)

  2034. {

  2035.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  2036. 

  2037. #if HAVE_INLINE_ASM

  2038.     if (!high_bit_depth) {

  2039.         c->put_pixels_tab[0][1] = put_pixels16_x2_3dnow;

  2040.         c->put_pixels_tab[0][2] = put_pixels16_y2_3dnow;

  2041. 

  2042.         c->avg_pixels_tab[0][0] = avg_pixels16_3dnow;

  2043.         c->avg_pixels_tab[0][1] = avg_pixels16_x2_3dnow;

  2044.         c->avg_pixels_tab[0][2] = avg_pixels16_y2_3dnow;

  2045. 

  2046.         c->put_pixels_tab[1][1] = put_pixels8_x2_3dnow;

  2047.         c->put_pixels_tab[1][2] = put_pixels8_y2_3dnow;

  2048. 

  2049.         c->avg_pixels_tab[1][0] = avg_pixels8_3dnow;

  2050.         c->avg_pixels_tab[1][1] = avg_pixels8_x2_3dnow;

  2051.         c->avg_pixels_tab[1][2] = avg_pixels8_y2_3dnow;

  2052. 

  2053.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)){

  2054.             c->put_no_rnd_pixels_tab[0][1] = put_no_rnd_pixels16_x2_3dnow;

  2055.             c->put_no_rnd_pixels_tab[0][2] = put_no_rnd_pixels16_y2_3dnow;

  2056.             c->put_no_rnd_pixels_tab[1][1] = put_no_rnd_pixels8_x2_3dnow;

  2057.             c->put_no_rnd_pixels_tab[1][2] = put_no_rnd_pixels8_y2_3dnow;

  2058. 

  2059.             c->avg_pixels_tab[0][3] = avg_pixels16_xy2_3dnow;

  2060.             c->avg_pixels_tab[1][3] = avg_pixels8_xy2_3dnow;

  2061.         }

  2062.     }

  2063. 

  2064.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  2065.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  2066.         c->put_no_rnd_pixels_tab[1][1] = put_no_rnd_pixels8_x2_exact_3dnow;

  2067.         c->put_no_rnd_pixels_tab[1][2] = put_no_rnd_pixels8_y2_exact_3dnow;

  2068.     }

  2069. #endif /* HAVE_INLINE_ASM */

  2070. 

  2071. #if HAVE_YASM

  2072.     if (!high_bit_depth && CONFIG_H264CHROMA) {

  2073.         c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_3dnow;

  2074.         c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_3dnow;

  2075.     }

  2076. #endif /* HAVE_YASM */

  2077. }

  2078. 

  2079. static void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx, int mm_flags)

  2080. {

  2081.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  2082. 

  2083. #if HAVE_INLINE_ASM

  2084.     if (!high_bit_depth) {

  2085.         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {

  2086.             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */

  2087.             c->clear_block  = clear_block_sse;

  2088.             c->clear_blocks = clear_blocks_sse;

  2089.         }

  2090.     }

  2091. 

  2092.     c->vector_clipf = vector_clipf_sse;

  2093. #endif /* HAVE_INLINE_ASM */

  2094. }

  2095. 

  2096. static void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,

  2097.                               int mm_flags)

  2098. {

  2099.     const int bit_depth      = avctx->bits_per_raw_sample;

  2100.     const int high_bit_depth = bit_depth > 8;

  2101. 

  2102. #if HAVE_SSE2_INLINE

  2103.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  2104.         c->idct_put              = ff_idct_xvid_sse2_put;

  2105.         c->idct_add              = ff_idct_xvid_sse2_add;

  2106.         c->idct                  = ff_idct_xvid_sse2;

  2107.         c->idct_permutation_type = FF_SSE2_IDCT_PERM;

  2108.     }

  2109. #endif /* HAVE_SSE2_INLINE */

  2110. 

  2111. #if HAVE_SSE2_EXTERNAL

  2112.     if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  2113.         // these functions are slower than mmx on AMD, but faster on Intel

  2114.         if (!high_bit_depth) {

  2115.             c->put_pixels_tab[0][0]        = ff_put_pixels16_sse2;

  2116.             c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;

  2117.             c->avg_pixels_tab[0][0]        = ff_avg_pixels16_sse2;

  2118.         }

  2119.     }

  2120. 

  2121.     if (bit_depth == 10) {

  2122.         if (CONFIG_H264CHROMA) {

  2123.             c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_sse2;

  2124.             c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_sse2;

  2125.         }

  2126.     }

  2127. 

  2128.     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;

  2129.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;

  2130.     if (mm_flags & AV_CPU_FLAG_ATOM) {

  2131.         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;

  2132.     } else {

  2133.         c->vector_clip_int32 = ff_vector_clip_int32_sse2;

  2134.     }

  2135.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  2136.         c->apply_window_int16 = ff_apply_window_int16_sse2;

  2137.     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  2138.         c->apply_window_int16 = ff_apply_window_int16_round_sse2;

  2139.     }

  2140.     c->bswap_buf = ff_bswap32_buf_sse2;

  2141. #endif /* HAVE_SSE2_EXTERNAL */

  2142. }

  2143. 

  2144. static void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,

  2145.                                int mm_flags)

  2146. {

  2147. #if HAVE_SSSE3_EXTERNAL

  2148.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  2149. 

  2150.     if (!high_bit_depth && CONFIG_H264CHROMA) {

  2151.         c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_ssse3;

  2152.         c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_ssse3;

  2153.         c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_ssse3;

  2154.         c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_ssse3;

  2155.     }

  2156.     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;

  2157.     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe

  2158.         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;

  2159. 

  2160.     if (mm_flags & AV_CPU_FLAG_ATOM)

  2161.         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;

  2162.     else

  2163.         c->apply_window_int16 = ff_apply_window_int16_ssse3;

  2164.     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit

  2165.         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;

  2166.     c->bswap_buf = ff_bswap32_buf_ssse3;

  2167. #endif /* HAVE_SSSE3_EXTERNAL */

  2168. }

  2169. 

  2170. static void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,

  2171.                               int mm_flags)

  2172. {

  2173. #if HAVE_SSE4_EXTERNAL

  2174.     c->vector_clip_int32 = ff_vector_clip_int32_sse4;

  2175. #endif /* HAVE_SSE4_EXTERNAL */

  2176. }

  2177. 

  2178. static void dsputil_init_avx(DSPContext *c, AVCodecContext *avctx, int mm_flags)

  2179. {

  2180. #if HAVE_AVX_EXTERNAL

  2181.     const int bit_depth = avctx->bits_per_raw_sample;

  2182. 

  2183.     if (bit_depth == 10) {

  2184.         // AVX implies !cache64.

  2185.         // TODO: Port cache(32|64) detection from x264.

  2186.         if (CONFIG_H264CHROMA) {

  2187.             c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_avx;

  2188.             c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_avx;

  2189.         }

  2190.     }

  2191. #endif /* HAVE_AVX_EXTERNAL */

  2192. }

  2193. 

  2194. void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)

  2195. {

  2196.     int mm_flags = av_get_cpu_flags();

  2197. 

  2198. #if HAVE_7REGS && HAVE_INLINE_ASM

  2199.     if (mm_flags & AV_CPU_FLAG_CMOV)

  2200.         c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;

  2201. #endif

  2202. 

  2203.     if (mm_flags & AV_CPU_FLAG_MMX)

  2204.         dsputil_init_mmx(c, avctx, mm_flags);

  2205. 

  2206.     if (mm_flags & AV_CPU_FLAG_MMXEXT)

  2207.         dsputil_init_mmxext(c, avctx, mm_flags);

  2208. 

  2209.     if (mm_flags & AV_CPU_FLAG_3DNOW)

  2210.         dsputil_init_3dnow(c, avctx, mm_flags);

  2211. 

  2212.     if (mm_flags & AV_CPU_FLAG_SSE)

  2213.         dsputil_init_sse(c, avctx, mm_flags);

  2214. 

  2215.     if (mm_flags & AV_CPU_FLAG_SSE2)

  2216.         dsputil_init_sse2(c, avctx, mm_flags);

  2217. 

  2218.     if (mm_flags & AV_CPU_FLAG_SSSE3)

  2219.         dsputil_init_ssse3(c, avctx, mm_flags);

  2220. 

  2221.     if (mm_flags & AV_CPU_FLAG_SSE4)

  2222.         dsputil_init_sse4(c, avctx, mm_flags);

  2223. 

  2224.     if (mm_flags & AV_CPU_FLAG_AVX)

  2225.         dsputil_init_avx(c, avctx, mm_flags);

  2226. 

  2227.     if (CONFIG_ENCODERS)

  2228.         ff_dsputilenc_init_mmx(c, avctx);

  2229. }