falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36686 Commits
32 Branches
444MB
Tree: fc386f2eea
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'fc386f2eea'
${ noResults }
FFmpeg/libavcodec/x86/dsputil_mmx.c

1732 lines
83KB
Raw Blame History 

  1. /*

  2.  * MMX optimized DSP utils

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  *

  8.  * This file is part of Libav.

  9.  *

  10.  * Libav is free software; you can redistribute it and/or

  11.  * modify it under the terms of the GNU Lesser General Public

  12.  * License as published by the Free Software Foundation; either

  13.  * version 2.1 of the License, or (at your option) any later version.

  14.  *

  15.  * Libav is distributed in the hope that it will be useful,

  16.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  17.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  18.  * Lesser General Public License for more details.

  19.  *

  20.  * You should have received a copy of the GNU Lesser General Public

  21.  * License along with Libav; if not, write to the Free Software

  22.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  23.  */

  24. 

  25. #include "libavutil/attributes.h"

  26. #include "libavutil/cpu.h"

  27. #include "libavutil/x86/asm.h"

  28. #include "libavcodec/dsputil.h"

  29. #include "libavcodec/h264dsp.h"

  30. #include "libavcodec/mpegvideo.h"

  31. #include "libavcodec/simple_idct.h"

  32. #include "dsputil_mmx.h"

  33. #include "idct_xvid.h"

  34. 

  35. //#undef NDEBUG

  36. //#include <assert.h>

  37. 

  38. /* pixel operations */

  39. DECLARE_ALIGNED(8,  const uint64_t, ff_bone) = 0x0101010101010101ULL;

  40. DECLARE_ALIGNED(8,  const uint64_t, ff_wtwo) = 0x0002000200020002ULL;

  41. 

  42. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1)    = { 0x0001000100010001ULL, 0x0001000100010001ULL };

  43. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_2)    = { 0x0002000200020002ULL, 0x0002000200020002ULL };

  44. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_3)    = { 0x0003000300030003ULL, 0x0003000300030003ULL };

  45. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_4)    = { 0x0004000400040004ULL, 0x0004000400040004ULL };

  46. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_5)    = { 0x0005000500050005ULL, 0x0005000500050005ULL };

  47. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_8)    = { 0x0008000800080008ULL, 0x0008000800080008ULL };

  48. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_9)    = { 0x0009000900090009ULL, 0x0009000900090009ULL };

  49. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_15)   =   0x000F000F000F000FULL;

  50. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_16)   = { 0x0010001000100010ULL, 0x0010001000100010ULL };

  51. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_17)   = { 0x0011001100110011ULL, 0x0011001100110011ULL };

  52. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_18)   = { 0x0012001200120012ULL, 0x0012001200120012ULL };

  53. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_20)   =   0x0014001400140014ULL;

  54. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_27)   = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };

  55. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_28)   = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };

  56. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_32)   = { 0x0020002000200020ULL, 0x0020002000200020ULL };

  57. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_42)   =   0x002A002A002A002AULL;

  58. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_53)   =   0x0035003500350035ULL;

  59. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_63)   = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };

  60. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_64)   = { 0x0040004000400040ULL, 0x0040004000400040ULL };

  61. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_96)   =   0x0060006000600060ULL;

  62. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_128)  =   0x0080008000800080ULL;

  63. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_255)  =   0x00ff00ff00ff00ffULL;

  64. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_512)  = { 0x0200020002000200ULL, 0x0200020002000200ULL };

  65. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };

  66. 

  67. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_0)    = { 0x0000000000000000ULL, 0x0000000000000000ULL };

  68. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_1)    = { 0x0101010101010101ULL, 0x0101010101010101ULL };

  69. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_3)    = { 0x0303030303030303ULL, 0x0303030303030303ULL };

  70. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_4)    = { 0x0404040404040404ULL, 0x0404040404040404ULL };

  71. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_7)    =   0x0707070707070707ULL;

  72. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_1F)   =   0x1F1F1F1F1F1F1F1FULL;

  73. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_3F)   =   0x3F3F3F3F3F3F3F3FULL;

  74. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_80)   = { 0x8080808080808080ULL, 0x8080808080808080ULL };

  75. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_81)   =   0x8181818181818181ULL;

  76. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_A1)   = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };

  77. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_F8)   = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };

  78. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_FC)   =   0xFCFCFCFCFCFCFCFCULL;

  79. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_FE)   = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };

  80. 

  81. DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };

  82. DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };

  83. 

  84. 

  85. #if HAVE_YASM

  86. void ff_put_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  87.                               ptrdiff_t line_size, int h);

  88. void ff_put_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  89.                              ptrdiff_t line_size, int h);

  90. void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  91.                               int dstStride, int src1Stride, int h);

  92. void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,

  93.                                      uint8_t *src2, int dstStride,

  94.                                      int src1Stride, int h);

  95. void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  96.                               int dstStride, int src1Stride, int h);

  97. void ff_put_pixels16_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  98.                                ptrdiff_t line_size, int h);

  99. void ff_put_pixels16_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  100.                               ptrdiff_t line_size, int h);

  101. void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  102.                                int dstStride, int src1Stride, int h);

  103. void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  104.                                int dstStride, int src1Stride, int h);

  105. void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  106.                                       int dstStride, int src1Stride, int h);

  107. void ff_put_no_rnd_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  108.                                      ptrdiff_t line_size, int h);

  109. void ff_put_no_rnd_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  110.                                     ptrdiff_t line_size, int h);

  111. void ff_put_no_rnd_pixels8_x2_exact_mmxext(uint8_t *block,

  112.                                            const uint8_t *pixels,

  113.                                            ptrdiff_t line_size, int h);

  114. void ff_put_no_rnd_pixels8_x2_exact_3dnow(uint8_t *block,

  115.                                           const uint8_t *pixels,

  116.                                           ptrdiff_t line_size, int h);

  117. void ff_put_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  118.                               ptrdiff_t line_size, int h);

  119. void ff_put_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  120.                              ptrdiff_t line_size, int h);

  121. void ff_put_no_rnd_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  122.                                      ptrdiff_t line_size, int h);

  123. void ff_put_no_rnd_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  124.                                     ptrdiff_t line_size, int h);

  125. void ff_put_no_rnd_pixels8_y2_exact_mmxext(uint8_t *block,

  126.                                            const uint8_t *pixels,

  127.                                            ptrdiff_t line_size, int h);

  128. void ff_put_no_rnd_pixels8_y2_exact_3dnow(uint8_t *block,

  129.                                           const uint8_t *pixels,

  130.                                           ptrdiff_t line_size, int h);

  131. void ff_avg_pixels8_mmxext(uint8_t *block, const uint8_t *pixels,

  132.                            ptrdiff_t line_size, int h);

  133. void ff_avg_pixels8_3dnow(uint8_t *block, const uint8_t *pixels,

  134.                           ptrdiff_t line_size, int h);

  135. void ff_avg_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  136.                               ptrdiff_t line_size, int h);

  137. void ff_avg_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  138.                              ptrdiff_t line_size, int h);

  139. void ff_avg_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  140.                               ptrdiff_t line_size, int h);

  141. void ff_avg_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  142.                              ptrdiff_t line_size, int h);

  143. void ff_avg_pixels8_xy2_mmxext(uint8_t *block, const uint8_t *pixels,

  144.                                ptrdiff_t line_size, int h);

  145. void ff_avg_pixels8_xy2_3dnow(uint8_t *block, const uint8_t *pixels,

  146.                               ptrdiff_t line_size, int h);

  147. 

  148. void ff_put_pixels8_mmxext(uint8_t *block, const uint8_t *pixels, ptrdiff_t line_size, int h);

  149. static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,

  150.                                    ptrdiff_t line_size, int h)

  151. {

  152.     ff_put_pixels8_mmxext(block,     pixels,     line_size, h);

  153.     ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);

  154. }

  155. 

  156. void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  157.                                          int dstStride, int srcStride, int h);

  158. void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  159.                                          int dstStride, int srcStride, int h);

  160. void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  161.                                                  int dstStride, int srcStride,

  162.                                                  int h);

  163. void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  164.                                         int dstStride, int srcStride, int h);

  165. void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  166.                                         int dstStride, int srcStride, int h);

  167. void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  168.                                                 int dstStride, int srcStride,

  169.                                                 int h);

  170. void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  171.                                          int dstStride, int srcStride);

  172. void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  173.                                          int dstStride, int srcStride);

  174. void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  175.                                                  int dstStride, int srcStride);

  176. void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  177.                                         int dstStride, int srcStride);

  178. void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  179.                                         int dstStride, int srcStride);

  180. void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  181.                                                 int dstStride, int srcStride);

  182. #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext

  183. #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext

  184. #endif /* HAVE_YASM */

  185. 

  186. 

  187. #if HAVE_INLINE_ASM

  188. 

  189. #define JUMPALIGN()     __asm__ volatile (".p2align 3"::)

  190. #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)

  191. 

  192. #define MOVQ_BFE(regd)                                  \

  193.     __asm__ volatile (                                  \

  194.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  195.         "paddb   %%"#regd", %%"#regd"   \n\t" ::)

  196. 

  197. #ifndef PIC

  198. #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))

  199. #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))

  200. #else

  201. // for shared library it's better to use this way for accessing constants

  202. // pcmpeqd -> -1

  203. #define MOVQ_BONE(regd)                                 \

  204.     __asm__ volatile (                                  \

  205.         "pcmpeqd  %%"#regd", %%"#regd"  \n\t"           \

  206.         "psrlw          $15, %%"#regd"  \n\t"           \

  207.         "packuswb %%"#regd", %%"#regd"  \n\t" ::)

  208. 

  209. #define MOVQ_WTWO(regd)                                 \

  210.     __asm__ volatile (                                  \

  211.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  212.         "psrlw         $15, %%"#regd"   \n\t"           \

  213.         "psllw          $1, %%"#regd"   \n\t"::)

  214. 

  215. #endif

  216. 

  217. // using regr as temporary and for the output result

  218. // first argument is unmodifed and second is trashed

  219. // regfe is supposed to contain 0xfefefefefefefefe

  220. #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe)                \

  221.     "movq   "#rega", "#regr"            \n\t"                    \

  222.     "pand   "#regb", "#regr"            \n\t"                    \

  223.     "pxor   "#rega", "#regb"            \n\t"                    \

  224.     "pand  "#regfe", "#regb"            \n\t"                    \

  225.     "psrlq       $1, "#regb"            \n\t"                    \

  226.     "paddb  "#regb", "#regr"            \n\t"

  227. 

  228. #define PAVGB_MMX(rega, regb, regr, regfe)                       \

  229.     "movq   "#rega", "#regr"            \n\t"                    \

  230.     "por    "#regb", "#regr"            \n\t"                    \

  231.     "pxor   "#rega", "#regb"            \n\t"                    \

  232.     "pand  "#regfe", "#regb"            \n\t"                    \

  233.     "psrlq       $1, "#regb"            \n\t"                    \

  234.     "psubb  "#regb", "#regr"            \n\t"

  235. 

  236. // mm6 is supposed to contain 0xfefefefefefefefe

  237. #define PAVGBP_MMX_NO_RND(rega, regb, regr,  regc, regd, regp)   \

  238.     "movq  "#rega", "#regr"             \n\t"                    \

  239.     "movq  "#regc", "#regp"             \n\t"                    \

  240.     "pand  "#regb", "#regr"             \n\t"                    \

  241.     "pand  "#regd", "#regp"             \n\t"                    \

  242.     "pxor  "#rega", "#regb"             \n\t"                    \

  243.     "pxor  "#regc", "#regd"             \n\t"                    \

  244.     "pand    %%mm6, "#regb"             \n\t"                    \

  245.     "pand    %%mm6, "#regd"             \n\t"                    \

  246.     "psrlq      $1, "#regb"             \n\t"                    \

  247.     "psrlq      $1, "#regd"             \n\t"                    \

  248.     "paddb "#regb", "#regr"             \n\t"                    \

  249.     "paddb "#regd", "#regp"             \n\t"

  250. 

  251. #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp)           \

  252.     "movq  "#rega", "#regr"             \n\t"                    \

  253.     "movq  "#regc", "#regp"             \n\t"                    \

  254.     "por   "#regb", "#regr"             \n\t"                    \

  255.     "por   "#regd", "#regp"             \n\t"                    \

  256.     "pxor  "#rega", "#regb"             \n\t"                    \

  257.     "pxor  "#regc", "#regd"             \n\t"                    \

  258.     "pand    %%mm6, "#regb"             \n\t"                    \

  259.     "pand    %%mm6, "#regd"             \n\t"                    \

  260.     "psrlq      $1, "#regd"             \n\t"                    \

  261.     "psrlq      $1, "#regb"             \n\t"                    \

  262.     "psubb "#regb", "#regr"             \n\t"                    \

  263.     "psubb "#regd", "#regp"             \n\t"

  264. 

  265. /***********************************/

  266. /* MMX no rounding */

  267. #define NO_RND 1

  268. #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx

  269. #define SET_RND  MOVQ_WONE

  270. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX_NO_RND(a, b, c, d, e, f)

  271. #define PAVGB(a, b, c, e)               PAVGB_MMX_NO_RND(a, b, c, e)

  272. #define OP_AVG(a, b, c, e)              PAVGB_MMX(a, b, c, e)

  273. 

  274. #include "dsputil_rnd_template.c"

  275. 

  276. #undef DEF

  277. #undef SET_RND

  278. #undef PAVGBP

  279. #undef PAVGB

  280. #undef NO_RND

  281. /***********************************/

  282. /* MMX rounding */

  283. 

  284. #define DEF(x, y) x ## _ ## y ## _mmx

  285. #define SET_RND  MOVQ_WTWO

  286. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX(a, b, c, d, e, f)

  287. #define PAVGB(a, b, c, e)               PAVGB_MMX(a, b, c, e)

  288. 

  289. #include "dsputil_rnd_template.c"

  290. 

  291. #undef DEF

  292. #undef SET_RND

  293. #undef PAVGBP

  294. #undef PAVGB

  295. #undef OP_AVG

  296. 

  297. #endif /* HAVE_INLINE_ASM */

  298. 

  299. 

  300. #if HAVE_YASM

  301. #define ff_put_pixels8_mmx ff_put_pixels8_mmxext

  302. 

  303. /***********************************/

  304. /* 3Dnow specific */

  305. 

  306. #define DEF(x) x ## _3dnow

  307. 

  308. #include "dsputil_avg_template.c"

  309. 

  310. #undef DEF

  311. 

  312. /***********************************/

  313. /* MMXEXT specific */

  314. 

  315. #define DEF(x) x ## _mmxext

  316. 

  317. #include "dsputil_avg_template.c"

  318. 

  319. #undef DEF

  320. 

  321. #endif /* HAVE_YASM */

  322. 

  323. 

  324. #if HAVE_INLINE_ASM

  325. #define put_no_rnd_pixels16_mmx put_pixels16_mmx

  326. #define put_no_rnd_pixels8_mmx put_pixels8_mmx

  327. #define put_pixels16_mmxext put_pixels16_mmx

  328. #define put_pixels8_mmxext put_pixels8_mmx

  329. #define put_pixels4_mmxext put_pixels4_mmx

  330. #define put_no_rnd_pixels16_mmxext put_no_rnd_pixels16_mmx

  331. #define put_no_rnd_pixels8_mmxext put_no_rnd_pixels8_mmx

  332. 

  333. /***********************************/

  334. /* standard MMX */

  335. 

  336. void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  337.                                int line_size)

  338. {

  339.     const int16_t *p;

  340.     uint8_t *pix;

  341. 

  342.     /* read the pixels */

  343.     p   = block;

  344.     pix = pixels;

  345.     /* unrolled loop */

  346.     __asm__ volatile (

  347.         "movq      (%3), %%mm0          \n\t"

  348.         "movq     8(%3), %%mm1          \n\t"

  349.         "movq    16(%3), %%mm2          \n\t"

  350.         "movq    24(%3), %%mm3          \n\t"

  351.         "movq    32(%3), %%mm4          \n\t"

  352.         "movq    40(%3), %%mm5          \n\t"

  353.         "movq    48(%3), %%mm6          \n\t"

  354.         "movq    56(%3), %%mm7          \n\t"

  355.         "packuswb %%mm1, %%mm0          \n\t"

  356.         "packuswb %%mm3, %%mm2          \n\t"

  357.         "packuswb %%mm5, %%mm4          \n\t"

  358.         "packuswb %%mm7, %%mm6          \n\t"

  359.         "movq     %%mm0, (%0)           \n\t"

  360.         "movq     %%mm2, (%0, %1)       \n\t"

  361.         "movq     %%mm4, (%0, %1, 2)    \n\t"

  362.         "movq     %%mm6, (%0, %2)       \n\t"

  363.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),

  364.            "r"(p)

  365.         : "memory");

  366.     pix += line_size * 4;

  367.     p   += 32;

  368. 

  369.     // if here would be an exact copy of the code above

  370.     // compiler would generate some very strange code

  371.     // thus using "r"

  372.     __asm__ volatile (

  373.         "movq       (%3), %%mm0         \n\t"

  374.         "movq      8(%3), %%mm1         \n\t"

  375.         "movq     16(%3), %%mm2         \n\t"

  376.         "movq     24(%3), %%mm3         \n\t"

  377.         "movq     32(%3), %%mm4         \n\t"

  378.         "movq     40(%3), %%mm5         \n\t"

  379.         "movq     48(%3), %%mm6         \n\t"

  380.         "movq     56(%3), %%mm7         \n\t"

  381.         "packuswb  %%mm1, %%mm0         \n\t"

  382.         "packuswb  %%mm3, %%mm2         \n\t"

  383.         "packuswb  %%mm5, %%mm4         \n\t"

  384.         "packuswb  %%mm7, %%mm6         \n\t"

  385.         "movq      %%mm0, (%0)          \n\t"

  386.         "movq      %%mm2, (%0, %1)      \n\t"

  387.         "movq      %%mm4, (%0, %1, 2)   \n\t"

  388.         "movq      %%mm6, (%0, %2)      \n\t"

  389.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)

  390.         : "memory");

  391. }

  392. 

  393. #define put_signed_pixels_clamped_mmx_half(off)             \

  394.     "movq          "#off"(%2), %%mm1        \n\t"           \

  395.     "movq     16 + "#off"(%2), %%mm2        \n\t"           \

  396.     "movq     32 + "#off"(%2), %%mm3        \n\t"           \

  397.     "movq     48 + "#off"(%2), %%mm4        \n\t"           \

  398.     "packsswb  8 + "#off"(%2), %%mm1        \n\t"           \

  399.     "packsswb 24 + "#off"(%2), %%mm2        \n\t"           \

  400.     "packsswb 40 + "#off"(%2), %%mm3        \n\t"           \

  401.     "packsswb 56 + "#off"(%2), %%mm4        \n\t"           \

  402.     "paddb              %%mm0, %%mm1        \n\t"           \

  403.     "paddb              %%mm0, %%mm2        \n\t"           \

  404.     "paddb              %%mm0, %%mm3        \n\t"           \

  405.     "paddb              %%mm0, %%mm4        \n\t"           \

  406.     "movq               %%mm1, (%0)         \n\t"           \

  407.     "movq               %%mm2, (%0, %3)     \n\t"           \

  408.     "movq               %%mm3, (%0, %3, 2)  \n\t"           \

  409.     "movq               %%mm4, (%0, %1)     \n\t"

  410. 

  411. void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  412.                                       int line_size)

  413. {

  414.     x86_reg line_skip = line_size;

  415.     x86_reg line_skip3;

  416. 

  417.     __asm__ volatile (

  418.         "movq "MANGLE(ff_pb_80)", %%mm0     \n\t"

  419.         "lea         (%3, %3, 2), %1        \n\t"

  420.         put_signed_pixels_clamped_mmx_half(0)

  421.         "lea         (%0, %3, 4), %0        \n\t"

  422.         put_signed_pixels_clamped_mmx_half(64)

  423.         : "+&r"(pixels), "=&r"(line_skip3)

  424.         : "r"(block), "r"(line_skip)

  425.         : "memory");

  426. }

  427. 

  428. void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  429.                                int line_size)

  430. {

  431.     const int16_t *p;

  432.     uint8_t *pix;

  433.     int i;

  434. 

  435.     /* read the pixels */

  436.     p   = block;

  437.     pix = pixels;

  438.     MOVQ_ZERO(mm7);

  439.     i = 4;

  440.     do {

  441.         __asm__ volatile (

  442.             "movq        (%2), %%mm0    \n\t"

  443.             "movq       8(%2), %%mm1    \n\t"

  444.             "movq      16(%2), %%mm2    \n\t"

  445.             "movq      24(%2), %%mm3    \n\t"

  446.             "movq          %0, %%mm4    \n\t"

  447.             "movq          %1, %%mm6    \n\t"

  448.             "movq       %%mm4, %%mm5    \n\t"

  449.             "punpcklbw  %%mm7, %%mm4    \n\t"

  450.             "punpckhbw  %%mm7, %%mm5    \n\t"

  451.             "paddsw     %%mm4, %%mm0    \n\t"

  452.             "paddsw     %%mm5, %%mm1    \n\t"

  453.             "movq       %%mm6, %%mm5    \n\t"

  454.             "punpcklbw  %%mm7, %%mm6    \n\t"

  455.             "punpckhbw  %%mm7, %%mm5    \n\t"

  456.             "paddsw     %%mm6, %%mm2    \n\t"

  457.             "paddsw     %%mm5, %%mm3    \n\t"

  458.             "packuswb   %%mm1, %%mm0    \n\t"

  459.             "packuswb   %%mm3, %%mm2    \n\t"

  460.             "movq       %%mm0, %0       \n\t"

  461.             "movq       %%mm2, %1       \n\t"

  462.             : "+m"(*pix), "+m"(*(pix + line_size))

  463.             : "r"(p)

  464.             : "memory");

  465.         pix += line_size * 2;

  466.         p   += 16;

  467.     } while (--i);

  468. }

  469. 

  470. static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,

  471.                             int line_size, int h)

  472. {

  473.     __asm__ volatile (

  474.         "lea   (%3, %3), %%"REG_a"      \n\t"

  475.         ".p2align     3                 \n\t"

  476.         "1:                             \n\t"

  477.         "movq  (%1    ), %%mm0          \n\t"

  478.         "movq  (%1, %3), %%mm1          \n\t"

  479.         "movq     %%mm0, (%2)           \n\t"

  480.         "movq     %%mm1, (%2, %3)       \n\t"

  481.         "add  %%"REG_a", %1             \n\t"

  482.         "add  %%"REG_a", %2             \n\t"

  483.         "movq  (%1    ), %%mm0          \n\t"

  484.         "movq  (%1, %3), %%mm1          \n\t"

  485.         "movq     %%mm0, (%2)           \n\t"

  486.         "movq     %%mm1, (%2, %3)       \n\t"

  487.         "add  %%"REG_a", %1             \n\t"

  488.         "add  %%"REG_a", %2             \n\t"

  489.         "subl        $4, %0             \n\t"

  490.         "jnz         1b                 \n\t"

  491.         : "+g"(h), "+r"(pixels),  "+r"(block)

  492.         : "r"((x86_reg)line_size)

  493.         : "%"REG_a, "memory"

  494.         );

  495. }

  496. 

  497. static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,

  498.                              int line_size, int h)

  499. {

  500.     __asm__ volatile (

  501.         "lea   (%3, %3), %%"REG_a"      \n\t"

  502.         ".p2align     3                 \n\t"

  503.         "1:                             \n\t"

  504.         "movq  (%1    ), %%mm0          \n\t"

  505.         "movq 8(%1    ), %%mm4          \n\t"

  506.         "movq  (%1, %3), %%mm1          \n\t"

  507.         "movq 8(%1, %3), %%mm5          \n\t"

  508.         "movq     %%mm0,  (%2)          \n\t"

  509.         "movq     %%mm4, 8(%2)          \n\t"

  510.         "movq     %%mm1,  (%2, %3)      \n\t"

  511.         "movq     %%mm5, 8(%2, %3)      \n\t"

  512.         "add  %%"REG_a", %1             \n\t"

  513.         "add  %%"REG_a", %2             \n\t"

  514.         "movq  (%1    ), %%mm0          \n\t"

  515.         "movq 8(%1    ), %%mm4          \n\t"

  516.         "movq  (%1, %3), %%mm1          \n\t"

  517.         "movq 8(%1, %3), %%mm5          \n\t"

  518.         "movq     %%mm0,  (%2)          \n\t"

  519.         "movq     %%mm4, 8(%2)          \n\t"

  520.         "movq     %%mm1,  (%2, %3)      \n\t"

  521.         "movq     %%mm5, 8(%2, %3)      \n\t"

  522.         "add  %%"REG_a", %1             \n\t"

  523.         "add  %%"REG_a", %2             \n\t"

  524.         "subl        $4, %0             \n\t"

  525.         "jnz         1b                 \n\t"

  526.         : "+g"(h), "+r"(pixels),  "+r"(block)

  527.         : "r"((x86_reg)line_size)

  528.         : "%"REG_a, "memory"

  529.         );

  530. }

  531. 

  532. #define CLEAR_BLOCKS(name, n)                           \

  533. static void name(int16_t *blocks)                       \

  534. {                                                       \

  535.     __asm__ volatile (                                  \

  536.         "pxor %%mm7, %%mm7              \n\t"           \

  537.         "mov     %1,        %%"REG_a"   \n\t"           \

  538.         "1:                             \n\t"           \

  539.         "movq %%mm7,   (%0, %%"REG_a")  \n\t"           \

  540.         "movq %%mm7,  8(%0, %%"REG_a")  \n\t"           \

  541.         "movq %%mm7, 16(%0, %%"REG_a")  \n\t"           \

  542.         "movq %%mm7, 24(%0, %%"REG_a")  \n\t"           \

  543.         "add    $32, %%"REG_a"          \n\t"           \

  544.         "js      1b                     \n\t"           \

  545.         :: "r"(((uint8_t *)blocks) + 128 * n),          \

  546.            "i"(-128 * n)                                \

  547.         : "%"REG_a                                      \

  548.         );                                              \

  549. }

  550. CLEAR_BLOCKS(clear_blocks_mmx, 6)

  551. CLEAR_BLOCKS(clear_block_mmx, 1)

  552. 

  553. static void clear_block_sse(int16_t *block)

  554. {

  555.     __asm__ volatile (

  556.         "xorps  %%xmm0, %%xmm0          \n"

  557.         "movaps %%xmm0,    (%0)         \n"

  558.         "movaps %%xmm0,  16(%0)         \n"

  559.         "movaps %%xmm0,  32(%0)         \n"

  560.         "movaps %%xmm0,  48(%0)         \n"

  561.         "movaps %%xmm0,  64(%0)         \n"

  562.         "movaps %%xmm0,  80(%0)         \n"

  563.         "movaps %%xmm0,  96(%0)         \n"

  564.         "movaps %%xmm0, 112(%0)         \n"

  565.         :: "r"(block)

  566.         : "memory"

  567.     );

  568. }

  569. 

  570. static void clear_blocks_sse(int16_t *blocks)

  571. {

  572.     __asm__ volatile (

  573.         "xorps  %%xmm0, %%xmm0              \n"

  574.         "mov        %1,         %%"REG_a"   \n"

  575.         "1:                                 \n"

  576.         "movaps %%xmm0,    (%0, %%"REG_a")  \n"

  577.         "movaps %%xmm0,  16(%0, %%"REG_a")  \n"

  578.         "movaps %%xmm0,  32(%0, %%"REG_a")  \n"

  579.         "movaps %%xmm0,  48(%0, %%"REG_a")  \n"

  580.         "movaps %%xmm0,  64(%0, %%"REG_a")  \n"

  581.         "movaps %%xmm0,  80(%0, %%"REG_a")  \n"

  582.         "movaps %%xmm0,  96(%0, %%"REG_a")  \n"

  583.         "movaps %%xmm0, 112(%0, %%"REG_a")  \n"

  584.         "add      $128,         %%"REG_a"   \n"

  585.         "js         1b                      \n"

  586.         :: "r"(((uint8_t *)blocks) + 128 * 6),

  587.            "i"(-128 * 6)

  588.         : "%"REG_a

  589.     );

  590. }

  591. 

  592. static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)

  593. {

  594.     x86_reg i = 0;

  595.     __asm__ volatile (

  596.         "jmp          2f                \n\t"

  597.         "1:                             \n\t"

  598.         "movq   (%1, %0), %%mm0         \n\t"

  599.         "movq   (%2, %0), %%mm1         \n\t"

  600.         "paddb     %%mm0, %%mm1         \n\t"

  601.         "movq      %%mm1, (%2, %0)      \n\t"

  602.         "movq  8(%1, %0), %%mm0         \n\t"

  603.         "movq  8(%2, %0), %%mm1         \n\t"

  604.         "paddb     %%mm0, %%mm1         \n\t"

  605.         "movq      %%mm1, 8(%2, %0)     \n\t"

  606.         "add         $16, %0            \n\t"

  607.         "2:                             \n\t"

  608.         "cmp          %3, %0            \n\t"

  609.         "js           1b                \n\t"

  610.         : "+r"(i)

  611.         : "r"(src), "r"(dst), "r"((x86_reg)w - 15)

  612.     );

  613.     for ( ; i < w; i++)

  614.         dst[i + 0] += src[i + 0];

  615. }

  616. 

  617. #if HAVE_7REGS

  618. static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,

  619.                                             const uint8_t *diff, int w,

  620.                                             int *left, int *left_top)

  621. {

  622.     x86_reg w2 = -w;

  623.     x86_reg x;

  624.     int l  = *left     & 0xff;

  625.     int tl = *left_top & 0xff;

  626.     int t;

  627.     __asm__ volatile (

  628.         "mov          %7, %3            \n"

  629.         "1:                             \n"

  630.         "movzbl (%3, %4), %2            \n"

  631.         "mov          %2, %k3           \n"

  632.         "sub         %b1, %b3           \n"

  633.         "add         %b0, %b3           \n"

  634.         "mov          %2, %1            \n"

  635.         "cmp          %0, %2            \n"

  636.         "cmovg        %0, %2            \n"

  637.         "cmovg        %1, %0            \n"

  638.         "cmp         %k3, %0            \n"

  639.         "cmovg       %k3, %0            \n"

  640.         "mov          %7, %3            \n"

  641.         "cmp          %2, %0            \n"

  642.         "cmovl        %2, %0            \n"

  643.         "add    (%6, %4), %b0           \n"

  644.         "mov         %b0, (%5, %4)      \n"

  645.         "inc          %4                \n"

  646.         "jl           1b                \n"

  647.         : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)

  648.         : "r"(dst + w), "r"(diff + w), "rm"(top + w)

  649.     );

  650.     *left     = l;

  651.     *left_top = tl;

  652. }

  653. #endif

  654. #endif /* HAVE_INLINE_ASM */

  655. 

  656. void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  657. void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  658. 

  659. #if HAVE_INLINE_ASM

  660. /* Draw the edges of width 'w' of an image of size width, height

  661.  * this MMX version can only handle w == 8 || w == 16. */

  662. static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,

  663.                            int w, int h, int sides)

  664. {

  665.     uint8_t *ptr, *last_line;

  666.     int i;

  667. 

  668.     last_line = buf + (height - 1) * wrap;

  669.     /* left and right */

  670.     ptr = buf;

  671.     if (w == 8) {

  672.         __asm__ volatile (

  673.             "1:                             \n\t"

  674.             "movd            (%0), %%mm0    \n\t"

  675.             "punpcklbw      %%mm0, %%mm0    \n\t"

  676.             "punpcklwd      %%mm0, %%mm0    \n\t"

  677.             "punpckldq      %%mm0, %%mm0    \n\t"

  678.             "movq           %%mm0, -8(%0)   \n\t"

  679.             "movq      -8(%0, %2), %%mm1    \n\t"

  680.             "punpckhbw      %%mm1, %%mm1    \n\t"

  681.             "punpckhwd      %%mm1, %%mm1    \n\t"

  682.             "punpckhdq      %%mm1, %%mm1    \n\t"

  683.             "movq           %%mm1, (%0, %2) \n\t"

  684.             "add               %1, %0       \n\t"

  685.             "cmp               %3, %0       \n\t"

  686.             "jb                1b           \n\t"

  687.             : "+r"(ptr)

  688.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  689.             );

  690.     } else {

  691.         __asm__ volatile (

  692.             "1:                                 \n\t"

  693.             "movd            (%0), %%mm0        \n\t"

  694.             "punpcklbw      %%mm0, %%mm0        \n\t"

  695.             "punpcklwd      %%mm0, %%mm0        \n\t"

  696.             "punpckldq      %%mm0, %%mm0        \n\t"

  697.             "movq           %%mm0, -8(%0)       \n\t"

  698.             "movq           %%mm0, -16(%0)      \n\t"

  699.             "movq      -8(%0, %2), %%mm1        \n\t"

  700.             "punpckhbw      %%mm1, %%mm1        \n\t"

  701.             "punpckhwd      %%mm1, %%mm1        \n\t"

  702.             "punpckhdq      %%mm1, %%mm1        \n\t"

  703.             "movq           %%mm1,  (%0, %2)    \n\t"

  704.             "movq           %%mm1, 8(%0, %2)    \n\t"

  705.             "add               %1, %0           \n\t"

  706.             "cmp               %3, %0           \n\t"

  707.             "jb                1b               \n\t"

  708.             : "+r"(ptr)

  709.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  710.             );

  711.     }

  712. 

  713.     /* top and bottom (and hopefully also the corners) */

  714.     if (sides & EDGE_TOP) {

  715.         for (i = 0; i < h; i += 4) {

  716.             ptr = buf - (i + 1) * wrap - w;

  717.             __asm__ volatile (

  718.                 "1:                             \n\t"

  719.                 "movq (%1, %0), %%mm0           \n\t"

  720.                 "movq    %%mm0, (%0)            \n\t"

  721.                 "movq    %%mm0, (%0, %2)        \n\t"

  722.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  723.                 "movq    %%mm0, (%0, %3)        \n\t"

  724.                 "add        $8, %0              \n\t"

  725.                 "cmp        %4, %0              \n\t"

  726.                 "jb         1b                  \n\t"

  727.                 : "+r"(ptr)

  728.                 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),

  729.                   "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)

  730.                 );

  731.         }

  732.     }

  733. 

  734.     if (sides & EDGE_BOTTOM) {

  735.         for (i = 0; i < h; i += 4) {

  736.             ptr = last_line + (i + 1) * wrap - w;

  737.             __asm__ volatile (

  738.                 "1:                             \n\t"

  739.                 "movq (%1, %0), %%mm0           \n\t"

  740.                 "movq    %%mm0, (%0)            \n\t"

  741.                 "movq    %%mm0, (%0, %2)        \n\t"

  742.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  743.                 "movq    %%mm0, (%0, %3)        \n\t"

  744.                 "add        $8, %0              \n\t"

  745.                 "cmp        %4, %0              \n\t"

  746.                 "jb         1b                  \n\t"

  747.                 : "+r"(ptr)

  748.                 : "r"((x86_reg)last_line - (x86_reg)ptr - w),

  749.                   "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),

  750.                   "r"(ptr + width + 2 * w)

  751.                 );

  752.         }

  753.     }

  754. }

  755. #endif /* HAVE_INLINE_ASM */

  756. 

  757. 

  758. #if HAVE_YASM

  759. #define QPEL_OP(OPNAME, ROUNDER, RND, OP, MMX)                          \

  760. static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \

  761.                                           int stride)                   \

  762. {                                                                       \

  763.     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \

  764. }                                                                       \

  765.                                                                         \

  766. static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \

  767.                                          int stride)                    \

  768. {                                                                       \

  769.     uint64_t temp[8];                                                   \

  770.     uint8_t * const half = (uint8_t*)temp;                              \

  771.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  772.                                                    stride, 8);          \

  773.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  774.                                         stride, stride, 8);             \

  775. }                                                                       \

  776.                                                                         \

  777. static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \

  778.                                          int stride)                    \

  779. {                                                                       \

  780.     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \

  781.                                                    stride, 8);          \

  782. }                                                                       \

  783.                                                                         \

  784. static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \

  785.                                          int stride)                    \

  786. {                                                                       \

  787.     uint64_t temp[8];                                                   \

  788.     uint8_t * const half = (uint8_t*)temp;                              \

  789.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  790.                                                    stride, 8);          \

  791.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \

  792.                                         stride, 8);                     \

  793. }                                                                       \

  794.                                                                         \

  795. static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \

  796.                                          int stride)                    \

  797. {                                                                       \

  798.     uint64_t temp[8];                                                   \

  799.     uint8_t * const half = (uint8_t*)temp;                              \

  800.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  801.                                                    8, stride);          \

  802.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  803.                                         stride, stride, 8);             \

  804. }                                                                       \

  805.                                                                         \

  806. static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \

  807.                                          int stride)                    \

  808. {                                                                       \

  809.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \

  810.                                                    stride, stride);     \

  811. }                                                                       \

  812.                                                                         \

  813. static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \

  814.                                          int stride)                    \

  815. {                                                                       \

  816.     uint64_t temp[8];                                                   \

  817.     uint8_t * const half = (uint8_t*)temp;                              \

  818.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  819.                                                    8, stride);          \

  820.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\

  821.                                         stride, 8);                     \

  822. }                                                                       \

  823.                                                                         \

  824. static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \

  825.                                          int stride)                    \

  826. {                                                                       \

  827.     uint64_t half[8 + 9];                                               \

  828.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  829.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  830.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  831.                                                    stride, 9);          \

  832.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  833.                                         stride, 9);                     \

  834.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  835.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  836.                                         stride, 8, 8);                  \

  837. }                                                                       \

  838.                                                                         \

  839. static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \

  840.                                          int stride)                    \

  841. {                                                                       \

  842.     uint64_t half[8 + 9];                                               \

  843.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  844.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  845.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  846.                                                    stride, 9);          \

  847.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  848.                                         stride, 9);                     \

  849.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  850.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  851.                                         stride, 8, 8);                  \

  852. }                                                                       \

  853.                                                                         \

  854. static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \

  855.                                          int stride)                    \

  856. {                                                                       \

  857.     uint64_t half[8 + 9];                                               \

  858.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  859.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  860.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  861.                                                    stride, 9);          \

  862.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  863.                                         stride, 9);                     \

  864.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  865.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  866.                                         stride, 8, 8);                  \

  867. }                                                                       \

  868.                                                                         \

  869. static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \

  870.                                          int stride)                    \

  871. {                                                                       \

  872.     uint64_t half[8 + 9];                                               \

  873.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  874.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  875.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  876.                                                    stride, 9);          \

  877.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  878.                                         stride, 9);                     \

  879.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  880.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  881.                                         stride, 8, 8);                  \

  882. }                                                                       \

  883.                                                                         \

  884. static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \

  885.                                          int stride)                    \

  886. {                                                                       \

  887.     uint64_t half[8 + 9];                                               \

  888.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  889.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  890.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  891.                                                    stride, 9);          \

  892.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  893.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  894.                                         stride, 8, 8);                  \

  895. }                                                                       \

  896.                                                                         \

  897. static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \

  898.                                          int stride)                    \

  899. {                                                                       \

  900.     uint64_t half[8 + 9];                                               \

  901.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  902.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  903.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  904.                                                    stride, 9);          \

  905.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  906.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  907.                                         stride, 8, 8);                  \

  908. }                                                                       \

  909.                                                                         \

  910. static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \

  911.                                          int stride)                    \

  912. {                                                                       \

  913.     uint64_t half[8 + 9];                                               \

  914.     uint8_t * const halfH = ((uint8_t*)half);                           \

  915.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  916.                                                    stride, 9);          \

  917.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \

  918.                                         8, stride, 9);                  \

  919.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  920.                                                    stride, 8);          \

  921. }                                                                       \

  922.                                                                         \

  923. static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \

  924.                                          int stride)                    \

  925. {                                                                       \

  926.     uint64_t half[8 + 9];                                               \

  927.     uint8_t * const halfH = ((uint8_t*)half);                           \

  928.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  929.                                                    stride, 9);          \

  930.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  931.                                         stride, 9);                     \

  932.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  933.                                                    stride, 8);          \

  934. }                                                                       \

  935.                                                                         \

  936. static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \

  937.                                          int stride)                    \

  938. {                                                                       \

  939.     uint64_t half[9];                                                   \

  940.     uint8_t * const halfH = ((uint8_t*)half);                           \

  941.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  942.                                                    stride, 9);          \

  943.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  944.                                                    stride, 8);          \

  945. }                                                                       \

  946.                                                                         \

  947. static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \

  948.                                            int stride)                  \

  949. {                                                                       \

  950.     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \

  951. }                                                                       \

  952.                                                                         \

  953. static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \

  954.                                           int stride)                   \

  955. {                                                                       \

  956.     uint64_t temp[32];                                                  \

  957.     uint8_t * const half = (uint8_t*)temp;                              \

  958.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  959.                                                     stride, 16);        \

  960.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  961.                                          stride, 16);                   \

  962. }                                                                       \

  963.                                                                         \

  964. static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \

  965.                                           int stride)                   \

  966. {                                                                       \

  967.     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \

  968.                                                     stride, stride, 16);\

  969. }                                                                       \

  970.                                                                         \

  971. static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \

  972.                                           int stride)                   \

  973. {                                                                       \

  974.     uint64_t temp[32];                                                  \

  975.     uint8_t * const half = (uint8_t*)temp;                              \

  976.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  977.                                                     stride, 16);        \

  978.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \

  979.                                          stride, stride, 16);           \

  980. }                                                                       \

  981.                                                                         \

  982. static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \

  983.                                           int stride)                   \

  984. {                                                                       \

  985.     uint64_t temp[32];                                                  \

  986.     uint8_t * const half = (uint8_t*)temp;                              \

  987.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  988.                                                     stride);            \

  989.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  990.                                          stride, 16);                   \

  991. }                                                                       \

  992.                                                                         \

  993. static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \

  994.                                           int stride)                   \

  995. {                                                                       \

  996.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \

  997.                                                     stride, stride);    \

  998. }                                                                       \

  999.                                                                         \

  1000. static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1001.                                           int stride)                   \

  1002. {                                                                       \

  1003.     uint64_t temp[32];                                                  \

  1004.     uint8_t * const half = (uint8_t*)temp;                              \

  1005.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  1006.                                                     stride);            \

  1007.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \

  1008.                                          stride, stride, 16);           \

  1009. }                                                                       \

  1010.                                                                         \

  1011. static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1012.                                           int stride)                   \

  1013. {                                                                       \

  1014.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1015.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1016.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1017.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1018.                                                     stride, 17);        \

  1019.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1020.                                          stride, 17);                   \

  1021.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1022.                                                     16, 16);            \

  1023.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1024.                                          stride, 16, 16);               \

  1025. }                                                                       \

  1026.                                                                         \

  1027. static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1028.                                           int stride)                   \

  1029. {                                                                       \

  1030.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1031.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1032.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1033.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1034.                                                     stride, 17);        \

  1035.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1036.                                          stride, 17);                   \

  1037.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1038.                                                     16, 16);            \

  1039.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1040.                                          stride, 16, 16);               \

  1041. }                                                                       \

  1042.                                                                         \

  1043. static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1044.                                           int stride)                   \

  1045. {                                                                       \

  1046.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1047.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1048.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1049.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1050.                                                     stride, 17);        \

  1051.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1052.                                          stride, 17);                   \

  1053.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1054.                                                     16, 16);            \

  1055.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1056.                                          stride, 16, 16);               \

  1057. }                                                                       \

  1058.                                                                         \

  1059. static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1060.                                           int stride)                   \

  1061. {                                                                       \

  1062.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1063.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1064.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1065.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1066.                                                     stride, 17);        \

  1067.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1068.                                          stride, 17);                   \

  1069.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1070.                                                     16, 16);            \

  1071.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1072.                                          stride, 16, 16);               \

  1073. }                                                                       \

  1074.                                                                         \

  1075. static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1076.                                           int stride)                   \

  1077. {                                                                       \

  1078.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1079.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1080.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1081.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1082.                                                     stride, 17);        \

  1083.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1084.                                                     16, 16);            \

  1085.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1086.                                          stride, 16, 16);               \

  1087. }                                                                       \

  1088.                                                                         \

  1089. static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1090.                                           int stride)                   \

  1091. {                                                                       \

  1092.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1093.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1094.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1095.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1096.                                                     stride, 17);        \

  1097.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1098.                                                     16, 16);            \

  1099.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1100.                                          stride, 16, 16);               \

  1101. }                                                                       \

  1102.                                                                         \

  1103. static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1104.                                           int stride)                   \

  1105. {                                                                       \

  1106.     uint64_t half[17 * 2];                                              \

  1107.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1108.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1109.                                                     stride, 17);        \

  1110.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1111.                                          stride, 17);                   \

  1112.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1113.                                                     stride, 16);        \

  1114. }                                                                       \

  1115.                                                                         \

  1116. static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1117.                                           int stride)                   \

  1118. {                                                                       \

  1119.     uint64_t half[17 * 2];                                              \

  1120.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1121.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1122.                                                     stride, 17);        \

  1123.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1124.                                          stride, 17);                   \

  1125.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1126.                                                     stride, 16);        \

  1127. }                                                                       \

  1128.                                                                         \

  1129. static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1130.                                           int stride)                   \

  1131. {                                                                       \

  1132.     uint64_t half[17 * 2];                                              \

  1133.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1134.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1135.                                                     stride, 17);        \

  1136.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1137.                                                     stride, 16);        \

  1138. }

  1139. 

  1140. #define PUT_OP(a, b, temp, size)                \

  1141.     "mov"#size"        "#a", "#b"       \n\t"

  1142. 

  1143. #define AVG_MMXEXT_OP(a, b, temp, size)         \

  1144.     "mov"#size"        "#b", "#temp"    \n\t"   \

  1145.     "pavgb          "#temp", "#a"       \n\t"   \

  1146.     "mov"#size"        "#a", "#b"       \n\t"

  1147. 

  1148. QPEL_OP(put_,          ff_pw_16, _,        PUT_OP,        mmxext)

  1149. QPEL_OP(avg_,          ff_pw_16, _,        AVG_MMXEXT_OP, mmxext)

  1150. QPEL_OP(put_no_rnd_,   ff_pw_15, _no_rnd_, PUT_OP,        mmxext)

  1151. #endif /* HAVE_YASM */

  1152. 

  1153. 

  1154. #if HAVE_INLINE_ASM

  1155. void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1156. {

  1157.   put_pixels8_xy2_mmx(dst, src, stride, 8);

  1158. }

  1159. void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1160. {

  1161.   put_pixels16_xy2_mmx(dst, src, stride, 16);

  1162. }

  1163. void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1164. {

  1165.   avg_pixels8_xy2_mmx(dst, src, stride, 8);

  1166. }

  1167. void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)

  1168. {

  1169.   avg_pixels16_xy2_mmx(dst, src, stride, 16);

  1170. }

  1171. 

  1172. static void gmc_mmx(uint8_t *dst, uint8_t *src,

  1173.                     int stride, int h, int ox, int oy,

  1174.                     int dxx, int dxy, int dyx, int dyy,

  1175.                     int shift, int r, int width, int height)

  1176. {

  1177.     const int w    = 8;

  1178.     const int ix   = ox  >> (16 + shift);

  1179.     const int iy   = oy  >> (16 + shift);

  1180.     const int oxs  = ox  >> 4;

  1181.     const int oys  = oy  >> 4;

  1182.     const int dxxs = dxx >> 4;

  1183.     const int dxys = dxy >> 4;

  1184.     const int dyxs = dyx >> 4;

  1185.     const int dyys = dyy >> 4;

  1186.     const uint16_t r4[4]   = { r, r, r, r };

  1187.     const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };

  1188.     const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };

  1189.     const uint64_t shift2 = 2 * shift;

  1190.     int x, y;

  1191. 

  1192.     const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);

  1193.     const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);

  1194.     const int dxh = dxy * (h - 1);

  1195.     const int dyw = dyx * (w - 1);

  1196.     if ( // non-constant fullpel offset (3% of blocks)

  1197.         ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |

  1198.          (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)

  1199.         // uses more than 16 bits of subpel mv (only at huge resolution)

  1200.         || (dxx | dxy | dyx | dyy) & 15 ||

  1201.         (unsigned)ix >= width  - w ||

  1202.         (unsigned)iy >= height - h) {

  1203.         // FIXME could still use mmx for some of the rows

  1204.         ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,

  1205.                  shift, r, width, height);

  1206.         return;

  1207.     }

  1208. 

  1209.     src += ix + iy * stride;

  1210. 

  1211.     __asm__ volatile (

  1212.         "movd         %0, %%mm6         \n\t"

  1213.         "pxor      %%mm7, %%mm7         \n\t"

  1214.         "punpcklwd %%mm6, %%mm6         \n\t"

  1215.         "punpcklwd %%mm6, %%mm6         \n\t"

  1216.         :: "r"(1<<shift)

  1217.     );

  1218. 

  1219.     for (x = 0; x < w; x += 4) {

  1220.         uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),

  1221.                             oxs - dxys + dxxs * (x + 1),

  1222.                             oxs - dxys + dxxs * (x + 2),

  1223.                             oxs - dxys + dxxs * (x + 3) };

  1224.         uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),

  1225.                             oys - dyys + dyxs * (x + 1),

  1226.                             oys - dyys + dyxs * (x + 2),

  1227.                             oys - dyys + dyxs * (x + 3) };

  1228. 

  1229.         for (y = 0; y < h; y++) {

  1230.             __asm__ volatile (

  1231.                 "movq      %0, %%mm4    \n\t"

  1232.                 "movq      %1, %%mm5    \n\t"

  1233.                 "paddw     %2, %%mm4    \n\t"

  1234.                 "paddw     %3, %%mm5    \n\t"

  1235.                 "movq   %%mm4, %0       \n\t"

  1236.                 "movq   %%mm5, %1       \n\t"

  1237.                 "psrlw    $12, %%mm4    \n\t"

  1238.                 "psrlw    $12, %%mm5    \n\t"

  1239.                 : "+m"(*dx4), "+m"(*dy4)

  1240.                 : "m"(*dxy4), "m"(*dyy4)

  1241.             );

  1242. 

  1243.             __asm__ volatile (

  1244.                 "movq      %%mm6, %%mm2 \n\t"

  1245.                 "movq      %%mm6, %%mm1 \n\t"

  1246.                 "psubw     %%mm4, %%mm2 \n\t"

  1247.                 "psubw     %%mm5, %%mm1 \n\t"

  1248.                 "movq      %%mm2, %%mm0 \n\t"

  1249.                 "movq      %%mm4, %%mm3 \n\t"

  1250.                 "pmullw    %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)

  1251.                 "pmullw    %%mm5, %%mm3 \n\t" // dx * dy

  1252.                 "pmullw    %%mm5, %%mm2 \n\t" // (s - dx) * dy

  1253.                 "pmullw    %%mm4, %%mm1 \n\t" // dx * (s - dy)

  1254. 

  1255.                 "movd         %4, %%mm5 \n\t"

  1256.                 "movd         %3, %%mm4 \n\t"

  1257.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1258.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1259.                 "pmullw    %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy

  1260.                 "pmullw    %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy

  1261. 

  1262.                 "movd         %2, %%mm5 \n\t"

  1263.                 "movd         %1, %%mm4 \n\t"

  1264.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1265.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1266.                 "pmullw    %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)

  1267.                 "pmullw    %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)

  1268.                 "paddw        %5, %%mm1 \n\t"

  1269.                 "paddw     %%mm3, %%mm2 \n\t"

  1270.                 "paddw     %%mm1, %%mm0 \n\t"

  1271.                 "paddw     %%mm2, %%mm0 \n\t"

  1272. 

  1273.                 "psrlw        %6, %%mm0 \n\t"

  1274.                 "packuswb  %%mm0, %%mm0 \n\t"

  1275.                 "movd      %%mm0, %0    \n\t"

  1276. 

  1277.                 : "=m"(dst[x + y * stride])

  1278.                 : "m"(src[0]), "m"(src[1]),

  1279.                   "m"(src[stride]), "m"(src[stride + 1]),

  1280.                   "m"(*r4), "m"(shift2)

  1281.             );

  1282.             src += stride;

  1283.         }

  1284.         src += 4 - h * stride;

  1285.     }

  1286. }

  1287. #endif /* HAVE_INLINE_ASM */

  1288. 

  1289. void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1290.                           ptrdiff_t line_size, int h);

  1291. void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1292.                           ptrdiff_t line_size, int h);

  1293. 

  1294. #if HAVE_INLINE_ASM

  1295. 

  1296. /* CAVS-specific */

  1297. void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1298. {

  1299.     put_pixels8_mmx(dst, src, stride, 8);

  1300. }

  1301. 

  1302. void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1303. {

  1304.     avg_pixels8_mmx(dst, src, stride, 8);

  1305. }

  1306. 

  1307. void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1308. {

  1309.     put_pixels16_mmx(dst, src, stride, 16);

  1310. }

  1311. 

  1312. void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)

  1313. {

  1314.     avg_pixels16_mmx(dst, src, stride, 16);

  1315. }

  1316. #endif /* HAVE_INLINE_ASM */

  1317. 

  1318. #if HAVE_YASM

  1319. /* VC-1-specific */

  1320. void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,

  1321.                                int stride, int rnd)

  1322. {

  1323.     ff_put_pixels8_mmx(dst, src, stride, 8);

  1324. }

  1325. 

  1326. void ff_avg_vc1_mspel_mc00_mmxext(uint8_t *dst, const uint8_t *src,

  1327.                                   int stride, int rnd)

  1328. {

  1329.     ff_avg_pixels8_mmxext(dst, src, stride, 8);

  1330. }

  1331. #endif /* HAVE_YASM */

  1332. 

  1333. #if HAVE_INLINE_ASM

  1334. static void vector_clipf_sse(float *dst, const float *src,

  1335.                              float min, float max, int len)

  1336. {

  1337.     x86_reg i = (len - 16) * 4;

  1338.     __asm__ volatile (

  1339.         "movss          %3, %%xmm4      \n\t"

  1340.         "movss          %4, %%xmm5      \n\t"

  1341.         "shufps $0, %%xmm4, %%xmm4      \n\t"

  1342.         "shufps $0, %%xmm5, %%xmm5      \n\t"

  1343.         "1:                             \n\t"

  1344.         "movaps   (%2, %0), %%xmm0      \n\t" // 3/1 on intel

  1345.         "movaps 16(%2, %0), %%xmm1      \n\t"

  1346.         "movaps 32(%2, %0), %%xmm2      \n\t"

  1347.         "movaps 48(%2, %0), %%xmm3      \n\t"

  1348.         "maxps      %%xmm4, %%xmm0      \n\t"

  1349.         "maxps      %%xmm4, %%xmm1      \n\t"

  1350.         "maxps      %%xmm4, %%xmm2      \n\t"

  1351.         "maxps      %%xmm4, %%xmm3      \n\t"

  1352.         "minps      %%xmm5, %%xmm0      \n\t"

  1353.         "minps      %%xmm5, %%xmm1      \n\t"

  1354.         "minps      %%xmm5, %%xmm2      \n\t"

  1355.         "minps      %%xmm5, %%xmm3      \n\t"

  1356.         "movaps     %%xmm0,   (%1, %0)  \n\t"

  1357.         "movaps     %%xmm1, 16(%1, %0)  \n\t"

  1358.         "movaps     %%xmm2, 32(%1, %0)  \n\t"

  1359.         "movaps     %%xmm3, 48(%1, %0)  \n\t"

  1360.         "sub           $64, %0          \n\t"

  1361.         "jge            1b              \n\t"

  1362.         : "+&r"(i)

  1363.         : "r"(dst), "r"(src), "m"(min), "m"(max)

  1364.         : "memory"

  1365.     );

  1366. }

  1367. 

  1368. #endif /* HAVE_INLINE_ASM */

  1369. 

  1370. int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,

  1371.                                       int order);

  1372. int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,

  1373.                                     int order);

  1374. int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,

  1375.                                                const int16_t *v3,

  1376.                                                int order, int mul);

  1377. int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,

  1378.                                              const int16_t *v3,

  1379.                                              int order, int mul);

  1380. int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,

  1381.                                               const int16_t *v3,

  1382.                                               int order, int mul);

  1383. 

  1384. void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,

  1385.                                         const int16_t *window, unsigned int len);

  1386. void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,

  1387.                                       const int16_t *window, unsigned int len);

  1388. void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,

  1389.                                   const int16_t *window, unsigned int len);

  1390. void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,

  1391.                                 const int16_t *window, unsigned int len);

  1392. void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,

  1393.                                  const int16_t *window, unsigned int len);

  1394. void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,

  1395.                                       const int16_t *window, unsigned int len);

  1396. 

  1397. void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);

  1398. void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);

  1399. 

  1400. void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,

  1401.                                           const uint8_t *diff, int w,

  1402.                                           int *left, int *left_top);

  1403. int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,

  1404.                                        int w, int left);

  1405. int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,

  1406.                                       int w, int left);

  1407. 

  1408. void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,

  1409.                                    int32_t min, int32_t max, unsigned int len);

  1410. void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,

  1411.                                    int32_t min, int32_t max, unsigned int len);

  1412. void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,

  1413.                                    int32_t min, int32_t max, unsigned int len);

  1414. void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,

  1415.                                    int32_t min, int32_t max, unsigned int len);

  1416. 

  1417. #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \

  1418.     do {                                                                     \

  1419.     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \

  1420.     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \

  1421.     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \

  1422.     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \

  1423.     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \

  1424.     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \

  1425.     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \

  1426.     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \

  1427.     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \

  1428.     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \

  1429.     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \

  1430.     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \

  1431.     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \

  1432.     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \

  1433.     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \

  1434.     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \

  1435.     } while (0)

  1436. 

  1437. #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU)                                     \

  1438.     do {                                                                        \

  1439.         c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _     ## CPU; \

  1440.         c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_  ## CPU; \

  1441.         c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_  ## CPU; \

  1442.         c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \

  1443.     } while (0)

  1444. 

  1445. static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,

  1446.                                      int mm_flags)

  1447. {

  1448.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1449. 

  1450. #if HAVE_INLINE_ASM

  1451.     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;

  1452.     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;

  1453.     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;

  1454. 

  1455.     if (!high_bit_depth) {

  1456.         c->clear_block  = clear_block_mmx;

  1457.         c->clear_blocks = clear_blocks_mmx;

  1458.         c->draw_edges   = draw_edges_mmx;

  1459. 

  1460.         SET_HPEL_FUNCS(put,        [0], 16, mmx);

  1461.         SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);

  1462.         SET_HPEL_FUNCS(avg,        [0], 16, mmx);

  1463.         SET_HPEL_FUNCS(avg_no_rnd,    , 16, mmx);

  1464.         SET_HPEL_FUNCS(put,        [1],  8, mmx);

  1465.         SET_HPEL_FUNCS(put_no_rnd, [1],  8, mmx);

  1466.         SET_HPEL_FUNCS(avg,        [1],  8, mmx);

  1467. 

  1468.         switch (avctx->idct_algo) {

  1469.         case FF_IDCT_AUTO:

  1470.         case FF_IDCT_SIMPLEMMX:

  1471.             c->idct_put              = ff_simple_idct_put_mmx;

  1472.             c->idct_add              = ff_simple_idct_add_mmx;

  1473.             c->idct                  = ff_simple_idct_mmx;

  1474.             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;

  1475.             break;

  1476.         case FF_IDCT_XVIDMMX:

  1477.             c->idct_put              = ff_idct_xvid_mmx_put;

  1478.             c->idct_add              = ff_idct_xvid_mmx_add;

  1479.             c->idct                  = ff_idct_xvid_mmx;

  1480.             break;

  1481.         }

  1482.     }

  1483. 

  1484.     c->gmc = gmc_mmx;

  1485. 

  1486.     c->add_bytes = add_bytes_mmx;

  1487. #endif /* HAVE_INLINE_ASM */

  1488. 

  1489. #if HAVE_YASM

  1490.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  1491.         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;

  1492.         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;

  1493.     }

  1494. 

  1495.     c->vector_clip_int32 = ff_vector_clip_int32_mmx;

  1496. #endif

  1497. 

  1498. }

  1499. 

  1500. static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,

  1501.                                         int mm_flags)

  1502. {

  1503.     const int bit_depth      = avctx->bits_per_raw_sample;

  1504.     const int high_bit_depth = bit_depth > 8;

  1505. 

  1506. #if HAVE_YASM

  1507.     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );

  1508.     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );

  1509. 

  1510.     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );

  1511.     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );

  1512.     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );

  1513.     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );

  1514. 

  1515.     if (!high_bit_depth) {

  1516.         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_mmxext;

  1517.         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_mmxext;

  1518. 

  1519.         c->avg_pixels_tab[0][0] = ff_avg_pixels16_mmxext;

  1520.         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_mmxext;

  1521.         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_mmxext;

  1522. 

  1523.         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_mmxext;

  1524.         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_mmxext;

  1525. 

  1526.         c->avg_pixels_tab[1][0] = ff_avg_pixels8_mmxext;

  1527.         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_mmxext;

  1528.         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_mmxext;

  1529.     }

  1530. 

  1531.     if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  1532.         if (!high_bit_depth) {

  1533.             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_mmxext;

  1534.             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_mmxext;

  1535.             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_mmxext;

  1536.             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_mmxext;

  1537. 

  1538.             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_mmxext;

  1539.             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_mmxext;

  1540.         }

  1541.     }

  1542. #endif /* HAVE_YASM */

  1543. 

  1544. #if HAVE_INLINE_ASM

  1545.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1546.         c->idct_put = ff_idct_xvid_mmxext_put;

  1547.         c->idct_add = ff_idct_xvid_mmxext_add;

  1548.         c->idct     = ff_idct_xvid_mmxext;

  1549.     }

  1550. #endif /* HAVE_INLINE_ASM */

  1551. 

  1552. #if HAVE_MMXEXT_EXTERNAL

  1553.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  1554.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  1555.         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_mmxext;

  1556.         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_mmxext;

  1557.     }

  1558. 

  1559.     /* slower than cmov version on AMD */

  1560.     if (!(mm_flags & AV_CPU_FLAG_3DNOW))

  1561.         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;

  1562. 

  1563.     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;

  1564.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;

  1565. 

  1566.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1567.         c->apply_window_int16 = ff_apply_window_int16_mmxext;

  1568.     } else {

  1569.         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;

  1570.     }

  1571. #endif /* HAVE_MMXEXT_EXTERNAL */

  1572. }

  1573. 

  1574. static av_cold void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,

  1575.                                        int mm_flags)

  1576. {

  1577.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1578. 

  1579. #if HAVE_YASM

  1580.     if (!high_bit_depth) {

  1581.         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_3dnow;

  1582.         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_3dnow;

  1583. 

  1584.         c->avg_pixels_tab[0][0] = ff_avg_pixels16_3dnow;

  1585.         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_3dnow;

  1586.         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_3dnow;

  1587. 

  1588.         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_3dnow;

  1589.         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_3dnow;

  1590. 

  1591.         c->avg_pixels_tab[1][0] = ff_avg_pixels8_3dnow;

  1592.         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_3dnow;

  1593.         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_3dnow;

  1594. 

  1595.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)){

  1596.             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_3dnow;

  1597.             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_3dnow;

  1598.             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_3dnow;

  1599.             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_3dnow;

  1600. 

  1601.             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_3dnow;

  1602.             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_3dnow;

  1603.         }

  1604.     }

  1605. 

  1606.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  1607.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  1608.         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_3dnow;

  1609.         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_3dnow;

  1610.     }

  1611. #endif /* HAVE_YASM */

  1612. }

  1613. 

  1614. static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,

  1615.                                      int mm_flags)

  1616. {

  1617.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1618. 

  1619. #if HAVE_INLINE_ASM

  1620.     if (!high_bit_depth) {

  1621.         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {

  1622.             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */

  1623.             c->clear_block  = clear_block_sse;

  1624.             c->clear_blocks = clear_blocks_sse;

  1625.         }

  1626.     }

  1627. 

  1628.     c->vector_clipf = vector_clipf_sse;

  1629. #endif /* HAVE_INLINE_ASM */

  1630. }

  1631. 

  1632. static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,

  1633.                                       int mm_flags)

  1634. {

  1635.     const int bit_depth      = avctx->bits_per_raw_sample;

  1636.     const int high_bit_depth = bit_depth > 8;

  1637. 

  1638. #if HAVE_SSE2_INLINE

  1639.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1640.         c->idct_put              = ff_idct_xvid_sse2_put;

  1641.         c->idct_add              = ff_idct_xvid_sse2_add;

  1642.         c->idct                  = ff_idct_xvid_sse2;

  1643.         c->idct_permutation_type = FF_SSE2_IDCT_PERM;

  1644.     }

  1645. #endif /* HAVE_SSE2_INLINE */

  1646. 

  1647. #if HAVE_SSE2_EXTERNAL

  1648.     if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  1649.         // these functions are slower than mmx on AMD, but faster on Intel

  1650.         if (!high_bit_depth) {

  1651.             c->put_pixels_tab[0][0]        = ff_put_pixels16_sse2;

  1652.             c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;

  1653.             c->avg_pixels_tab[0][0]        = ff_avg_pixels16_sse2;

  1654.         }

  1655.     }

  1656. 

  1657.     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;

  1658.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;

  1659.     if (mm_flags & AV_CPU_FLAG_ATOM) {

  1660.         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;

  1661.     } else {

  1662.         c->vector_clip_int32 = ff_vector_clip_int32_sse2;

  1663.     }

  1664.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1665.         c->apply_window_int16 = ff_apply_window_int16_sse2;

  1666.     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  1667.         c->apply_window_int16 = ff_apply_window_int16_round_sse2;

  1668.     }

  1669.     c->bswap_buf = ff_bswap32_buf_sse2;

  1670. #endif /* HAVE_SSE2_EXTERNAL */

  1671. }

  1672. 

  1673. static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,

  1674.                                        int mm_flags)

  1675. {

  1676. #if HAVE_SSSE3_EXTERNAL

  1677.     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;

  1678.     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe

  1679.         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;

  1680. 

  1681.     if (mm_flags & AV_CPU_FLAG_ATOM)

  1682.         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;

  1683.     else

  1684.         c->apply_window_int16 = ff_apply_window_int16_ssse3;

  1685.     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit

  1686.         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;

  1687.     c->bswap_buf = ff_bswap32_buf_ssse3;

  1688. #endif /* HAVE_SSSE3_EXTERNAL */

  1689. }

  1690. 

  1691. static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,

  1692.                                       int mm_flags)

  1693. {

  1694. #if HAVE_SSE4_EXTERNAL

  1695.     c->vector_clip_int32 = ff_vector_clip_int32_sse4;

  1696. #endif /* HAVE_SSE4_EXTERNAL */

  1697. }

  1698. 

  1699. av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)

  1700. {

  1701.     int mm_flags = av_get_cpu_flags();

  1702. 

  1703. #if HAVE_7REGS && HAVE_INLINE_ASM

  1704.     if (mm_flags & AV_CPU_FLAG_CMOV)

  1705.         c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;

  1706. #endif

  1707. 

  1708.     if (mm_flags & AV_CPU_FLAG_MMX)

  1709.         dsputil_init_mmx(c, avctx, mm_flags);

  1710. 

  1711.     if (mm_flags & AV_CPU_FLAG_MMXEXT)

  1712.         dsputil_init_mmxext(c, avctx, mm_flags);

  1713. 

  1714.     if (mm_flags & AV_CPU_FLAG_3DNOW)

  1715.         dsputil_init_3dnow(c, avctx, mm_flags);

  1716. 

  1717.     if (mm_flags & AV_CPU_FLAG_SSE)

  1718.         dsputil_init_sse(c, avctx, mm_flags);

  1719. 

  1720.     if (mm_flags & AV_CPU_FLAG_SSE2)

  1721.         dsputil_init_sse2(c, avctx, mm_flags);

  1722. 

  1723.     if (mm_flags & AV_CPU_FLAG_SSSE3)

  1724.         dsputil_init_ssse3(c, avctx, mm_flags);

  1725. 

  1726.     if (mm_flags & AV_CPU_FLAG_SSE4)

  1727.         dsputil_init_sse4(c, avctx, mm_flags);

  1728. 

  1729.     if (CONFIG_ENCODERS)

  1730.         ff_dsputilenc_init_mmx(c, avctx);

  1731. }