falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36851 Commits
32 Branches
748MB
Tree: be40d6cc2b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'be40d6cc2b'
${ noResults }
FFmpeg/libavcodec/x86/dsputil_mmx.c

1706 lines
83KB
Raw Blame History 

  1. /*

  2.  * MMX optimized DSP utils

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  *

  8.  * This file is part of Libav.

  9.  *

  10.  * Libav is free software; you can redistribute it and/or

  11.  * modify it under the terms of the GNU Lesser General Public

  12.  * License as published by the Free Software Foundation; either

  13.  * version 2.1 of the License, or (at your option) any later version.

  14.  *

  15.  * Libav is distributed in the hope that it will be useful,

  16.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  17.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  18.  * Lesser General Public License for more details.

  19.  *

  20.  * You should have received a copy of the GNU Lesser General Public

  21.  * License along with Libav; if not, write to the Free Software

  22.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  23.  */

  24. 

  25. #include "libavutil/attributes.h"

  26. #include "libavutil/cpu.h"

  27. #include "libavutil/x86/asm.h"

  28. #include "libavcodec/dsputil.h"

  29. #include "libavcodec/h264dsp.h"

  30. #include "libavcodec/mpegvideo.h"

  31. #include "libavcodec/simple_idct.h"

  32. #include "dsputil_mmx.h"

  33. #include "idct_xvid.h"

  34. 

  35. //#undef NDEBUG

  36. //#include <assert.h>

  37. 

  38. /* pixel operations */

  39. DECLARE_ALIGNED(8,  const uint64_t, ff_bone) = 0x0101010101010101ULL;

  40. DECLARE_ALIGNED(8,  const uint64_t, ff_wtwo) = 0x0002000200020002ULL;

  41. 

  42. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1)    = { 0x0001000100010001ULL, 0x0001000100010001ULL };

  43. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_2)    = { 0x0002000200020002ULL, 0x0002000200020002ULL };

  44. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_3)    = { 0x0003000300030003ULL, 0x0003000300030003ULL };

  45. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_4)    = { 0x0004000400040004ULL, 0x0004000400040004ULL };

  46. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_5)    = { 0x0005000500050005ULL, 0x0005000500050005ULL };

  47. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_8)    = { 0x0008000800080008ULL, 0x0008000800080008ULL };

  48. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_9)    = { 0x0009000900090009ULL, 0x0009000900090009ULL };

  49. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_15)   =   0x000F000F000F000FULL;

  50. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_16)   = { 0x0010001000100010ULL, 0x0010001000100010ULL };

  51. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_17)   = { 0x0011001100110011ULL, 0x0011001100110011ULL };

  52. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_18)   = { 0x0012001200120012ULL, 0x0012001200120012ULL };

  53. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_20)   =   0x0014001400140014ULL;

  54. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_27)   = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };

  55. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_28)   = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };

  56. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_32)   = { 0x0020002000200020ULL, 0x0020002000200020ULL };

  57. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_42)   =   0x002A002A002A002AULL;

  58. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_53)   =   0x0035003500350035ULL;

  59. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_63)   = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };

  60. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_64)   = { 0x0040004000400040ULL, 0x0040004000400040ULL };

  61. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_96)   =   0x0060006000600060ULL;

  62. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_128)  =   0x0080008000800080ULL;

  63. DECLARE_ALIGNED(8,  const uint64_t, ff_pw_255)  =   0x00ff00ff00ff00ffULL;

  64. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_512)  = { 0x0200020002000200ULL, 0x0200020002000200ULL };

  65. DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };

  66. 

  67. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_0)    = { 0x0000000000000000ULL, 0x0000000000000000ULL };

  68. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_1)    = { 0x0101010101010101ULL, 0x0101010101010101ULL };

  69. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_3)    = { 0x0303030303030303ULL, 0x0303030303030303ULL };

  70. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_4)    = { 0x0404040404040404ULL, 0x0404040404040404ULL };

  71. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_7)    =   0x0707070707070707ULL;

  72. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_1F)   =   0x1F1F1F1F1F1F1F1FULL;

  73. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_3F)   =   0x3F3F3F3F3F3F3F3FULL;

  74. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_80)   = { 0x8080808080808080ULL, 0x8080808080808080ULL };

  75. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_81)   =   0x8181818181818181ULL;

  76. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_A1)   = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };

  77. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_F8)   = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };

  78. DECLARE_ALIGNED(8,  const uint64_t, ff_pb_FC)   =   0xFCFCFCFCFCFCFCFCULL;

  79. DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_FE)   = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };

  80. 

  81. DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };

  82. DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };

  83. 

  84. 

  85. #if HAVE_YASM

  86. void ff_put_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  87.                               ptrdiff_t line_size, int h);

  88. void ff_put_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  89.                              ptrdiff_t line_size, int h);

  90. void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  91.                               int dstStride, int src1Stride, int h);

  92. void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,

  93.                                      uint8_t *src2, int dstStride,

  94.                                      int src1Stride, int h);

  95. void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  96.                               int dstStride, int src1Stride, int h);

  97. void ff_put_pixels16_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  98.                                ptrdiff_t line_size, int h);

  99. void ff_put_pixels16_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  100.                               ptrdiff_t line_size, int h);

  101. void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  102.                                int dstStride, int src1Stride, int h);

  103. void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  104.                                int dstStride, int src1Stride, int h);

  105. void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,

  106.                                       int dstStride, int src1Stride, int h);

  107. void ff_put_no_rnd_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  108.                                      ptrdiff_t line_size, int h);

  109. void ff_put_no_rnd_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  110.                                     ptrdiff_t line_size, int h);

  111. void ff_put_no_rnd_pixels8_x2_exact_mmxext(uint8_t *block,

  112.                                            const uint8_t *pixels,

  113.                                            ptrdiff_t line_size, int h);

  114. void ff_put_no_rnd_pixels8_x2_exact_3dnow(uint8_t *block,

  115.                                           const uint8_t *pixels,

  116.                                           ptrdiff_t line_size, int h);

  117. void ff_put_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  118.                               ptrdiff_t line_size, int h);

  119. void ff_put_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  120.                              ptrdiff_t line_size, int h);

  121. void ff_put_no_rnd_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  122.                                      ptrdiff_t line_size, int h);

  123. void ff_put_no_rnd_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  124.                                     ptrdiff_t line_size, int h);

  125. void ff_put_no_rnd_pixels8_y2_exact_mmxext(uint8_t *block,

  126.                                            const uint8_t *pixels,

  127.                                            ptrdiff_t line_size, int h);

  128. void ff_put_no_rnd_pixels8_y2_exact_3dnow(uint8_t *block,

  129.                                           const uint8_t *pixels,

  130.                                           ptrdiff_t line_size, int h);

  131. void ff_avg_pixels8_3dnow(uint8_t *block, const uint8_t *pixels,

  132.                           ptrdiff_t line_size, int h);

  133. void ff_avg_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,

  134.                               ptrdiff_t line_size, int h);

  135. void ff_avg_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,

  136.                              ptrdiff_t line_size, int h);

  137. void ff_avg_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,

  138.                               ptrdiff_t line_size, int h);

  139. void ff_avg_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,

  140.                              ptrdiff_t line_size, int h);

  141. void ff_avg_pixels8_xy2_mmxext(uint8_t *block, const uint8_t *pixels,

  142.                                ptrdiff_t line_size, int h);

  143. void ff_avg_pixels8_xy2_3dnow(uint8_t *block, const uint8_t *pixels,

  144.                               ptrdiff_t line_size, int h);

  145. 

  146. void ff_put_pixels8_mmxext(uint8_t *block, const uint8_t *pixels, ptrdiff_t line_size, int h);

  147. static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,

  148.                                    ptrdiff_t line_size, int h)

  149. {

  150.     ff_put_pixels8_mmxext(block,     pixels,     line_size, h);

  151.     ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);

  152. }

  153. 

  154. void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  155.                                          int dstStride, int srcStride, int h);

  156. void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  157.                                          int dstStride, int srcStride, int h);

  158. void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  159.                                                  int dstStride, int srcStride,

  160.                                                  int h);

  161. void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  162.                                         int dstStride, int srcStride, int h);

  163. void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  164.                                         int dstStride, int srcStride, int h);

  165. void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  166.                                                 int dstStride, int srcStride,

  167.                                                 int h);

  168. void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  169.                                          int dstStride, int srcStride);

  170. void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  171.                                          int dstStride, int srcStride);

  172. void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  173.                                                  int dstStride, int srcStride);

  174. void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  175.                                         int dstStride, int srcStride);

  176. void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  177.                                         int dstStride, int srcStride);

  178. void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,

  179.                                                 int dstStride, int srcStride);

  180. #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext

  181. #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext

  182. #endif /* HAVE_YASM */

  183. 

  184. 

  185. #if HAVE_INLINE_ASM

  186. 

  187. #define JUMPALIGN()     __asm__ volatile (".p2align 3"::)

  188. #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)

  189. 

  190. #define MOVQ_BFE(regd)                                  \

  191.     __asm__ volatile (                                  \

  192.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  193.         "paddb   %%"#regd", %%"#regd"   \n\t" ::)

  194. 

  195. #ifndef PIC

  196. #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))

  197. #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))

  198. #else

  199. // for shared library it's better to use this way for accessing constants

  200. // pcmpeqd -> -1

  201. #define MOVQ_BONE(regd)                                 \

  202.     __asm__ volatile (                                  \

  203.         "pcmpeqd  %%"#regd", %%"#regd"  \n\t"           \

  204.         "psrlw          $15, %%"#regd"  \n\t"           \

  205.         "packuswb %%"#regd", %%"#regd"  \n\t" ::)

  206. 

  207. #define MOVQ_WTWO(regd)                                 \

  208.     __asm__ volatile (                                  \

  209.         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \

  210.         "psrlw         $15, %%"#regd"   \n\t"           \

  211.         "psllw          $1, %%"#regd"   \n\t"::)

  212. 

  213. #endif

  214. 

  215. // using regr as temporary and for the output result

  216. // first argument is unmodifed and second is trashed

  217. // regfe is supposed to contain 0xfefefefefefefefe

  218. #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe)                \

  219.     "movq   "#rega", "#regr"            \n\t"                    \

  220.     "pand   "#regb", "#regr"            \n\t"                    \

  221.     "pxor   "#rega", "#regb"            \n\t"                    \

  222.     "pand  "#regfe", "#regb"            \n\t"                    \

  223.     "psrlq       $1, "#regb"            \n\t"                    \

  224.     "paddb  "#regb", "#regr"            \n\t"

  225. 

  226. #define PAVGB_MMX(rega, regb, regr, regfe)                       \

  227.     "movq   "#rega", "#regr"            \n\t"                    \

  228.     "por    "#regb", "#regr"            \n\t"                    \

  229.     "pxor   "#rega", "#regb"            \n\t"                    \

  230.     "pand  "#regfe", "#regb"            \n\t"                    \

  231.     "psrlq       $1, "#regb"            \n\t"                    \

  232.     "psubb  "#regb", "#regr"            \n\t"

  233. 

  234. // mm6 is supposed to contain 0xfefefefefefefefe

  235. #define PAVGBP_MMX_NO_RND(rega, regb, regr,  regc, regd, regp)   \

  236.     "movq  "#rega", "#regr"             \n\t"                    \

  237.     "movq  "#regc", "#regp"             \n\t"                    \

  238.     "pand  "#regb", "#regr"             \n\t"                    \

  239.     "pand  "#regd", "#regp"             \n\t"                    \

  240.     "pxor  "#rega", "#regb"             \n\t"                    \

  241.     "pxor  "#regc", "#regd"             \n\t"                    \

  242.     "pand    %%mm6, "#regb"             \n\t"                    \

  243.     "pand    %%mm6, "#regd"             \n\t"                    \

  244.     "psrlq      $1, "#regb"             \n\t"                    \

  245.     "psrlq      $1, "#regd"             \n\t"                    \

  246.     "paddb "#regb", "#regr"             \n\t"                    \

  247.     "paddb "#regd", "#regp"             \n\t"

  248. 

  249. #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp)           \

  250.     "movq  "#rega", "#regr"             \n\t"                    \

  251.     "movq  "#regc", "#regp"             \n\t"                    \

  252.     "por   "#regb", "#regr"             \n\t"                    \

  253.     "por   "#regd", "#regp"             \n\t"                    \

  254.     "pxor  "#rega", "#regb"             \n\t"                    \

  255.     "pxor  "#regc", "#regd"             \n\t"                    \

  256.     "pand    %%mm6, "#regb"             \n\t"                    \

  257.     "pand    %%mm6, "#regd"             \n\t"                    \

  258.     "psrlq      $1, "#regd"             \n\t"                    \

  259.     "psrlq      $1, "#regb"             \n\t"                    \

  260.     "psubb "#regb", "#regr"             \n\t"                    \

  261.     "psubb "#regd", "#regp"             \n\t"

  262. 

  263. /***********************************/

  264. /* MMX no rounding */

  265. #define NO_RND 1

  266. #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx

  267. #define SET_RND  MOVQ_WONE

  268. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX_NO_RND(a, b, c, d, e, f)

  269. #define PAVGB(a, b, c, e)               PAVGB_MMX_NO_RND(a, b, c, e)

  270. #define OP_AVG(a, b, c, e)              PAVGB_MMX(a, b, c, e)

  271. 

  272. #include "dsputil_rnd_template.c"

  273. 

  274. #undef DEF

  275. #undef SET_RND

  276. #undef PAVGBP

  277. #undef PAVGB

  278. #undef NO_RND

  279. /***********************************/

  280. /* MMX rounding */

  281. 

  282. #define DEF(x, y) x ## _ ## y ## _mmx

  283. #define SET_RND  MOVQ_WTWO

  284. #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX(a, b, c, d, e, f)

  285. #define PAVGB(a, b, c, e)               PAVGB_MMX(a, b, c, e)

  286. 

  287. #include "dsputil_rnd_template.c"

  288. 

  289. #undef DEF

  290. #undef SET_RND

  291. #undef PAVGBP

  292. #undef PAVGB

  293. #undef OP_AVG

  294. 

  295. #endif /* HAVE_INLINE_ASM */

  296. 

  297. 

  298. #if HAVE_YASM

  299. 

  300. /***********************************/

  301. /* 3Dnow specific */

  302. 

  303. #define DEF(x) x ## _3dnow

  304. 

  305. #include "dsputil_avg_template.c"

  306. 

  307. #undef DEF

  308. 

  309. /***********************************/

  310. /* MMXEXT specific */

  311. 

  312. #define DEF(x) x ## _mmxext

  313. 

  314. #include "dsputil_avg_template.c"

  315. 

  316. #undef DEF

  317. 

  318. #endif /* HAVE_YASM */

  319. 

  320. 

  321. #if HAVE_INLINE_ASM

  322. #define put_no_rnd_pixels16_mmx put_pixels16_mmx

  323. #define put_no_rnd_pixels8_mmx put_pixels8_mmx

  324. 

  325. /***********************************/

  326. /* standard MMX */

  327. 

  328. void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  329.                                int line_size)

  330. {

  331.     const int16_t *p;

  332.     uint8_t *pix;

  333. 

  334.     /* read the pixels */

  335.     p   = block;

  336.     pix = pixels;

  337.     /* unrolled loop */

  338.     __asm__ volatile (

  339.         "movq      (%3), %%mm0          \n\t"

  340.         "movq     8(%3), %%mm1          \n\t"

  341.         "movq    16(%3), %%mm2          \n\t"

  342.         "movq    24(%3), %%mm3          \n\t"

  343.         "movq    32(%3), %%mm4          \n\t"

  344.         "movq    40(%3), %%mm5          \n\t"

  345.         "movq    48(%3), %%mm6          \n\t"

  346.         "movq    56(%3), %%mm7          \n\t"

  347.         "packuswb %%mm1, %%mm0          \n\t"

  348.         "packuswb %%mm3, %%mm2          \n\t"

  349.         "packuswb %%mm5, %%mm4          \n\t"

  350.         "packuswb %%mm7, %%mm6          \n\t"

  351.         "movq     %%mm0, (%0)           \n\t"

  352.         "movq     %%mm2, (%0, %1)       \n\t"

  353.         "movq     %%mm4, (%0, %1, 2)    \n\t"

  354.         "movq     %%mm6, (%0, %2)       \n\t"

  355.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),

  356.            "r"(p)

  357.         : "memory");

  358.     pix += line_size * 4;

  359.     p   += 32;

  360. 

  361.     // if here would be an exact copy of the code above

  362.     // compiler would generate some very strange code

  363.     // thus using "r"

  364.     __asm__ volatile (

  365.         "movq       (%3), %%mm0         \n\t"

  366.         "movq      8(%3), %%mm1         \n\t"

  367.         "movq     16(%3), %%mm2         \n\t"

  368.         "movq     24(%3), %%mm3         \n\t"

  369.         "movq     32(%3), %%mm4         \n\t"

  370.         "movq     40(%3), %%mm5         \n\t"

  371.         "movq     48(%3), %%mm6         \n\t"

  372.         "movq     56(%3), %%mm7         \n\t"

  373.         "packuswb  %%mm1, %%mm0         \n\t"

  374.         "packuswb  %%mm3, %%mm2         \n\t"

  375.         "packuswb  %%mm5, %%mm4         \n\t"

  376.         "packuswb  %%mm7, %%mm6         \n\t"

  377.         "movq      %%mm0, (%0)          \n\t"

  378.         "movq      %%mm2, (%0, %1)      \n\t"

  379.         "movq      %%mm4, (%0, %1, 2)   \n\t"

  380.         "movq      %%mm6, (%0, %2)      \n\t"

  381.         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)

  382.         : "memory");

  383. }

  384. 

  385. #define put_signed_pixels_clamped_mmx_half(off)             \

  386.     "movq          "#off"(%2), %%mm1        \n\t"           \

  387.     "movq     16 + "#off"(%2), %%mm2        \n\t"           \

  388.     "movq     32 + "#off"(%2), %%mm3        \n\t"           \

  389.     "movq     48 + "#off"(%2), %%mm4        \n\t"           \

  390.     "packsswb  8 + "#off"(%2), %%mm1        \n\t"           \

  391.     "packsswb 24 + "#off"(%2), %%mm2        \n\t"           \

  392.     "packsswb 40 + "#off"(%2), %%mm3        \n\t"           \

  393.     "packsswb 56 + "#off"(%2), %%mm4        \n\t"           \

  394.     "paddb              %%mm0, %%mm1        \n\t"           \

  395.     "paddb              %%mm0, %%mm2        \n\t"           \

  396.     "paddb              %%mm0, %%mm3        \n\t"           \

  397.     "paddb              %%mm0, %%mm4        \n\t"           \

  398.     "movq               %%mm1, (%0)         \n\t"           \

  399.     "movq               %%mm2, (%0, %3)     \n\t"           \

  400.     "movq               %%mm3, (%0, %3, 2)  \n\t"           \

  401.     "movq               %%mm4, (%0, %1)     \n\t"

  402. 

  403. void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  404.                                       int line_size)

  405. {

  406.     x86_reg line_skip = line_size;

  407.     x86_reg line_skip3;

  408. 

  409.     __asm__ volatile (

  410.         "movq "MANGLE(ff_pb_80)", %%mm0     \n\t"

  411.         "lea         (%3, %3, 2), %1        \n\t"

  412.         put_signed_pixels_clamped_mmx_half(0)

  413.         "lea         (%0, %3, 4), %0        \n\t"

  414.         put_signed_pixels_clamped_mmx_half(64)

  415.         : "+&r"(pixels), "=&r"(line_skip3)

  416.         : "r"(block), "r"(line_skip)

  417.         : "memory");

  418. }

  419. 

  420. void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,

  421.                                int line_size)

  422. {

  423.     const int16_t *p;

  424.     uint8_t *pix;

  425.     int i;

  426. 

  427.     /* read the pixels */

  428.     p   = block;

  429.     pix = pixels;

  430.     MOVQ_ZERO(mm7);

  431.     i = 4;

  432.     do {

  433.         __asm__ volatile (

  434.             "movq        (%2), %%mm0    \n\t"

  435.             "movq       8(%2), %%mm1    \n\t"

  436.             "movq      16(%2), %%mm2    \n\t"

  437.             "movq      24(%2), %%mm3    \n\t"

  438.             "movq          %0, %%mm4    \n\t"

  439.             "movq          %1, %%mm6    \n\t"

  440.             "movq       %%mm4, %%mm5    \n\t"

  441.             "punpcklbw  %%mm7, %%mm4    \n\t"

  442.             "punpckhbw  %%mm7, %%mm5    \n\t"

  443.             "paddsw     %%mm4, %%mm0    \n\t"

  444.             "paddsw     %%mm5, %%mm1    \n\t"

  445.             "movq       %%mm6, %%mm5    \n\t"

  446.             "punpcklbw  %%mm7, %%mm6    \n\t"

  447.             "punpckhbw  %%mm7, %%mm5    \n\t"

  448.             "paddsw     %%mm6, %%mm2    \n\t"

  449.             "paddsw     %%mm5, %%mm3    \n\t"

  450.             "packuswb   %%mm1, %%mm0    \n\t"

  451.             "packuswb   %%mm3, %%mm2    \n\t"

  452.             "movq       %%mm0, %0       \n\t"

  453.             "movq       %%mm2, %1       \n\t"

  454.             : "+m"(*pix), "+m"(*(pix + line_size))

  455.             : "r"(p)

  456.             : "memory");

  457.         pix += line_size * 2;

  458.         p   += 16;

  459.     } while (--i);

  460. }

  461. 

  462. static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,

  463.                             ptrdiff_t line_size, int h)

  464. {

  465.     __asm__ volatile (

  466.         "lea   (%3, %3), %%"REG_a"      \n\t"

  467.         ".p2align     3                 \n\t"

  468.         "1:                             \n\t"

  469.         "movq  (%1    ), %%mm0          \n\t"

  470.         "movq  (%1, %3), %%mm1          \n\t"

  471.         "movq     %%mm0, (%2)           \n\t"

  472.         "movq     %%mm1, (%2, %3)       \n\t"

  473.         "add  %%"REG_a", %1             \n\t"

  474.         "add  %%"REG_a", %2             \n\t"

  475.         "movq  (%1    ), %%mm0          \n\t"

  476.         "movq  (%1, %3), %%mm1          \n\t"

  477.         "movq     %%mm0, (%2)           \n\t"

  478.         "movq     %%mm1, (%2, %3)       \n\t"

  479.         "add  %%"REG_a", %1             \n\t"

  480.         "add  %%"REG_a", %2             \n\t"

  481.         "subl        $4, %0             \n\t"

  482.         "jnz         1b                 \n\t"

  483.         : "+g"(h), "+r"(pixels),  "+r"(block)

  484.         : "r"((x86_reg)line_size)

  485.         : "%"REG_a, "memory"

  486.         );

  487. }

  488. 

  489. static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,

  490.                              ptrdiff_t line_size, int h)

  491. {

  492.     __asm__ volatile (

  493.         "lea   (%3, %3), %%"REG_a"      \n\t"

  494.         ".p2align     3                 \n\t"

  495.         "1:                             \n\t"

  496.         "movq  (%1    ), %%mm0          \n\t"

  497.         "movq 8(%1    ), %%mm4          \n\t"

  498.         "movq  (%1, %3), %%mm1          \n\t"

  499.         "movq 8(%1, %3), %%mm5          \n\t"

  500.         "movq     %%mm0,  (%2)          \n\t"

  501.         "movq     %%mm4, 8(%2)          \n\t"

  502.         "movq     %%mm1,  (%2, %3)      \n\t"

  503.         "movq     %%mm5, 8(%2, %3)      \n\t"

  504.         "add  %%"REG_a", %1             \n\t"

  505.         "add  %%"REG_a", %2             \n\t"

  506.         "movq  (%1    ), %%mm0          \n\t"

  507.         "movq 8(%1    ), %%mm4          \n\t"

  508.         "movq  (%1, %3), %%mm1          \n\t"

  509.         "movq 8(%1, %3), %%mm5          \n\t"

  510.         "movq     %%mm0,  (%2)          \n\t"

  511.         "movq     %%mm4, 8(%2)          \n\t"

  512.         "movq     %%mm1,  (%2, %3)      \n\t"

  513.         "movq     %%mm5, 8(%2, %3)      \n\t"

  514.         "add  %%"REG_a", %1             \n\t"

  515.         "add  %%"REG_a", %2             \n\t"

  516.         "subl        $4, %0             \n\t"

  517.         "jnz         1b                 \n\t"

  518.         : "+g"(h), "+r"(pixels),  "+r"(block)

  519.         : "r"((x86_reg)line_size)

  520.         : "%"REG_a, "memory"

  521.         );

  522. }

  523. 

  524. #define CLEAR_BLOCKS(name, n)                           \

  525. static void name(int16_t *blocks)                       \

  526. {                                                       \

  527.     __asm__ volatile (                                  \

  528.         "pxor %%mm7, %%mm7              \n\t"           \

  529.         "mov     %1,        %%"REG_a"   \n\t"           \

  530.         "1:                             \n\t"           \

  531.         "movq %%mm7,   (%0, %%"REG_a")  \n\t"           \

  532.         "movq %%mm7,  8(%0, %%"REG_a")  \n\t"           \

  533.         "movq %%mm7, 16(%0, %%"REG_a")  \n\t"           \

  534.         "movq %%mm7, 24(%0, %%"REG_a")  \n\t"           \

  535.         "add    $32, %%"REG_a"          \n\t"           \

  536.         "js      1b                     \n\t"           \

  537.         :: "r"(((uint8_t *)blocks) + 128 * n),          \

  538.            "i"(-128 * n)                                \

  539.         : "%"REG_a                                      \

  540.         );                                              \

  541. }

  542. CLEAR_BLOCKS(clear_blocks_mmx, 6)

  543. CLEAR_BLOCKS(clear_block_mmx, 1)

  544. 

  545. static void clear_block_sse(int16_t *block)

  546. {

  547.     __asm__ volatile (

  548.         "xorps  %%xmm0, %%xmm0          \n"

  549.         "movaps %%xmm0,    (%0)         \n"

  550.         "movaps %%xmm0,  16(%0)         \n"

  551.         "movaps %%xmm0,  32(%0)         \n"

  552.         "movaps %%xmm0,  48(%0)         \n"

  553.         "movaps %%xmm0,  64(%0)         \n"

  554.         "movaps %%xmm0,  80(%0)         \n"

  555.         "movaps %%xmm0,  96(%0)         \n"

  556.         "movaps %%xmm0, 112(%0)         \n"

  557.         :: "r"(block)

  558.         : "memory"

  559.     );

  560. }

  561. 

  562. static void clear_blocks_sse(int16_t *blocks)

  563. {

  564.     __asm__ volatile (

  565.         "xorps  %%xmm0, %%xmm0              \n"

  566.         "mov        %1,         %%"REG_a"   \n"

  567.         "1:                                 \n"

  568.         "movaps %%xmm0,    (%0, %%"REG_a")  \n"

  569.         "movaps %%xmm0,  16(%0, %%"REG_a")  \n"

  570.         "movaps %%xmm0,  32(%0, %%"REG_a")  \n"

  571.         "movaps %%xmm0,  48(%0, %%"REG_a")  \n"

  572.         "movaps %%xmm0,  64(%0, %%"REG_a")  \n"

  573.         "movaps %%xmm0,  80(%0, %%"REG_a")  \n"

  574.         "movaps %%xmm0,  96(%0, %%"REG_a")  \n"

  575.         "movaps %%xmm0, 112(%0, %%"REG_a")  \n"

  576.         "add      $128,         %%"REG_a"   \n"

  577.         "js         1b                      \n"

  578.         :: "r"(((uint8_t *)blocks) + 128 * 6),

  579.            "i"(-128 * 6)

  580.         : "%"REG_a

  581.     );

  582. }

  583. 

  584. static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)

  585. {

  586.     x86_reg i = 0;

  587.     __asm__ volatile (

  588.         "jmp          2f                \n\t"

  589.         "1:                             \n\t"

  590.         "movq   (%1, %0), %%mm0         \n\t"

  591.         "movq   (%2, %0), %%mm1         \n\t"

  592.         "paddb     %%mm0, %%mm1         \n\t"

  593.         "movq      %%mm1, (%2, %0)      \n\t"

  594.         "movq  8(%1, %0), %%mm0         \n\t"

  595.         "movq  8(%2, %0), %%mm1         \n\t"

  596.         "paddb     %%mm0, %%mm1         \n\t"

  597.         "movq      %%mm1, 8(%2, %0)     \n\t"

  598.         "add         $16, %0            \n\t"

  599.         "2:                             \n\t"

  600.         "cmp          %3, %0            \n\t"

  601.         "js           1b                \n\t"

  602.         : "+r"(i)

  603.         : "r"(src), "r"(dst), "r"((x86_reg)w - 15)

  604.     );

  605.     for ( ; i < w; i++)

  606.         dst[i + 0] += src[i + 0];

  607. }

  608. 

  609. #if HAVE_7REGS

  610. static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,

  611.                                             const uint8_t *diff, int w,

  612.                                             int *left, int *left_top)

  613. {

  614.     x86_reg w2 = -w;

  615.     x86_reg x;

  616.     int l  = *left     & 0xff;

  617.     int tl = *left_top & 0xff;

  618.     int t;

  619.     __asm__ volatile (

  620.         "mov          %7, %3            \n"

  621.         "1:                             \n"

  622.         "movzbl (%3, %4), %2            \n"

  623.         "mov          %2, %k3           \n"

  624.         "sub         %b1, %b3           \n"

  625.         "add         %b0, %b3           \n"

  626.         "mov          %2, %1            \n"

  627.         "cmp          %0, %2            \n"

  628.         "cmovg        %0, %2            \n"

  629.         "cmovg        %1, %0            \n"

  630.         "cmp         %k3, %0            \n"

  631.         "cmovg       %k3, %0            \n"

  632.         "mov          %7, %3            \n"

  633.         "cmp          %2, %0            \n"

  634.         "cmovl        %2, %0            \n"

  635.         "add    (%6, %4), %b0           \n"

  636.         "mov         %b0, (%5, %4)      \n"

  637.         "inc          %4                \n"

  638.         "jl           1b                \n"

  639.         : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)

  640.         : "r"(dst + w), "r"(diff + w), "rm"(top + w)

  641.     );

  642.     *left     = l;

  643.     *left_top = tl;

  644. }

  645. #endif

  646. #endif /* HAVE_INLINE_ASM */

  647. 

  648. void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  649. void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);

  650. 

  651. #if HAVE_INLINE_ASM

  652. /* Draw the edges of width 'w' of an image of size width, height

  653.  * this MMX version can only handle w == 8 || w == 16. */

  654. static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,

  655.                            int w, int h, int sides)

  656. {

  657.     uint8_t *ptr, *last_line;

  658.     int i;

  659. 

  660.     last_line = buf + (height - 1) * wrap;

  661.     /* left and right */

  662.     ptr = buf;

  663.     if (w == 8) {

  664.         __asm__ volatile (

  665.             "1:                             \n\t"

  666.             "movd            (%0), %%mm0    \n\t"

  667.             "punpcklbw      %%mm0, %%mm0    \n\t"

  668.             "punpcklwd      %%mm0, %%mm0    \n\t"

  669.             "punpckldq      %%mm0, %%mm0    \n\t"

  670.             "movq           %%mm0, -8(%0)   \n\t"

  671.             "movq      -8(%0, %2), %%mm1    \n\t"

  672.             "punpckhbw      %%mm1, %%mm1    \n\t"

  673.             "punpckhwd      %%mm1, %%mm1    \n\t"

  674.             "punpckhdq      %%mm1, %%mm1    \n\t"

  675.             "movq           %%mm1, (%0, %2) \n\t"

  676.             "add               %1, %0       \n\t"

  677.             "cmp               %3, %0       \n\t"

  678.             "jb                1b           \n\t"

  679.             : "+r"(ptr)

  680.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  681.             );

  682.     } else {

  683.         __asm__ volatile (

  684.             "1:                                 \n\t"

  685.             "movd            (%0), %%mm0        \n\t"

  686.             "punpcklbw      %%mm0, %%mm0        \n\t"

  687.             "punpcklwd      %%mm0, %%mm0        \n\t"

  688.             "punpckldq      %%mm0, %%mm0        \n\t"

  689.             "movq           %%mm0, -8(%0)       \n\t"

  690.             "movq           %%mm0, -16(%0)      \n\t"

  691.             "movq      -8(%0, %2), %%mm1        \n\t"

  692.             "punpckhbw      %%mm1, %%mm1        \n\t"

  693.             "punpckhwd      %%mm1, %%mm1        \n\t"

  694.             "punpckhdq      %%mm1, %%mm1        \n\t"

  695.             "movq           %%mm1,  (%0, %2)    \n\t"

  696.             "movq           %%mm1, 8(%0, %2)    \n\t"

  697.             "add               %1, %0           \n\t"

  698.             "cmp               %3, %0           \n\t"

  699.             "jb                1b               \n\t"

  700.             : "+r"(ptr)

  701.             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)

  702.             );

  703.     }

  704. 

  705.     /* top and bottom (and hopefully also the corners) */

  706.     if (sides & EDGE_TOP) {

  707.         for (i = 0; i < h; i += 4) {

  708.             ptr = buf - (i + 1) * wrap - w;

  709.             __asm__ volatile (

  710.                 "1:                             \n\t"

  711.                 "movq (%1, %0), %%mm0           \n\t"

  712.                 "movq    %%mm0, (%0)            \n\t"

  713.                 "movq    %%mm0, (%0, %2)        \n\t"

  714.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  715.                 "movq    %%mm0, (%0, %3)        \n\t"

  716.                 "add        $8, %0              \n\t"

  717.                 "cmp        %4, %0              \n\t"

  718.                 "jb         1b                  \n\t"

  719.                 : "+r"(ptr)

  720.                 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),

  721.                   "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)

  722.                 );

  723.         }

  724.     }

  725. 

  726.     if (sides & EDGE_BOTTOM) {

  727.         for (i = 0; i < h; i += 4) {

  728.             ptr = last_line + (i + 1) * wrap - w;

  729.             __asm__ volatile (

  730.                 "1:                             \n\t"

  731.                 "movq (%1, %0), %%mm0           \n\t"

  732.                 "movq    %%mm0, (%0)            \n\t"

  733.                 "movq    %%mm0, (%0, %2)        \n\t"

  734.                 "movq    %%mm0, (%0, %2, 2)     \n\t"

  735.                 "movq    %%mm0, (%0, %3)        \n\t"

  736.                 "add        $8, %0              \n\t"

  737.                 "cmp        %4, %0              \n\t"

  738.                 "jb         1b                  \n\t"

  739.                 : "+r"(ptr)

  740.                 : "r"((x86_reg)last_line - (x86_reg)ptr - w),

  741.                   "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),

  742.                   "r"(ptr + width + 2 * w)

  743.                 );

  744.         }

  745.     }

  746. }

  747. #endif /* HAVE_INLINE_ASM */

  748. 

  749. 

  750. #if HAVE_YASM

  751. #define QPEL_OP(OPNAME, ROUNDER, RND, MMX)                              \

  752. static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \

  753.                                           ptrdiff_t stride)             \

  754. {                                                                       \

  755.     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \

  756. }                                                                       \

  757.                                                                         \

  758. static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \

  759.                                          ptrdiff_t stride)              \

  760. {                                                                       \

  761.     uint64_t temp[8];                                                   \

  762.     uint8_t * const half = (uint8_t*)temp;                              \

  763.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  764.                                                    stride, 8);          \

  765.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  766.                                         stride, stride, 8);             \

  767. }                                                                       \

  768.                                                                         \

  769. static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \

  770.                                          ptrdiff_t stride)              \

  771. {                                                                       \

  772.     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \

  773.                                                    stride, 8);          \

  774. }                                                                       \

  775.                                                                         \

  776. static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \

  777.                                          ptrdiff_t stride)              \

  778. {                                                                       \

  779.     uint64_t temp[8];                                                   \

  780.     uint8_t * const half = (uint8_t*)temp;                              \

  781.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \

  782.                                                    stride, 8);          \

  783.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \

  784.                                         stride, 8);                     \

  785. }                                                                       \

  786.                                                                         \

  787. static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \

  788.                                          ptrdiff_t stride)              \

  789. {                                                                       \

  790.     uint64_t temp[8];                                                   \

  791.     uint8_t * const half = (uint8_t*)temp;                              \

  792.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  793.                                                    8, stride);          \

  794.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \

  795.                                         stride, stride, 8);             \

  796. }                                                                       \

  797.                                                                         \

  798. static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \

  799.                                          ptrdiff_t stride)              \

  800. {                                                                       \

  801.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \

  802.                                                    stride, stride);     \

  803. }                                                                       \

  804.                                                                         \

  805. static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \

  806.                                          ptrdiff_t stride)              \

  807. {                                                                       \

  808.     uint64_t temp[8];                                                   \

  809.     uint8_t * const half = (uint8_t*)temp;                              \

  810.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \

  811.                                                    8, stride);          \

  812.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\

  813.                                         stride, 8);                     \

  814. }                                                                       \

  815.                                                                         \

  816. static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \

  817.                                          ptrdiff_t stride)              \

  818. {                                                                       \

  819.     uint64_t half[8 + 9];                                               \

  820.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  821.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  822.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  823.                                                    stride, 9);          \

  824.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  825.                                         stride, 9);                     \

  826.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  827.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  828.                                         stride, 8, 8);                  \

  829. }                                                                       \

  830.                                                                         \

  831. static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \

  832.                                          ptrdiff_t stride)              \

  833. {                                                                       \

  834.     uint64_t half[8 + 9];                                               \

  835.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  836.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  837.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  838.                                                    stride, 9);          \

  839.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  840.                                         stride, 9);                     \

  841.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  842.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  843.                                         stride, 8, 8);                  \

  844. }                                                                       \

  845.                                                                         \

  846. static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \

  847.                                          ptrdiff_t stride)              \

  848. {                                                                       \

  849.     uint64_t half[8 + 9];                                               \

  850.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  851.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  852.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  853.                                                    stride, 9);          \

  854.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \

  855.                                         stride, 9);                     \

  856.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  857.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  858.                                         stride, 8, 8);                  \

  859. }                                                                       \

  860.                                                                         \

  861. static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \

  862.                                          ptrdiff_t stride)              \

  863. {                                                                       \

  864.     uint64_t half[8 + 9];                                               \

  865.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  866.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  867.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  868.                                                    stride, 9);          \

  869.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  870.                                         stride, 9);                     \

  871.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  872.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  873.                                         stride, 8, 8);                  \

  874. }                                                                       \

  875.                                                                         \

  876. static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \

  877.                                          ptrdiff_t stride)              \

  878. {                                                                       \

  879.     uint64_t half[8 + 9];                                               \

  880.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  881.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  882.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  883.                                                    stride, 9);          \

  884.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  885.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \

  886.                                         stride, 8, 8);                  \

  887. }                                                                       \

  888.                                                                         \

  889. static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \

  890.                                          ptrdiff_t stride)              \

  891. {                                                                       \

  892.     uint64_t half[8 + 9];                                               \

  893.     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \

  894.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  895.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  896.                                                    stride, 9);          \

  897.     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\

  898.     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \

  899.                                         stride, 8, 8);                  \

  900. }                                                                       \

  901.                                                                         \

  902. static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \

  903.                                          ptrdiff_t stride)              \

  904. {                                                                       \

  905.     uint64_t half[8 + 9];                                               \

  906.     uint8_t * const halfH = ((uint8_t*)half);                           \

  907.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  908.                                                    stride, 9);          \

  909.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \

  910.                                         8, stride, 9);                  \

  911.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  912.                                                    stride, 8);          \

  913. }                                                                       \

  914.                                                                         \

  915. static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \

  916.                                          ptrdiff_t stride)              \

  917. {                                                                       \

  918.     uint64_t half[8 + 9];                                               \

  919.     uint8_t * const halfH = ((uint8_t*)half);                           \

  920.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  921.                                                    stride, 9);          \

  922.     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \

  923.                                         stride, 9);                     \

  924.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  925.                                                    stride, 8);          \

  926. }                                                                       \

  927.                                                                         \

  928. static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \

  929.                                          ptrdiff_t stride)              \

  930. {                                                                       \

  931.     uint64_t half[9];                                                   \

  932.     uint8_t * const halfH = ((uint8_t*)half);                           \

  933.     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \

  934.                                                    stride, 9);          \

  935.     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \

  936.                                                    stride, 8);          \

  937. }                                                                       \

  938.                                                                         \

  939. static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \

  940.                                            ptrdiff_t stride)            \

  941. {                                                                       \

  942.     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \

  943. }                                                                       \

  944.                                                                         \

  945. static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \

  946.                                           ptrdiff_t stride)             \

  947. {                                                                       \

  948.     uint64_t temp[32];                                                  \

  949.     uint8_t * const half = (uint8_t*)temp;                              \

  950.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  951.                                                     stride, 16);        \

  952.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  953.                                          stride, 16);                   \

  954. }                                                                       \

  955.                                                                         \

  956. static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \

  957.                                           ptrdiff_t stride)             \

  958. {                                                                       \

  959.     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \

  960.                                                     stride, stride, 16);\

  961. }                                                                       \

  962.                                                                         \

  963. static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \

  964.                                           ptrdiff_t stride)             \

  965. {                                                                       \

  966.     uint64_t temp[32];                                                  \

  967.     uint8_t * const half = (uint8_t*)temp;                              \

  968.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \

  969.                                                     stride, 16);        \

  970.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \

  971.                                          stride, stride, 16);           \

  972. }                                                                       \

  973.                                                                         \

  974. static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \

  975.                                           ptrdiff_t stride)             \

  976. {                                                                       \

  977.     uint64_t temp[32];                                                  \

  978.     uint8_t * const half = (uint8_t*)temp;                              \

  979.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  980.                                                     stride);            \

  981.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \

  982.                                          stride, 16);                   \

  983. }                                                                       \

  984.                                                                         \

  985. static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \

  986.                                           ptrdiff_t stride)             \

  987. {                                                                       \

  988.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \

  989.                                                     stride, stride);    \

  990. }                                                                       \

  991.                                                                         \

  992. static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \

  993.                                           ptrdiff_t stride)             \

  994. {                                                                       \

  995.     uint64_t temp[32];                                                  \

  996.     uint8_t * const half = (uint8_t*)temp;                              \

  997.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \

  998.                                                     stride);            \

  999.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \

  1000.                                          stride, stride, 16);           \

  1001. }                                                                       \

  1002.                                                                         \

  1003. static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1004.                                           ptrdiff_t stride)             \

  1005. {                                                                       \

  1006.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1007.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1008.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1009.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1010.                                                     stride, 17);        \

  1011.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1012.                                          stride, 17);                   \

  1013.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1014.                                                     16, 16);            \

  1015.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1016.                                          stride, 16, 16);               \

  1017. }                                                                       \

  1018.                                                                         \

  1019. static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1020.                                           ptrdiff_t stride)             \

  1021. {                                                                       \

  1022.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1023.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1024.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1025.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1026.                                                     stride, 17);        \

  1027.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1028.                                          stride, 17);                   \

  1029.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1030.                                                     16, 16);            \

  1031.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1032.                                          stride, 16, 16);               \

  1033. }                                                                       \

  1034.                                                                         \

  1035. static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1036.                                           ptrdiff_t stride)             \

  1037. {                                                                       \

  1038.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1039.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1040.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1041.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1042.                                                     stride, 17);        \

  1043.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1044.                                          stride, 17);                   \

  1045.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1046.                                                     16, 16);            \

  1047.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1048.                                          stride, 16, 16);               \

  1049. }                                                                       \

  1050.                                                                         \

  1051. static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1052.                                           ptrdiff_t stride)             \

  1053. {                                                                       \

  1054.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1055.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1056.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1057.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1058.                                                     stride, 17);        \

  1059.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1060.                                          stride, 17);                   \

  1061.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1062.                                                     16, 16);            \

  1063.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1064.                                          stride, 16, 16);               \

  1065. }                                                                       \

  1066.                                                                         \

  1067. static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1068.                                           ptrdiff_t stride)             \

  1069. {                                                                       \

  1070.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1071.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1072.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1073.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1074.                                                     stride, 17);        \

  1075.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1076.                                                     16, 16);            \

  1077.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \

  1078.                                          stride, 16, 16);               \

  1079. }                                                                       \

  1080.                                                                         \

  1081. static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1082.                                           ptrdiff_t stride)             \

  1083. {                                                                       \

  1084.     uint64_t half[16 * 2 + 17 * 2];                                     \

  1085.     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \

  1086.     uint8_t * const halfHV = ((uint8_t*)half);                          \

  1087.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1088.                                                     stride, 17);        \

  1089.     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \

  1090.                                                     16, 16);            \

  1091.     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \

  1092.                                          stride, 16, 16);               \

  1093. }                                                                       \

  1094.                                                                         \

  1095. static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1096.                                           ptrdiff_t stride)             \

  1097. {                                                                       \

  1098.     uint64_t half[17 * 2];                                              \

  1099.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1100.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1101.                                                     stride, 17);        \

  1102.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \

  1103.                                          stride, 17);                   \

  1104.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1105.                                                     stride, 16);        \

  1106. }                                                                       \

  1107.                                                                         \

  1108. static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1109.                                           ptrdiff_t stride)             \

  1110. {                                                                       \

  1111.     uint64_t half[17 * 2];                                              \

  1112.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1113.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1114.                                                     stride, 17);        \

  1115.     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \

  1116.                                          stride, 17);                   \

  1117.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1118.                                                     stride, 16);        \

  1119. }                                                                       \

  1120.                                                                         \

  1121. static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \

  1122.                                           ptrdiff_t stride)             \

  1123. {                                                                       \

  1124.     uint64_t half[17 * 2];                                              \

  1125.     uint8_t * const halfH = ((uint8_t*)half);                           \

  1126.     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \

  1127.                                                     stride, 17);        \

  1128.     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \

  1129.                                                     stride, 16);        \

  1130. }

  1131. 

  1132. QPEL_OP(put_,          ff_pw_16, _,        mmxext)

  1133. QPEL_OP(avg_,          ff_pw_16, _,        mmxext)

  1134. QPEL_OP(put_no_rnd_,   ff_pw_15, _no_rnd_, mmxext)

  1135. #endif /* HAVE_YASM */

  1136. 

  1137. 

  1138. #if HAVE_INLINE_ASM

  1139. void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1140. {

  1141.   put_pixels8_xy2_mmx(dst, src, stride, 8);

  1142. }

  1143. void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1144. {

  1145.   put_pixels16_xy2_mmx(dst, src, stride, 16);

  1146. }

  1147. void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1148. {

  1149.   avg_pixels8_xy2_mmx(dst, src, stride, 8);

  1150. }

  1151. void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1152. {

  1153.   avg_pixels16_xy2_mmx(dst, src, stride, 16);

  1154. }

  1155. 

  1156. static void gmc_mmx(uint8_t *dst, uint8_t *src,

  1157.                     int stride, int h, int ox, int oy,

  1158.                     int dxx, int dxy, int dyx, int dyy,

  1159.                     int shift, int r, int width, int height)

  1160. {

  1161.     const int w    = 8;

  1162.     const int ix   = ox  >> (16 + shift);

  1163.     const int iy   = oy  >> (16 + shift);

  1164.     const int oxs  = ox  >> 4;

  1165.     const int oys  = oy  >> 4;

  1166.     const int dxxs = dxx >> 4;

  1167.     const int dxys = dxy >> 4;

  1168.     const int dyxs = dyx >> 4;

  1169.     const int dyys = dyy >> 4;

  1170.     const uint16_t r4[4]   = { r, r, r, r };

  1171.     const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };

  1172.     const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };

  1173.     const uint64_t shift2 = 2 * shift;

  1174.     int x, y;

  1175. 

  1176.     const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);

  1177.     const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);

  1178.     const int dxh = dxy * (h - 1);

  1179.     const int dyw = dyx * (w - 1);

  1180.     if ( // non-constant fullpel offset (3% of blocks)

  1181.         ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |

  1182.          (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)

  1183.         // uses more than 16 bits of subpel mv (only at huge resolution)

  1184.         || (dxx | dxy | dyx | dyy) & 15 ||

  1185.         (unsigned)ix >= width  - w ||

  1186.         (unsigned)iy >= height - h) {

  1187.         // FIXME could still use mmx for some of the rows

  1188.         ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,

  1189.                  shift, r, width, height);

  1190.         return;

  1191.     }

  1192. 

  1193.     src += ix + iy * stride;

  1194. 

  1195.     __asm__ volatile (

  1196.         "movd         %0, %%mm6         \n\t"

  1197.         "pxor      %%mm7, %%mm7         \n\t"

  1198.         "punpcklwd %%mm6, %%mm6         \n\t"

  1199.         "punpcklwd %%mm6, %%mm6         \n\t"

  1200.         :: "r"(1<<shift)

  1201.     );

  1202. 

  1203.     for (x = 0; x < w; x += 4) {

  1204.         uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),

  1205.                             oxs - dxys + dxxs * (x + 1),

  1206.                             oxs - dxys + dxxs * (x + 2),

  1207.                             oxs - dxys + dxxs * (x + 3) };

  1208.         uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),

  1209.                             oys - dyys + dyxs * (x + 1),

  1210.                             oys - dyys + dyxs * (x + 2),

  1211.                             oys - dyys + dyxs * (x + 3) };

  1212. 

  1213.         for (y = 0; y < h; y++) {

  1214.             __asm__ volatile (

  1215.                 "movq      %0, %%mm4    \n\t"

  1216.                 "movq      %1, %%mm5    \n\t"

  1217.                 "paddw     %2, %%mm4    \n\t"

  1218.                 "paddw     %3, %%mm5    \n\t"

  1219.                 "movq   %%mm4, %0       \n\t"

  1220.                 "movq   %%mm5, %1       \n\t"

  1221.                 "psrlw    $12, %%mm4    \n\t"

  1222.                 "psrlw    $12, %%mm5    \n\t"

  1223.                 : "+m"(*dx4), "+m"(*dy4)

  1224.                 : "m"(*dxy4), "m"(*dyy4)

  1225.             );

  1226. 

  1227.             __asm__ volatile (

  1228.                 "movq      %%mm6, %%mm2 \n\t"

  1229.                 "movq      %%mm6, %%mm1 \n\t"

  1230.                 "psubw     %%mm4, %%mm2 \n\t"

  1231.                 "psubw     %%mm5, %%mm1 \n\t"

  1232.                 "movq      %%mm2, %%mm0 \n\t"

  1233.                 "movq      %%mm4, %%mm3 \n\t"

  1234.                 "pmullw    %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)

  1235.                 "pmullw    %%mm5, %%mm3 \n\t" // dx * dy

  1236.                 "pmullw    %%mm5, %%mm2 \n\t" // (s - dx) * dy

  1237.                 "pmullw    %%mm4, %%mm1 \n\t" // dx * (s - dy)

  1238. 

  1239.                 "movd         %4, %%mm5 \n\t"

  1240.                 "movd         %3, %%mm4 \n\t"

  1241.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1242.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1243.                 "pmullw    %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy

  1244.                 "pmullw    %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy

  1245. 

  1246.                 "movd         %2, %%mm5 \n\t"

  1247.                 "movd         %1, %%mm4 \n\t"

  1248.                 "punpcklbw %%mm7, %%mm5 \n\t"

  1249.                 "punpcklbw %%mm7, %%mm4 \n\t"

  1250.                 "pmullw    %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)

  1251.                 "pmullw    %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)

  1252.                 "paddw        %5, %%mm1 \n\t"

  1253.                 "paddw     %%mm3, %%mm2 \n\t"

  1254.                 "paddw     %%mm1, %%mm0 \n\t"

  1255.                 "paddw     %%mm2, %%mm0 \n\t"

  1256. 

  1257.                 "psrlw        %6, %%mm0 \n\t"

  1258.                 "packuswb  %%mm0, %%mm0 \n\t"

  1259.                 "movd      %%mm0, %0    \n\t"

  1260. 

  1261.                 : "=m"(dst[x + y * stride])

  1262.                 : "m"(src[0]), "m"(src[1]),

  1263.                   "m"(src[stride]), "m"(src[stride + 1]),

  1264.                   "m"(*r4), "m"(shift2)

  1265.             );

  1266.             src += stride;

  1267.         }

  1268.         src += 4 - h * stride;

  1269.     }

  1270. }

  1271. #endif /* HAVE_INLINE_ASM */

  1272. 

  1273. void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1274.                           ptrdiff_t line_size, int h);

  1275. void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,

  1276.                           ptrdiff_t line_size, int h);

  1277. 

  1278. #if HAVE_INLINE_ASM

  1279. 

  1280. /* CAVS-specific */

  1281. void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1282. {

  1283.     put_pixels8_mmx(dst, src, stride, 8);

  1284. }

  1285. 

  1286. void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1287. {

  1288.     avg_pixels8_mmx(dst, src, stride, 8);

  1289. }

  1290. 

  1291. void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1292. {

  1293.     put_pixels16_mmx(dst, src, stride, 16);

  1294. }

  1295. 

  1296. void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, ptrdiff_t stride)

  1297. {

  1298.     avg_pixels16_mmx(dst, src, stride, 16);

  1299. }

  1300. 

  1301. /* VC-1-specific */

  1302. void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,

  1303.                                ptrdiff_t stride, int rnd)

  1304. {

  1305.     put_pixels8_mmx(dst, src, stride, 8);

  1306. }

  1307. 

  1308. static void vector_clipf_sse(float *dst, const float *src,

  1309.                              float min, float max, int len)

  1310. {

  1311.     x86_reg i = (len - 16) * 4;

  1312.     __asm__ volatile (

  1313.         "movss          %3, %%xmm4      \n\t"

  1314.         "movss          %4, %%xmm5      \n\t"

  1315.         "shufps $0, %%xmm4, %%xmm4      \n\t"

  1316.         "shufps $0, %%xmm5, %%xmm5      \n\t"

  1317.         "1:                             \n\t"

  1318.         "movaps   (%2, %0), %%xmm0      \n\t" // 3/1 on intel

  1319.         "movaps 16(%2, %0), %%xmm1      \n\t"

  1320.         "movaps 32(%2, %0), %%xmm2      \n\t"

  1321.         "movaps 48(%2, %0), %%xmm3      \n\t"

  1322.         "maxps      %%xmm4, %%xmm0      \n\t"

  1323.         "maxps      %%xmm4, %%xmm1      \n\t"

  1324.         "maxps      %%xmm4, %%xmm2      \n\t"

  1325.         "maxps      %%xmm4, %%xmm3      \n\t"

  1326.         "minps      %%xmm5, %%xmm0      \n\t"

  1327.         "minps      %%xmm5, %%xmm1      \n\t"

  1328.         "minps      %%xmm5, %%xmm2      \n\t"

  1329.         "minps      %%xmm5, %%xmm3      \n\t"

  1330.         "movaps     %%xmm0,   (%1, %0)  \n\t"

  1331.         "movaps     %%xmm1, 16(%1, %0)  \n\t"

  1332.         "movaps     %%xmm2, 32(%1, %0)  \n\t"

  1333.         "movaps     %%xmm3, 48(%1, %0)  \n\t"

  1334.         "sub           $64, %0          \n\t"

  1335.         "jge            1b              \n\t"

  1336.         : "+&r"(i)

  1337.         : "r"(dst), "r"(src), "m"(min), "m"(max)

  1338.         : "memory"

  1339.     );

  1340. }

  1341. 

  1342. #endif /* HAVE_INLINE_ASM */

  1343. 

  1344. int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,

  1345.                                       int order);

  1346. int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,

  1347.                                     int order);

  1348. int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,

  1349.                                                const int16_t *v3,

  1350.                                                int order, int mul);

  1351. int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,

  1352.                                              const int16_t *v3,

  1353.                                              int order, int mul);

  1354. int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,

  1355.                                               const int16_t *v3,

  1356.                                               int order, int mul);

  1357. 

  1358. void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,

  1359.                                         const int16_t *window, unsigned int len);

  1360. void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,

  1361.                                       const int16_t *window, unsigned int len);

  1362. void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,

  1363.                                   const int16_t *window, unsigned int len);

  1364. void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,

  1365.                                 const int16_t *window, unsigned int len);

  1366. void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,

  1367.                                  const int16_t *window, unsigned int len);

  1368. void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,

  1369.                                       const int16_t *window, unsigned int len);

  1370. 

  1371. void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);

  1372. void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);

  1373. 

  1374. void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,

  1375.                                           const uint8_t *diff, int w,

  1376.                                           int *left, int *left_top);

  1377. int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,

  1378.                                        int w, int left);

  1379. int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,

  1380.                                       int w, int left);

  1381. 

  1382. void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,

  1383.                                    int32_t min, int32_t max, unsigned int len);

  1384. void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,

  1385.                                    int32_t min, int32_t max, unsigned int len);

  1386. void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,

  1387.                                    int32_t min, int32_t max, unsigned int len);

  1388. void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,

  1389.                                    int32_t min, int32_t max, unsigned int len);

  1390. 

  1391. #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \

  1392.     do {                                                                     \

  1393.     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \

  1394.     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \

  1395.     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \

  1396.     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \

  1397.     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \

  1398.     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \

  1399.     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \

  1400.     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \

  1401.     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \

  1402.     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \

  1403.     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \

  1404.     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \

  1405.     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \

  1406.     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \

  1407.     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \

  1408.     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \

  1409.     } while (0)

  1410. 

  1411. #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU)                                     \

  1412.     do {                                                                        \

  1413.         c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _     ## CPU; \

  1414.         c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_  ## CPU; \

  1415.         c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_  ## CPU; \

  1416.         c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \

  1417.     } while (0)

  1418. 

  1419. static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,

  1420.                                      int mm_flags)

  1421. {

  1422.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1423. 

  1424. #if HAVE_INLINE_ASM

  1425.     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;

  1426.     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;

  1427.     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;

  1428. 

  1429.     if (!high_bit_depth) {

  1430.         c->clear_block  = clear_block_mmx;

  1431.         c->clear_blocks = clear_blocks_mmx;

  1432.         c->draw_edges   = draw_edges_mmx;

  1433. 

  1434.         SET_HPEL_FUNCS(put,        [0], 16, mmx);

  1435.         SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);

  1436.         SET_HPEL_FUNCS(avg,        [0], 16, mmx);

  1437.         SET_HPEL_FUNCS(avg_no_rnd,    , 16, mmx);

  1438.         SET_HPEL_FUNCS(put,        [1],  8, mmx);

  1439.         SET_HPEL_FUNCS(put_no_rnd, [1],  8, mmx);

  1440.         SET_HPEL_FUNCS(avg,        [1],  8, mmx);

  1441. 

  1442.         switch (avctx->idct_algo) {

  1443.         case FF_IDCT_AUTO:

  1444.         case FF_IDCT_SIMPLEMMX:

  1445.             c->idct_put              = ff_simple_idct_put_mmx;

  1446.             c->idct_add              = ff_simple_idct_add_mmx;

  1447.             c->idct                  = ff_simple_idct_mmx;

  1448.             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;

  1449.             break;

  1450.         case FF_IDCT_XVIDMMX:

  1451.             c->idct_put              = ff_idct_xvid_mmx_put;

  1452.             c->idct_add              = ff_idct_xvid_mmx_add;

  1453.             c->idct                  = ff_idct_xvid_mmx;

  1454.             break;

  1455.         }

  1456.     }

  1457. 

  1458.     c->gmc = gmc_mmx;

  1459. 

  1460.     c->add_bytes = add_bytes_mmx;

  1461. #endif /* HAVE_INLINE_ASM */

  1462. 

  1463. #if HAVE_YASM

  1464.     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {

  1465.         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;

  1466.         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;

  1467.     }

  1468. 

  1469.     c->vector_clip_int32 = ff_vector_clip_int32_mmx;

  1470. #endif

  1471. 

  1472. }

  1473. 

  1474. static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,

  1475.                                         int mm_flags)

  1476. {

  1477.     const int bit_depth      = avctx->bits_per_raw_sample;

  1478.     const int high_bit_depth = bit_depth > 8;

  1479. 

  1480. #if HAVE_YASM

  1481.     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );

  1482.     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );

  1483. 

  1484.     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );

  1485.     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );

  1486.     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );

  1487.     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );

  1488. 

  1489.     if (!high_bit_depth) {

  1490.         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_mmxext;

  1491.         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_mmxext;

  1492. 

  1493.         c->avg_pixels_tab[0][0] = ff_avg_pixels16_mmxext;

  1494.         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_mmxext;

  1495.         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_mmxext;

  1496. 

  1497.         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_mmxext;

  1498.         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_mmxext;

  1499. 

  1500.         c->avg_pixels_tab[1][0] = ff_avg_pixels8_mmxext;

  1501.         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_mmxext;

  1502.         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_mmxext;

  1503.     }

  1504. 

  1505.     if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  1506.         if (!high_bit_depth) {

  1507.             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_mmxext;

  1508.             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_mmxext;

  1509.             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_mmxext;

  1510.             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_mmxext;

  1511. 

  1512.             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_mmxext;

  1513.             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_mmxext;

  1514.         }

  1515.     }

  1516. #endif /* HAVE_YASM */

  1517. 

  1518. #if HAVE_INLINE_ASM

  1519.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1520.         c->idct_put = ff_idct_xvid_mmxext_put;

  1521.         c->idct_add = ff_idct_xvid_mmxext_add;

  1522.         c->idct     = ff_idct_xvid_mmxext;

  1523.     }

  1524. #endif /* HAVE_INLINE_ASM */

  1525. 

  1526. #if HAVE_MMXEXT_EXTERNAL

  1527.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  1528.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  1529.         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_mmxext;

  1530.         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_mmxext;

  1531.     }

  1532. 

  1533.     /* slower than cmov version on AMD */

  1534.     if (!(mm_flags & AV_CPU_FLAG_3DNOW))

  1535.         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;

  1536. 

  1537.     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;

  1538.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;

  1539. 

  1540.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1541.         c->apply_window_int16 = ff_apply_window_int16_mmxext;

  1542.     } else {

  1543.         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;

  1544.     }

  1545. #endif /* HAVE_MMXEXT_EXTERNAL */

  1546. }

  1547. 

  1548. static av_cold void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,

  1549.                                        int mm_flags)

  1550. {

  1551.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1552. 

  1553. #if HAVE_YASM

  1554.     if (!high_bit_depth) {

  1555.         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_3dnow;

  1556.         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_3dnow;

  1557. 

  1558.         c->avg_pixels_tab[0][0] = ff_avg_pixels16_3dnow;

  1559.         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_3dnow;

  1560.         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_3dnow;

  1561. 

  1562.         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_3dnow;

  1563.         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_3dnow;

  1564. 

  1565.         c->avg_pixels_tab[1][0] = ff_avg_pixels8_3dnow;

  1566.         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_3dnow;

  1567.         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_3dnow;

  1568. 

  1569.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)){

  1570.             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_3dnow;

  1571.             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_3dnow;

  1572.             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_3dnow;

  1573.             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_3dnow;

  1574. 

  1575.             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_3dnow;

  1576.             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_3dnow;

  1577.         }

  1578.     }

  1579. 

  1580.     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||

  1581.                                avctx->codec_id == AV_CODEC_ID_THEORA)) {

  1582.         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_3dnow;

  1583.         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_3dnow;

  1584.     }

  1585. #endif /* HAVE_YASM */

  1586. }

  1587. 

  1588. static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,

  1589.                                      int mm_flags)

  1590. {

  1591.     const int high_bit_depth = avctx->bits_per_raw_sample > 8;

  1592. 

  1593. #if HAVE_INLINE_ASM

  1594.     if (!high_bit_depth) {

  1595.         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {

  1596.             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */

  1597.             c->clear_block  = clear_block_sse;

  1598.             c->clear_blocks = clear_blocks_sse;

  1599.         }

  1600.     }

  1601. 

  1602.     c->vector_clipf = vector_clipf_sse;

  1603. #endif /* HAVE_INLINE_ASM */

  1604. }

  1605. 

  1606. static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,

  1607.                                       int mm_flags)

  1608. {

  1609.     const int bit_depth      = avctx->bits_per_raw_sample;

  1610.     const int high_bit_depth = bit_depth > 8;

  1611. 

  1612. #if HAVE_SSE2_INLINE

  1613.     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {

  1614.         c->idct_put              = ff_idct_xvid_sse2_put;

  1615.         c->idct_add              = ff_idct_xvid_sse2_add;

  1616.         c->idct                  = ff_idct_xvid_sse2;

  1617.         c->idct_permutation_type = FF_SSE2_IDCT_PERM;

  1618.     }

  1619. #endif /* HAVE_SSE2_INLINE */

  1620. 

  1621. #if HAVE_SSE2_EXTERNAL

  1622.     if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  1623.         // these functions are slower than mmx on AMD, but faster on Intel

  1624.         if (!high_bit_depth) {

  1625.             c->put_pixels_tab[0][0]        = ff_put_pixels16_sse2;

  1626.             c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;

  1627.             c->avg_pixels_tab[0][0]        = ff_avg_pixels16_sse2;

  1628.         }

  1629.     }

  1630. 

  1631.     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;

  1632.     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;

  1633.     if (mm_flags & AV_CPU_FLAG_ATOM) {

  1634.         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;

  1635.     } else {

  1636.         c->vector_clip_int32 = ff_vector_clip_int32_sse2;

  1637.     }

  1638.     if (avctx->flags & CODEC_FLAG_BITEXACT) {

  1639.         c->apply_window_int16 = ff_apply_window_int16_sse2;

  1640.     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {

  1641.         c->apply_window_int16 = ff_apply_window_int16_round_sse2;

  1642.     }

  1643.     c->bswap_buf = ff_bswap32_buf_sse2;

  1644. #endif /* HAVE_SSE2_EXTERNAL */

  1645. }

  1646. 

  1647. static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,

  1648.                                        int mm_flags)

  1649. {

  1650. #if HAVE_SSSE3_EXTERNAL

  1651.     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;

  1652.     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe

  1653.         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;

  1654. 

  1655.     if (mm_flags & AV_CPU_FLAG_ATOM)

  1656.         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;

  1657.     else

  1658.         c->apply_window_int16 = ff_apply_window_int16_ssse3;

  1659.     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit

  1660.         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;

  1661.     c->bswap_buf = ff_bswap32_buf_ssse3;

  1662. #endif /* HAVE_SSSE3_EXTERNAL */

  1663. }

  1664. 

  1665. static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,

  1666.                                       int mm_flags)

  1667. {

  1668. #if HAVE_SSE4_EXTERNAL

  1669.     c->vector_clip_int32 = ff_vector_clip_int32_sse4;

  1670. #endif /* HAVE_SSE4_EXTERNAL */

  1671. }

  1672. 

  1673. av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)

  1674. {

  1675.     int mm_flags = av_get_cpu_flags();

  1676. 

  1677. #if HAVE_7REGS && HAVE_INLINE_ASM

  1678.     if (mm_flags & AV_CPU_FLAG_CMOV)

  1679.         c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;

  1680. #endif

  1681. 

  1682.     if (mm_flags & AV_CPU_FLAG_MMX)

  1683.         dsputil_init_mmx(c, avctx, mm_flags);

  1684. 

  1685.     if (mm_flags & AV_CPU_FLAG_MMXEXT)

  1686.         dsputil_init_mmxext(c, avctx, mm_flags);

  1687. 

  1688.     if (mm_flags & AV_CPU_FLAG_3DNOW)

  1689.         dsputil_init_3dnow(c, avctx, mm_flags);

  1690. 

  1691.     if (mm_flags & AV_CPU_FLAG_SSE)

  1692.         dsputil_init_sse(c, avctx, mm_flags);

  1693. 

  1694.     if (mm_flags & AV_CPU_FLAG_SSE2)

  1695.         dsputil_init_sse2(c, avctx, mm_flags);

  1696. 

  1697.     if (mm_flags & AV_CPU_FLAG_SSSE3)

  1698.         dsputil_init_ssse3(c, avctx, mm_flags);

  1699. 

  1700.     if (mm_flags & AV_CPU_FLAG_SSE4)

  1701.         dsputil_init_sse4(c, avctx, mm_flags);

  1702. 

  1703.     if (CONFIG_ENCODERS)

  1704.         ff_dsputilenc_init_mmx(c, avctx);

  1705. }