falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
26287 Commits
32 Branches
1.0GB
Tree: adbb9bcaa7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'adbb9bcaa7'
${ noResults }
FFmpeg/libavcodec/x86/h264_qpel_mmx.c

1210 lines
53KB
Raw Blame History 

  1. /*

  2.  * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt

  3.  *

  4.  * This file is part of FFmpeg.

  5.  *

  6.  * FFmpeg is free software; you can redistribute it and/or

  7.  * modify it under the terms of the GNU Lesser General Public

  8.  * License as published by the Free Software Foundation; either

  9.  * version 2.1 of the License, or (at your option) any later version.

  10.  *

  11.  * FFmpeg is distributed in the hope that it will be useful,

  12.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  13.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  14.  * Lesser General Public License for more details.

  15.  *

  16.  * You should have received a copy of the GNU Lesser General Public

  17.  * License along with FFmpeg; if not, write to the Free Software

  18.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  19.  */

  20. 

  21. #include "dsputil_mmx.h"

  22. 

  23. /***********************************/

  24. /* motion compensation */

  25. 

  26. #define QPEL_H264V_MM(A,B,C,D,E,F,OP,T,Z,d,q)\

  27.         "mov"#q" "#C", "#T"         \n\t"\

  28.         "mov"#d" (%0), "#F"         \n\t"\

  29.         "paddw "#D", "#T"           \n\t"\

  30.         "psllw $2, "#T"             \n\t"\

  31.         "psubw "#B", "#T"           \n\t"\

  32.         "psubw "#E", "#T"           \n\t"\

  33.         "punpcklbw "#Z", "#F"       \n\t"\

  34.         "pmullw %4, "#T"            \n\t"\

  35.         "paddw %5, "#A"             \n\t"\

  36.         "add %2, %0                 \n\t"\

  37.         "paddw "#F", "#A"           \n\t"\

  38.         "paddw "#A", "#T"           \n\t"\

  39.         "psraw $5, "#T"             \n\t"\

  40.         "packuswb "#T", "#T"        \n\t"\

  41.         OP(T, (%1), A, d)\

  42.         "add %3, %1                 \n\t"

  43. 

  44. #define QPEL_H264HV_MM(A,B,C,D,E,F,OF,T,Z,d,q)\

  45.         "mov"#q" "#C", "#T"         \n\t"\

  46.         "mov"#d" (%0), "#F"         \n\t"\

  47.         "paddw "#D", "#T"           \n\t"\

  48.         "psllw $2, "#T"             \n\t"\

  49.         "paddw %4, "#A"             \n\t"\

  50.         "psubw "#B", "#T"           \n\t"\

  51.         "psubw "#E", "#T"           \n\t"\

  52.         "punpcklbw "#Z", "#F"       \n\t"\

  53.         "pmullw %3, "#T"            \n\t"\

  54.         "paddw "#F", "#A"           \n\t"\

  55.         "add %2, %0                 \n\t"\

  56.         "paddw "#A", "#T"           \n\t"\

  57.         "mov"#q" "#T", "#OF"(%1)    \n\t"

  58. 

  59. #define QPEL_H264V(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%mm6,%%mm7,d,q)

  60. #define QPEL_H264HV(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%mm6,%%mm7,d,q)

  61. #define QPEL_H264V_XMM(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%xmm6,%%xmm7,q,dqa)

  62. #define QPEL_H264HV_XMM(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%xmm6,%%xmm7,q,dqa)

  63. 

  64. 

  65. #define QPEL_H264(OPNAME, OP, MMX)\

  66. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  67.     int h=4;\

  68. \

  69.     __asm__ volatile(\

  70.         "pxor %%mm7, %%mm7          \n\t"\

  71.         "movq "MANGLE(ff_pw_5) ", %%mm4\n\t"\

  72.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  73.         "1:                         \n\t"\

  74.         "movd  -1(%0), %%mm1        \n\t"\

  75.         "movd    (%0), %%mm2        \n\t"\

  76.         "movd   1(%0), %%mm3        \n\t"\

  77.         "movd   2(%0), %%mm0        \n\t"\

  78.         "punpcklbw %%mm7, %%mm1     \n\t"\

  79.         "punpcklbw %%mm7, %%mm2     \n\t"\

  80.         "punpcklbw %%mm7, %%mm3     \n\t"\

  81.         "punpcklbw %%mm7, %%mm0     \n\t"\

  82.         "paddw %%mm0, %%mm1         \n\t"\

  83.         "paddw %%mm3, %%mm2         \n\t"\

  84.         "movd  -2(%0), %%mm0        \n\t"\

  85.         "movd   3(%0), %%mm3        \n\t"\

  86.         "punpcklbw %%mm7, %%mm0     \n\t"\

  87.         "punpcklbw %%mm7, %%mm3     \n\t"\

  88.         "paddw %%mm3, %%mm0         \n\t"\

  89.         "psllw $2, %%mm2            \n\t"\

  90.         "psubw %%mm1, %%mm2         \n\t"\

  91.         "pmullw %%mm4, %%mm2        \n\t"\

  92.         "paddw %%mm5, %%mm0         \n\t"\

  93.         "paddw %%mm2, %%mm0         \n\t"\

  94.         "psraw $5, %%mm0            \n\t"\

  95.         "packuswb %%mm0, %%mm0      \n\t"\

  96.         OP(%%mm0, (%1),%%mm6, d)\

  97.         "add %3, %0                 \n\t"\

  98.         "add %4, %1                 \n\t"\

  99.         "decl %2                    \n\t"\

  100.         " jnz 1b                    \n\t"\

  101.         : "+a"(src), "+c"(dst), "+g"(h)\

  102.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  103.         : "memory"\

  104.     );\

  105. }\

  106. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  107.     int h=4;\

  108.     __asm__ volatile(\

  109.         "pxor %%mm7, %%mm7          \n\t"\

  110.         "movq %0, %%mm4             \n\t"\

  111.         "movq %1, %%mm5             \n\t"\

  112.         :: "m"(ff_pw_5), "m"(ff_pw_16)\

  113.     );\

  114.     do{\

  115.     __asm__ volatile(\

  116.         "movd  -1(%0), %%mm1        \n\t"\

  117.         "movd    (%0), %%mm2        \n\t"\

  118.         "movd   1(%0), %%mm3        \n\t"\

  119.         "movd   2(%0), %%mm0        \n\t"\

  120.         "punpcklbw %%mm7, %%mm1     \n\t"\

  121.         "punpcklbw %%mm7, %%mm2     \n\t"\

  122.         "punpcklbw %%mm7, %%mm3     \n\t"\

  123.         "punpcklbw %%mm7, %%mm0     \n\t"\

  124.         "paddw %%mm0, %%mm1         \n\t"\

  125.         "paddw %%mm3, %%mm2         \n\t"\

  126.         "movd  -2(%0), %%mm0        \n\t"\

  127.         "movd   3(%0), %%mm3        \n\t"\

  128.         "punpcklbw %%mm7, %%mm0     \n\t"\

  129.         "punpcklbw %%mm7, %%mm3     \n\t"\

  130.         "paddw %%mm3, %%mm0         \n\t"\

  131.         "psllw $2, %%mm2            \n\t"\

  132.         "psubw %%mm1, %%mm2         \n\t"\

  133.         "pmullw %%mm4, %%mm2        \n\t"\

  134.         "paddw %%mm5, %%mm0         \n\t"\

  135.         "paddw %%mm2, %%mm0         \n\t"\

  136.         "movd   (%2), %%mm3         \n\t"\

  137.         "psraw $5, %%mm0            \n\t"\

  138.         "packuswb %%mm0, %%mm0      \n\t"\

  139.         PAVGB" %%mm3, %%mm0         \n\t"\

  140.         OP(%%mm0, (%1),%%mm6, d)\

  141.         "add %4, %0                 \n\t"\

  142.         "add %4, %1                 \n\t"\

  143.         "add %3, %2                 \n\t"\

  144.         : "+a"(src), "+c"(dst), "+d"(src2)\

  145.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  146.         : "memory"\

  147.     );\

  148.     }while(--h);\

  149. }\

  150. static av_noinline void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  151.     src -= 2*srcStride;\

  152.     __asm__ volatile(\

  153.         "pxor %%mm7, %%mm7          \n\t"\

  154.         "movd (%0), %%mm0           \n\t"\

  155.         "add %2, %0                 \n\t"\

  156.         "movd (%0), %%mm1           \n\t"\

  157.         "add %2, %0                 \n\t"\

  158.         "movd (%0), %%mm2           \n\t"\

  159.         "add %2, %0                 \n\t"\

  160.         "movd (%0), %%mm3           \n\t"\

  161.         "add %2, %0                 \n\t"\

  162.         "movd (%0), %%mm4           \n\t"\

  163.         "add %2, %0                 \n\t"\

  164.         "punpcklbw %%mm7, %%mm0     \n\t"\

  165.         "punpcklbw %%mm7, %%mm1     \n\t"\

  166.         "punpcklbw %%mm7, %%mm2     \n\t"\

  167.         "punpcklbw %%mm7, %%mm3     \n\t"\

  168.         "punpcklbw %%mm7, %%mm4     \n\t"\

  169.         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  170.         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  171.         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  172.         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  173.          \

  174.         : "+a"(src), "+c"(dst)\

  175.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  176.         : "memory"\

  177.     );\

  178. }\

  179. static av_noinline void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  180.     int h=4;\

  181.     int w=3;\

  182.     src -= 2*srcStride+2;\

  183.     while(w--){\

  184.         __asm__ volatile(\

  185.             "pxor %%mm7, %%mm7      \n\t"\

  186.             "movd (%0), %%mm0       \n\t"\

  187.             "add %2, %0             \n\t"\

  188.             "movd (%0), %%mm1       \n\t"\

  189.             "add %2, %0             \n\t"\

  190.             "movd (%0), %%mm2       \n\t"\

  191.             "add %2, %0             \n\t"\

  192.             "movd (%0), %%mm3       \n\t"\

  193.             "add %2, %0             \n\t"\

  194.             "movd (%0), %%mm4       \n\t"\

  195.             "add %2, %0             \n\t"\

  196.             "punpcklbw %%mm7, %%mm0 \n\t"\

  197.             "punpcklbw %%mm7, %%mm1 \n\t"\

  198.             "punpcklbw %%mm7, %%mm2 \n\t"\

  199.             "punpcklbw %%mm7, %%mm3 \n\t"\

  200.             "punpcklbw %%mm7, %%mm4 \n\t"\

  201.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\

  202.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\

  203.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\

  204.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\

  205.              \

  206.             : "+a"(src)\

  207.             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  208.             : "memory"\

  209.         );\

  210.         tmp += 4;\

  211.         src += 4 - 9*srcStride;\

  212.     }\

  213.     tmp -= 3*4;\

  214.     __asm__ volatile(\

  215.         "1:                         \n\t"\

  216.         "movq     (%0), %%mm0       \n\t"\

  217.         "paddw  10(%0), %%mm0       \n\t"\

  218.         "movq    2(%0), %%mm1       \n\t"\

  219.         "paddw   8(%0), %%mm1       \n\t"\

  220.         "movq    4(%0), %%mm2       \n\t"\

  221.         "paddw   6(%0), %%mm2       \n\t"\

  222.         "psubw %%mm1, %%mm0         \n\t"/*a-b   (abccba)*/\

  223.         "psraw $2, %%mm0            \n\t"/*(a-b)/4 */\

  224.         "psubw %%mm1, %%mm0         \n\t"/*(a-b)/4-b */\

  225.         "paddsw %%mm2, %%mm0        \n\t"\

  226.         "psraw $2, %%mm0            \n\t"/*((a-b)/4-b+c)/4 */\

  227.         "paddw %%mm2, %%mm0         \n\t"/*(a-5*b+20*c)/16 */\

  228.         "psraw $6, %%mm0            \n\t"\

  229.         "packuswb %%mm0, %%mm0      \n\t"\

  230.         OP(%%mm0, (%1),%%mm7, d)\

  231.         "add $24, %0                \n\t"\

  232.         "add %3, %1                 \n\t"\

  233.         "decl %2                    \n\t"\

  234.         " jnz 1b                    \n\t"\

  235.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  236.         : "S"((x86_reg)dstStride)\

  237.         : "memory"\

  238.     );\

  239. }\

  240. \

  241. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  242.     int h=8;\

  243.     __asm__ volatile(\

  244.         "pxor %%mm7, %%mm7          \n\t"\

  245.         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\

  246.         "1:                         \n\t"\

  247.         "movq    (%0), %%mm0        \n\t"\

  248.         "movq   1(%0), %%mm2        \n\t"\

  249.         "movq %%mm0, %%mm1          \n\t"\

  250.         "movq %%mm2, %%mm3          \n\t"\

  251.         "punpcklbw %%mm7, %%mm0     \n\t"\

  252.         "punpckhbw %%mm7, %%mm1     \n\t"\

  253.         "punpcklbw %%mm7, %%mm2     \n\t"\

  254.         "punpckhbw %%mm7, %%mm3     \n\t"\

  255.         "paddw %%mm2, %%mm0         \n\t"\

  256.         "paddw %%mm3, %%mm1         \n\t"\

  257.         "psllw $2, %%mm0            \n\t"\

  258.         "psllw $2, %%mm1            \n\t"\

  259.         "movq   -1(%0), %%mm2       \n\t"\

  260.         "movq    2(%0), %%mm4       \n\t"\

  261.         "movq %%mm2, %%mm3          \n\t"\

  262.         "movq %%mm4, %%mm5          \n\t"\

  263.         "punpcklbw %%mm7, %%mm2     \n\t"\

  264.         "punpckhbw %%mm7, %%mm3     \n\t"\

  265.         "punpcklbw %%mm7, %%mm4     \n\t"\

  266.         "punpckhbw %%mm7, %%mm5     \n\t"\

  267.         "paddw %%mm4, %%mm2         \n\t"\

  268.         "paddw %%mm3, %%mm5         \n\t"\

  269.         "psubw %%mm2, %%mm0         \n\t"\

  270.         "psubw %%mm5, %%mm1         \n\t"\

  271.         "pmullw %%mm6, %%mm0        \n\t"\

  272.         "pmullw %%mm6, %%mm1        \n\t"\

  273.         "movd   -2(%0), %%mm2       \n\t"\

  274.         "movd    7(%0), %%mm5       \n\t"\

  275.         "punpcklbw %%mm7, %%mm2     \n\t"\

  276.         "punpcklbw %%mm7, %%mm5     \n\t"\

  277.         "paddw %%mm3, %%mm2         \n\t"\

  278.         "paddw %%mm5, %%mm4         \n\t"\

  279.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  280.         "paddw %%mm5, %%mm2         \n\t"\

  281.         "paddw %%mm5, %%mm4         \n\t"\

  282.         "paddw %%mm2, %%mm0         \n\t"\

  283.         "paddw %%mm4, %%mm1         \n\t"\

  284.         "psraw $5, %%mm0            \n\t"\

  285.         "psraw $5, %%mm1            \n\t"\

  286.         "packuswb %%mm1, %%mm0      \n\t"\

  287.         OP(%%mm0, (%1),%%mm5, q)\

  288.         "add %3, %0                 \n\t"\

  289.         "add %4, %1                 \n\t"\

  290.         "decl %2                    \n\t"\

  291.         " jnz 1b                    \n\t"\

  292.         : "+a"(src), "+c"(dst), "+g"(h)\

  293.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  294.         : "memory"\

  295.     );\

  296. }\

  297. \

  298. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  299.     int h=8;\

  300.     __asm__ volatile(\

  301.         "pxor %%mm7, %%mm7          \n\t"\

  302.         "movq %0, %%mm6             \n\t"\

  303.         :: "m"(ff_pw_5)\

  304.     );\

  305.     do{\

  306.     __asm__ volatile(\

  307.         "movq    (%0), %%mm0        \n\t"\

  308.         "movq   1(%0), %%mm2        \n\t"\

  309.         "movq %%mm0, %%mm1          \n\t"\

  310.         "movq %%mm2, %%mm3          \n\t"\

  311.         "punpcklbw %%mm7, %%mm0     \n\t"\

  312.         "punpckhbw %%mm7, %%mm1     \n\t"\

  313.         "punpcklbw %%mm7, %%mm2     \n\t"\

  314.         "punpckhbw %%mm7, %%mm3     \n\t"\

  315.         "paddw %%mm2, %%mm0         \n\t"\

  316.         "paddw %%mm3, %%mm1         \n\t"\

  317.         "psllw $2, %%mm0            \n\t"\

  318.         "psllw $2, %%mm1            \n\t"\

  319.         "movq   -1(%0), %%mm2       \n\t"\

  320.         "movq    2(%0), %%mm4       \n\t"\

  321.         "movq %%mm2, %%mm3          \n\t"\

  322.         "movq %%mm4, %%mm5          \n\t"\

  323.         "punpcklbw %%mm7, %%mm2     \n\t"\

  324.         "punpckhbw %%mm7, %%mm3     \n\t"\

  325.         "punpcklbw %%mm7, %%mm4     \n\t"\

  326.         "punpckhbw %%mm7, %%mm5     \n\t"\

  327.         "paddw %%mm4, %%mm2         \n\t"\

  328.         "paddw %%mm3, %%mm5         \n\t"\

  329.         "psubw %%mm2, %%mm0         \n\t"\

  330.         "psubw %%mm5, %%mm1         \n\t"\

  331.         "pmullw %%mm6, %%mm0        \n\t"\

  332.         "pmullw %%mm6, %%mm1        \n\t"\

  333.         "movd   -2(%0), %%mm2       \n\t"\

  334.         "movd    7(%0), %%mm5       \n\t"\

  335.         "punpcklbw %%mm7, %%mm2     \n\t"\

  336.         "punpcklbw %%mm7, %%mm5     \n\t"\

  337.         "paddw %%mm3, %%mm2         \n\t"\

  338.         "paddw %%mm5, %%mm4         \n\t"\

  339.         "movq %5, %%mm5             \n\t"\

  340.         "paddw %%mm5, %%mm2         \n\t"\

  341.         "paddw %%mm5, %%mm4         \n\t"\

  342.         "paddw %%mm2, %%mm0         \n\t"\

  343.         "paddw %%mm4, %%mm1         \n\t"\

  344.         "psraw $5, %%mm0            \n\t"\

  345.         "psraw $5, %%mm1            \n\t"\

  346.         "movq (%2), %%mm4           \n\t"\

  347.         "packuswb %%mm1, %%mm0      \n\t"\

  348.         PAVGB" %%mm4, %%mm0         \n\t"\

  349.         OP(%%mm0, (%1),%%mm5, q)\

  350.         "add %4, %0                 \n\t"\

  351.         "add %4, %1                 \n\t"\

  352.         "add %3, %2                 \n\t"\

  353.         : "+a"(src), "+c"(dst), "+d"(src2)\

  354.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\

  355.           "m"(ff_pw_16)\

  356.         : "memory"\

  357.     );\

  358.     }while(--h);\

  359. }\

  360. \

  361. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  362.     int w= 2;\

  363.     src -= 2*srcStride;\

  364.     \

  365.     while(w--){\

  366.       __asm__ volatile(\

  367.         "pxor %%mm7, %%mm7          \n\t"\

  368.         "movd (%0), %%mm0           \n\t"\

  369.         "add %2, %0                 \n\t"\

  370.         "movd (%0), %%mm1           \n\t"\

  371.         "add %2, %0                 \n\t"\

  372.         "movd (%0), %%mm2           \n\t"\

  373.         "add %2, %0                 \n\t"\

  374.         "movd (%0), %%mm3           \n\t"\

  375.         "add %2, %0                 \n\t"\

  376.         "movd (%0), %%mm4           \n\t"\

  377.         "add %2, %0                 \n\t"\

  378.         "punpcklbw %%mm7, %%mm0     \n\t"\

  379.         "punpcklbw %%mm7, %%mm1     \n\t"\

  380.         "punpcklbw %%mm7, %%mm2     \n\t"\

  381.         "punpcklbw %%mm7, %%mm3     \n\t"\

  382.         "punpcklbw %%mm7, %%mm4     \n\t"\

  383.         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  384.         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  385.         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  386.         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  387.         QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  388.         QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  389.         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  390.         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  391.          \

  392.         : "+a"(src), "+c"(dst)\

  393.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  394.         : "memory"\

  395.      );\

  396.      if(h==16){\

  397.         __asm__ volatile(\

  398.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  399.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  400.             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  401.             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  402.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  403.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  404.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  405.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  406.             \

  407.            : "+a"(src), "+c"(dst)\

  408.            : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  409.            : "memory"\

  410.         );\

  411.      }\

  412.      src += 4-(h+5)*srcStride;\

  413.      dst += 4-h*dstStride;\

  414.    }\

  415. }\

  416. static av_always_inline void OPNAME ## h264_qpel8or16_hv1_lowpass_ ## MMX(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){\

  417.     int w = (size+8)>>2;\

  418.     src -= 2*srcStride+2;\

  419.     while(w--){\

  420.         __asm__ volatile(\

  421.             "pxor %%mm7, %%mm7      \n\t"\

  422.             "movd (%0), %%mm0       \n\t"\

  423.             "add %2, %0             \n\t"\

  424.             "movd (%0), %%mm1       \n\t"\

  425.             "add %2, %0             \n\t"\

  426.             "movd (%0), %%mm2       \n\t"\

  427.             "add %2, %0             \n\t"\

  428.             "movd (%0), %%mm3       \n\t"\

  429.             "add %2, %0             \n\t"\

  430.             "movd (%0), %%mm4       \n\t"\

  431.             "add %2, %0             \n\t"\

  432.             "punpcklbw %%mm7, %%mm0 \n\t"\

  433.             "punpcklbw %%mm7, %%mm1 \n\t"\

  434.             "punpcklbw %%mm7, %%mm2 \n\t"\

  435.             "punpcklbw %%mm7, %%mm3 \n\t"\

  436.             "punpcklbw %%mm7, %%mm4 \n\t"\

  437.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\

  438.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\

  439.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\

  440.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\

  441.             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\

  442.             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\

  443.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\

  444.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\

  445.             : "+a"(src)\

  446.             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  447.             : "memory"\

  448.         );\

  449.         if(size==16){\

  450.             __asm__ volatile(\

  451.                 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1,  8*48)\

  452.                 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2,  9*48)\

  453.                 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\

  454.                 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\

  455.                 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\

  456.                 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\

  457.                 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\

  458.                 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\

  459.                 : "+a"(src)\

  460.                 : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  461.                 : "memory"\

  462.             );\

  463.         }\

  464.         tmp += 4;\

  465.         src += 4 - (size+5)*srcStride;\

  466.     }\

  467. }\

  468. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  469.     int w = size>>4;\

  470.     do{\

  471.     int h = size;\

  472.     __asm__ volatile(\

  473.         "1:                         \n\t"\

  474.         "movq     (%0), %%mm0       \n\t"\

  475.         "movq    8(%0), %%mm3       \n\t"\

  476.         "movq    2(%0), %%mm1       \n\t"\

  477.         "movq   10(%0), %%mm4       \n\t"\

  478.         "paddw   %%mm4, %%mm0       \n\t"\

  479.         "paddw   %%mm3, %%mm1       \n\t"\

  480.         "paddw  18(%0), %%mm3       \n\t"\

  481.         "paddw  16(%0), %%mm4       \n\t"\

  482.         "movq    4(%0), %%mm2       \n\t"\

  483.         "movq   12(%0), %%mm5       \n\t"\

  484.         "paddw   6(%0), %%mm2       \n\t"\

  485.         "paddw  14(%0), %%mm5       \n\t"\

  486.         "psubw %%mm1, %%mm0         \n\t"\

  487.         "psubw %%mm4, %%mm3         \n\t"\

  488.         "psraw $2, %%mm0            \n\t"\

  489.         "psraw $2, %%mm3            \n\t"\

  490.         "psubw %%mm1, %%mm0         \n\t"\

  491.         "psubw %%mm4, %%mm3         \n\t"\

  492.         "paddsw %%mm2, %%mm0        \n\t"\

  493.         "paddsw %%mm5, %%mm3        \n\t"\

  494.         "psraw $2, %%mm0            \n\t"\

  495.         "psraw $2, %%mm3            \n\t"\

  496.         "paddw %%mm2, %%mm0         \n\t"\

  497.         "paddw %%mm5, %%mm3         \n\t"\

  498.         "psraw $6, %%mm0            \n\t"\

  499.         "psraw $6, %%mm3            \n\t"\

  500.         "packuswb %%mm3, %%mm0      \n\t"\

  501.         OP(%%mm0, (%1),%%mm7, q)\

  502.         "add $48, %0                \n\t"\

  503.         "add %3, %1                 \n\t"\

  504.         "decl %2                    \n\t"\

  505.         " jnz 1b                    \n\t"\

  506.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  507.         : "S"((x86_reg)dstStride)\

  508.         : "memory"\

  509.     );\

  510.     tmp += 8 - size*24;\

  511.     dst += 8 - size*dstStride;\

  512.     }while(w--);\

  513. }\

  514. \

  515. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  516.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  517. }\

  518. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  519.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  520.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  521. }\

  522. \

  523. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  524.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  525.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  526.     src += 8*srcStride;\

  527.     dst += 8*dstStride;\

  528.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  529.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  530. }\

  531. \

  532. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  533.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  534.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  535.     src += 8*dstStride;\

  536.     dst += 8*dstStride;\

  537.     src2 += 8*src2Stride;\

  538.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  539.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  540. }\

  541. \

  542. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  543.           put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\

  544.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  545. }\

  546. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  547.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 8);\

  548. }\

  549. \

  550. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  551.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 16);\

  552. }\

  553. \

  554. static av_noinline void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  555. {\

  556.     __asm__ volatile(\

  557.         "movq      (%1), %%mm0          \n\t"\

  558.         "movq    24(%1), %%mm1          \n\t"\

  559.         "psraw      $5,  %%mm0          \n\t"\

  560.         "psraw      $5,  %%mm1          \n\t"\

  561.         "packuswb %%mm0, %%mm0          \n\t"\

  562.         "packuswb %%mm1, %%mm1          \n\t"\

  563.         PAVGB"     (%0), %%mm0          \n\t"\

  564.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  565.         OP(%%mm0, (%2),    %%mm4, d)\

  566.         OP(%%mm1, (%2,%4), %%mm5, d)\

  567.         "lea  (%0,%3,2), %0             \n\t"\

  568.         "lea  (%2,%4,2), %2             \n\t"\

  569.         "movq    48(%1), %%mm0          \n\t"\

  570.         "movq    72(%1), %%mm1          \n\t"\

  571.         "psraw      $5,  %%mm0          \n\t"\

  572.         "psraw      $5,  %%mm1          \n\t"\

  573.         "packuswb %%mm0, %%mm0          \n\t"\

  574.         "packuswb %%mm1, %%mm1          \n\t"\

  575.         PAVGB"     (%0), %%mm0          \n\t"\

  576.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  577.         OP(%%mm0, (%2),    %%mm4, d)\

  578.         OP(%%mm1, (%2,%4), %%mm5, d)\

  579.         :"+a"(src8), "+c"(src16), "+d"(dst)\

  580.         :"S"((x86_reg)src8Stride), "D"((x86_reg)dstStride)\

  581.         :"memory");\

  582. }\

  583. static av_noinline void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  584. {\

  585.     do{\

  586.     __asm__ volatile(\

  587.         "movq      (%1), %%mm0          \n\t"\

  588.         "movq     8(%1), %%mm1          \n\t"\

  589.         "movq    48(%1), %%mm2          \n\t"\

  590.         "movq  8+48(%1), %%mm3          \n\t"\

  591.         "psraw      $5,  %%mm0          \n\t"\

  592.         "psraw      $5,  %%mm1          \n\t"\

  593.         "psraw      $5,  %%mm2          \n\t"\

  594.         "psraw      $5,  %%mm3          \n\t"\

  595.         "packuswb %%mm1, %%mm0          \n\t"\

  596.         "packuswb %%mm3, %%mm2          \n\t"\

  597.         PAVGB"     (%0), %%mm0          \n\t"\

  598.         PAVGB"  (%0,%3), %%mm2          \n\t"\

  599.         OP(%%mm0, (%2), %%mm5, q)\

  600.         OP(%%mm2, (%2,%4), %%mm5, q)\

  601.         ::"a"(src8), "c"(src16), "d"(dst),\

  602.           "r"((x86_reg)src8Stride), "r"((x86_reg)dstStride)\

  603.         :"memory");\

  604.         src8 += 2L*src8Stride;\

  605.         src16 += 48;\

  606.         dst += 2L*dstStride;\

  607.     }while(h-=2);\

  608. }\

  609. static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  610. {\

  611.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst  , src16  , src8  , dstStride, src8Stride, h);\

  612.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\

  613. }\

  614. 

  615. 

  616. #if ARCH_X86_64

  617. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  618. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  619.     int h=16;\

  620.     __asm__ volatile(\

  621.         "pxor %%xmm15, %%xmm15      \n\t"\

  622.         "movdqa %6, %%xmm14         \n\t"\

  623.         "movdqa %7, %%xmm13         \n\t"\

  624.         "1:                         \n\t"\

  625.         "lddqu    6(%0), %%xmm1     \n\t"\

  626.         "lddqu   -2(%0), %%xmm7     \n\t"\

  627.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  628.         "punpckhbw %%xmm15, %%xmm1  \n\t"\

  629.         "punpcklbw %%xmm15, %%xmm0  \n\t"\

  630.         "punpcklbw %%xmm15, %%xmm7  \n\t"\

  631.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  632.         "movdqa  %%xmm0, %%xmm6     \n\t"\

  633.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  634.         "movdqa  %%xmm0, %%xmm8     \n\t"\

  635.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  636.         "movdqa  %%xmm0, %%xmm9     \n\t"\

  637.         "movdqa  %%xmm0, %%xmm12    \n\t"\

  638.         "movdqa  %%xmm1, %%xmm11    \n\t"\

  639.         "palignr $10,%%xmm0, %%xmm11\n\t"\

  640.         "palignr $10,%%xmm7, %%xmm12\n\t"\

  641.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  642.         "palignr $2, %%xmm7, %%xmm9 \n\t"\

  643.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  644.         "palignr $4, %%xmm7, %%xmm8 \n\t"\

  645.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  646.         "palignr $6, %%xmm7, %%xmm6 \n\t"\

  647.         "paddw   %%xmm0 ,%%xmm11    \n\t"\

  648.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  649.         "palignr $8, %%xmm7, %%xmm0 \n\t"\

  650.         "paddw   %%xmm12,%%xmm7     \n\t"\

  651.         "paddw   %%xmm3, %%xmm2     \n\t"\

  652.         "paddw   %%xmm8, %%xmm6     \n\t"\

  653.         "paddw   %%xmm4, %%xmm1     \n\t"\

  654.         "paddw   %%xmm9, %%xmm0     \n\t"\

  655.         "psllw   $2,     %%xmm2     \n\t"\

  656.         "psllw   $2,     %%xmm6     \n\t"\

  657.         "psubw   %%xmm1, %%xmm2     \n\t"\

  658.         "psubw   %%xmm0, %%xmm6     \n\t"\

  659.         "paddw   %%xmm13,%%xmm11    \n\t"\

  660.         "paddw   %%xmm13,%%xmm7     \n\t"\

  661.         "pmullw  %%xmm14,%%xmm2     \n\t"\

  662.         "pmullw  %%xmm14,%%xmm6     \n\t"\

  663.         "lddqu   (%2),   %%xmm3     \n\t"\

  664.         "paddw   %%xmm11,%%xmm2     \n\t"\

  665.         "paddw   %%xmm7, %%xmm6     \n\t"\

  666.         "psraw   $5,     %%xmm2     \n\t"\

  667.         "psraw   $5,     %%xmm6     \n\t"\

  668.         "packuswb %%xmm2,%%xmm6     \n\t"\

  669.         "pavgb   %%xmm3, %%xmm6     \n\t"\

  670.         OP(%%xmm6, (%1), %%xmm4, dqa)\

  671.         "add %5, %0                 \n\t"\

  672.         "add %5, %1                 \n\t"\

  673.         "add %4, %2                 \n\t"\

  674.         "decl %3                    \n\t"\

  675.         "jg 1b                      \n\t"\

  676.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  677.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\

  678.           "m"(ff_pw_5), "m"(ff_pw_16)\

  679.         : "memory"\

  680.     );\

  681. }

  682. #else // ARCH_X86_64

  683. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  684. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  685.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  686.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  687.     src += 8*dstStride;\

  688.     dst += 8*dstStride;\

  689.     src2 += 8*src2Stride;\

  690.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  691.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  692. }

  693. #endif // ARCH_X86_64

  694. 

  695. #define QPEL_H264_H_XMM(OPNAME, OP, MMX)\

  696. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  697.     int h=8;\

  698.     __asm__ volatile(\

  699.         "pxor %%xmm7, %%xmm7        \n\t"\

  700.         "movdqa %0, %%xmm6          \n\t"\

  701.         :: "m"(ff_pw_5)\

  702.     );\

  703.     do{\

  704.     __asm__ volatile(\

  705.         "lddqu   -2(%0), %%xmm1     \n\t"\

  706.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  707.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  708.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  709.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  710.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  711.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  712.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  713.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  714.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  715.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  716.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  717.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  718.         "paddw   %%xmm5, %%xmm0     \n\t"\

  719.         "paddw   %%xmm3, %%xmm2     \n\t"\

  720.         "paddw   %%xmm4, %%xmm1     \n\t"\

  721.         "psllw   $2,     %%xmm2     \n\t"\

  722.         "movq    (%2),   %%xmm3     \n\t"\

  723.         "psubw   %%xmm1, %%xmm2     \n\t"\

  724.         "paddw   %5,     %%xmm0     \n\t"\

  725.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  726.         "paddw   %%xmm0, %%xmm2     \n\t"\

  727.         "psraw   $5,     %%xmm2     \n\t"\

  728.         "packuswb %%xmm2, %%xmm2    \n\t"\

  729.         "pavgb   %%xmm3, %%xmm2     \n\t"\

  730.         OP(%%xmm2, (%1), %%xmm4, q)\

  731.         "add %4, %0                 \n\t"\

  732.         "add %4, %1                 \n\t"\

  733.         "add %3, %2                 \n\t"\

  734.         : "+a"(src), "+c"(dst), "+d"(src2)\

  735.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\

  736.           "m"(ff_pw_16)\

  737.         : "memory"\

  738.     );\

  739.     }while(--h);\

  740. }\

  741. QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  742. \

  743. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  744.     int h=8;\

  745.     __asm__ volatile(\

  746.         "pxor %%xmm7, %%xmm7        \n\t"\

  747.         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\

  748.         "1:                         \n\t"\

  749.         "lddqu   -2(%0), %%xmm1     \n\t"\

  750.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  751.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  752.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  753.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  754.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  755.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  756.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  757.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  758.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  759.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  760.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  761.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  762.         "paddw   %%xmm5, %%xmm0     \n\t"\

  763.         "paddw   %%xmm3, %%xmm2     \n\t"\

  764.         "paddw   %%xmm4, %%xmm1     \n\t"\

  765.         "psllw   $2,     %%xmm2     \n\t"\

  766.         "psubw   %%xmm1, %%xmm2     \n\t"\

  767.         "paddw   "MANGLE(ff_pw_16)", %%xmm0\n\t"\

  768.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  769.         "paddw   %%xmm0, %%xmm2     \n\t"\

  770.         "psraw   $5,     %%xmm2     \n\t"\

  771.         "packuswb %%xmm2, %%xmm2    \n\t"\

  772.         OP(%%xmm2, (%1), %%xmm4, q)\

  773.         "add %3, %0                 \n\t"\

  774.         "add %4, %1                 \n\t"\

  775.         "decl %2                    \n\t"\

  776.         " jnz 1b                    \n\t"\

  777.         : "+a"(src), "+c"(dst), "+g"(h)\

  778.         : "D"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  779.         : "memory"\

  780.     );\

  781. }\

  782. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  783.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  784.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  785.     src += 8*srcStride;\

  786.     dst += 8*dstStride;\

  787.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  788.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  789. }\

  790. 

  791. #define QPEL_H264_V_XMM(OPNAME, OP, MMX)\

  792. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  793.     src -= 2*srcStride;\

  794.     \

  795.     __asm__ volatile(\

  796.         "pxor %%xmm7, %%xmm7        \n\t"\

  797.         "movq (%0), %%xmm0          \n\t"\

  798.         "add %2, %0                 \n\t"\

  799.         "movq (%0), %%xmm1          \n\t"\

  800.         "add %2, %0                 \n\t"\

  801.         "movq (%0), %%xmm2          \n\t"\

  802.         "add %2, %0                 \n\t"\

  803.         "movq (%0), %%xmm3          \n\t"\

  804.         "add %2, %0                 \n\t"\

  805.         "movq (%0), %%xmm4          \n\t"\

  806.         "add %2, %0                 \n\t"\

  807.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  808.         "punpcklbw %%xmm7, %%xmm1   \n\t"\

  809.         "punpcklbw %%xmm7, %%xmm2   \n\t"\

  810.         "punpcklbw %%xmm7, %%xmm3   \n\t"\

  811.         "punpcklbw %%xmm7, %%xmm4   \n\t"\

  812.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  813.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  814.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  815.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  816.         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  817.         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  818.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  819.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  820.          \

  821.         : "+a"(src), "+c"(dst)\

  822.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  823.         : "memory"\

  824.     );\

  825.     if(h==16){\

  826.         __asm__ volatile(\

  827.             QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  828.             QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  829.             QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  830.             QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  831.             QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  832.             QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  833.             QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  834.             QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  835.             \

  836.             : "+a"(src), "+c"(dst)\

  837.             : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  838.             : "memory"\

  839.         );\

  840.     }\

  841. }\

  842. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  843.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  844. }\

  845. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  846.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  847.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  848. }

  849. 

  850. static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){

  851.     int w = (size+8)>>3;

  852.     src -= 2*srcStride+2;

  853.     while(w--){

  854.         __asm__ volatile(

  855.             "pxor %%xmm7, %%xmm7        \n\t"

  856.             "movq (%0), %%xmm0          \n\t"

  857.             "add %2, %0                 \n\t"

  858.             "movq (%0), %%xmm1          \n\t"

  859.             "add %2, %0                 \n\t"

  860.             "movq (%0), %%xmm2          \n\t"

  861.             "add %2, %0                 \n\t"

  862.             "movq (%0), %%xmm3          \n\t"

  863.             "add %2, %0                 \n\t"

  864.             "movq (%0), %%xmm4          \n\t"

  865.             "add %2, %0                 \n\t"

  866.             "punpcklbw %%xmm7, %%xmm0   \n\t"

  867.             "punpcklbw %%xmm7, %%xmm1   \n\t"

  868.             "punpcklbw %%xmm7, %%xmm2   \n\t"

  869.             "punpcklbw %%xmm7, %%xmm3   \n\t"

  870.             "punpcklbw %%xmm7, %%xmm4   \n\t"

  871.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 0*48)

  872.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 1*48)

  873.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 2*48)

  874.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 3*48)

  875.             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 4*48)

  876.             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 5*48)

  877.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 6*48)

  878.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 7*48)

  879.             : "+a"(src)

  880.             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)

  881.             : "memory"

  882.         );

  883.         if(size==16){

  884.             __asm__ volatile(

  885.                 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1,  8*48)

  886.                 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2,  9*48)

  887.                 QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 10*48)

  888.                 QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 11*48)

  889.                 QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 12*48)

  890.                 QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 13*48)

  891.                 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 14*48)

  892.                 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 15*48)

  893.                 : "+a"(src)

  894.                 : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)

  895.                 : "memory"

  896.             );

  897.         }

  898.         tmp += 8;

  899.         src += 8 - (size+5)*srcStride;

  900.     }

  901. }

  902. 

  903. #define QPEL_H264_HV2_XMM(OPNAME, OP, MMX)\

  904. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  905.     int h = size;\

  906.     if(size == 16){\

  907.         __asm__ volatile(\

  908.             "1:                         \n\t"\

  909.             "movdqa 32(%0), %%xmm4      \n\t"\

  910.             "movdqa 16(%0), %%xmm5      \n\t"\

  911.             "movdqa   (%0), %%xmm7      \n\t"\

  912.             "movdqa %%xmm4, %%xmm3      \n\t"\

  913.             "movdqa %%xmm4, %%xmm2      \n\t"\

  914.             "movdqa %%xmm4, %%xmm1      \n\t"\

  915.             "movdqa %%xmm4, %%xmm0      \n\t"\

  916.             "palignr $10, %%xmm5, %%xmm0 \n\t"\

  917.             "palignr  $8, %%xmm5, %%xmm1 \n\t"\

  918.             "palignr  $6, %%xmm5, %%xmm2 \n\t"\

  919.             "palignr  $4, %%xmm5, %%xmm3 \n\t"\

  920.             "palignr  $2, %%xmm5, %%xmm4 \n\t"\

  921.             "paddw  %%xmm5, %%xmm0      \n\t"\

  922.             "paddw  %%xmm4, %%xmm1      \n\t"\

  923.             "paddw  %%xmm3, %%xmm2      \n\t"\

  924.             "movdqa %%xmm5, %%xmm6      \n\t"\

  925.             "movdqa %%xmm5, %%xmm4      \n\t"\

  926.             "movdqa %%xmm5, %%xmm3      \n\t"\

  927.             "palignr  $8, %%xmm7, %%xmm4 \n\t"\

  928.             "palignr  $2, %%xmm7, %%xmm6 \n\t"\

  929.             "palignr $10, %%xmm7, %%xmm3 \n\t"\

  930.             "paddw  %%xmm6, %%xmm4      \n\t"\

  931.             "movdqa %%xmm5, %%xmm6      \n\t"\

  932.             "palignr  $6, %%xmm7, %%xmm5 \n\t"\

  933.             "palignr  $4, %%xmm7, %%xmm6 \n\t"\

  934.             "paddw  %%xmm7, %%xmm3      \n\t"\

  935.             "paddw  %%xmm6, %%xmm5      \n\t"\

  936.             \

  937.             "psubw  %%xmm1, %%xmm0      \n\t"\

  938.             "psubw  %%xmm4, %%xmm3      \n\t"\

  939.             "psraw      $2, %%xmm0      \n\t"\

  940.             "psraw      $2, %%xmm3      \n\t"\

  941.             "psubw  %%xmm1, %%xmm0      \n\t"\

  942.             "psubw  %%xmm4, %%xmm3      \n\t"\

  943.             "paddw  %%xmm2, %%xmm0      \n\t"\

  944.             "paddw  %%xmm5, %%xmm3      \n\t"\

  945.             "psraw      $2, %%xmm0      \n\t"\

  946.             "psraw      $2, %%xmm3      \n\t"\

  947.             "paddw  %%xmm2, %%xmm0      \n\t"\

  948.             "paddw  %%xmm5, %%xmm3      \n\t"\

  949.             "psraw      $6, %%xmm0      \n\t"\

  950.             "psraw      $6, %%xmm3      \n\t"\

  951.             "packuswb %%xmm0, %%xmm3    \n\t"\

  952.             OP(%%xmm3, (%1), %%xmm7, dqa)\

  953.             "add $48, %0                \n\t"\

  954.             "add %3, %1                 \n\t"\

  955.             "decl %2                    \n\t"\

  956.             " jnz 1b                    \n\t"\

  957.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  958.             : "S"((x86_reg)dstStride)\

  959.             : "memory"\

  960.         );\

  961.     }else{\

  962.         __asm__ volatile(\

  963.             "1:                         \n\t"\

  964.             "movdqa 16(%0), %%xmm1      \n\t"\

  965.             "movdqa   (%0), %%xmm0      \n\t"\

  966.             "movdqa %%xmm1, %%xmm2      \n\t"\

  967.             "movdqa %%xmm1, %%xmm3      \n\t"\

  968.             "movdqa %%xmm1, %%xmm4      \n\t"\

  969.             "movdqa %%xmm1, %%xmm5      \n\t"\

  970.             "palignr $10, %%xmm0, %%xmm5 \n\t"\

  971.             "palignr  $8, %%xmm0, %%xmm4 \n\t"\

  972.             "palignr  $6, %%xmm0, %%xmm3 \n\t"\

  973.             "palignr  $4, %%xmm0, %%xmm2 \n\t"\

  974.             "palignr  $2, %%xmm0, %%xmm1 \n\t"\

  975.             "paddw  %%xmm5, %%xmm0      \n\t"\

  976.             "paddw  %%xmm4, %%xmm1      \n\t"\

  977.             "paddw  %%xmm3, %%xmm2      \n\t"\

  978.             "psubw  %%xmm1, %%xmm0      \n\t"\

  979.             "psraw      $2, %%xmm0      \n\t"\

  980.             "psubw  %%xmm1, %%xmm0      \n\t"\

  981.             "paddw  %%xmm2, %%xmm0      \n\t"\

  982.             "psraw      $2, %%xmm0      \n\t"\

  983.             "paddw  %%xmm2, %%xmm0      \n\t"\

  984.             "psraw      $6, %%xmm0      \n\t"\

  985.             "packuswb %%xmm0, %%xmm0    \n\t"\

  986.             OP(%%xmm0, (%1), %%xmm7, q)\

  987.             "add $48, %0                \n\t"\

  988.             "add %3, %1                 \n\t"\

  989.             "decl %2                    \n\t"\

  990.             " jnz 1b                    \n\t"\

  991.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  992.             : "S"((x86_reg)dstStride)\

  993.             : "memory"\

  994.         );\

  995.     }\

  996. }

  997. 

  998. #define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\

  999. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  1000.           put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\

  1001.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  1002. }\

  1003. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  1004.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\

  1005. }\

  1006. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  1007.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\

  1008. }\

  1009. 

  1010. #define put_pixels8_l2_sse2 put_pixels8_l2_mmx2

  1011. #define avg_pixels8_l2_sse2 avg_pixels8_l2_mmx2

  1012. #define put_pixels16_l2_sse2 put_pixels16_l2_mmx2

  1013. #define avg_pixels16_l2_sse2 avg_pixels16_l2_mmx2

  1014. #define put_pixels8_l2_ssse3 put_pixels8_l2_mmx2

  1015. #define avg_pixels8_l2_ssse3 avg_pixels8_l2_mmx2

  1016. #define put_pixels16_l2_ssse3 put_pixels16_l2_mmx2

  1017. #define avg_pixels16_l2_ssse3 avg_pixels16_l2_mmx2

  1018. 

  1019. #define put_pixels8_l2_shift5_sse2 put_pixels8_l2_shift5_mmx2

  1020. #define avg_pixels8_l2_shift5_sse2 avg_pixels8_l2_shift5_mmx2

  1021. #define put_pixels16_l2_shift5_sse2 put_pixels16_l2_shift5_mmx2

  1022. #define avg_pixels16_l2_shift5_sse2 avg_pixels16_l2_shift5_mmx2

  1023. #define put_pixels8_l2_shift5_ssse3 put_pixels8_l2_shift5_mmx2

  1024. #define avg_pixels8_l2_shift5_ssse3 avg_pixels8_l2_shift5_mmx2

  1025. #define put_pixels16_l2_shift5_ssse3 put_pixels16_l2_shift5_mmx2

  1026. #define avg_pixels16_l2_shift5_ssse3 avg_pixels16_l2_shift5_mmx2

  1027. 

  1028. #define put_h264_qpel8_h_lowpass_l2_sse2 put_h264_qpel8_h_lowpass_l2_mmx2

  1029. #define avg_h264_qpel8_h_lowpass_l2_sse2 avg_h264_qpel8_h_lowpass_l2_mmx2

  1030. #define put_h264_qpel16_h_lowpass_l2_sse2 put_h264_qpel16_h_lowpass_l2_mmx2

  1031. #define avg_h264_qpel16_h_lowpass_l2_sse2 avg_h264_qpel16_h_lowpass_l2_mmx2

  1032. 

  1033. #define put_h264_qpel8_v_lowpass_ssse3 put_h264_qpel8_v_lowpass_sse2

  1034. #define avg_h264_qpel8_v_lowpass_ssse3 avg_h264_qpel8_v_lowpass_sse2

  1035. #define put_h264_qpel16_v_lowpass_ssse3 put_h264_qpel16_v_lowpass_sse2

  1036. #define avg_h264_qpel16_v_lowpass_ssse3 avg_h264_qpel16_v_lowpass_sse2

  1037. 

  1038. #define put_h264_qpel8or16_hv2_lowpass_sse2 put_h264_qpel8or16_hv2_lowpass_mmx2

  1039. #define avg_h264_qpel8or16_hv2_lowpass_sse2 avg_h264_qpel8or16_hv2_lowpass_mmx2

  1040. 

  1041. #define H264_MC(OPNAME, SIZE, MMX, ALIGN) \

  1042. H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\

  1043. H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\

  1044. H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\

  1045. H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\

  1046. 

  1047. static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1048.     put_pixels16_sse2(dst, src, stride, 16);

  1049. }

  1050. static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1051.     avg_pixels16_sse2(dst, src, stride, 16);

  1052. }

  1053. #define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmx2

  1054. #define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmx2

  1055. 

  1056. #define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \

  1057. static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\

  1058.     OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\

  1059. }\

  1060. 

  1061. #define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \

  1062. static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1063.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\

  1064. }\

  1065. \

  1066. static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1067.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\

  1068. }\

  1069. \

  1070. static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1071.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\

  1072. }\

  1073. 

  1074. #define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \

  1075. static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1076.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1077.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1078.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\

  1079. }\

  1080. \

  1081. static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1082.     OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\

  1083. }\

  1084. \

  1085. static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1086.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1087.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1088.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\

  1089. }\

  1090. 

  1091. #define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \

  1092. static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1093.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1094.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1095.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1096. }\

  1097. \

  1098. static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1099.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1100.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1101.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1102. }\

  1103. \

  1104. static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1105.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1106.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1107.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1108. }\

  1109. \

  1110. static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1111.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1112.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1113.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1114. }\

  1115. \

  1116. static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1117.     DECLARE_ALIGNED(ALIGN, uint16_t, temp)[SIZE*(SIZE<8?12:24)];\

  1118.     OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\

  1119. }\

  1120. \

  1121. static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1122.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1123.     uint8_t * const halfHV= temp;\

  1124.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1125.     assert(((int)temp & 7) == 0);\

  1126.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1127.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\

  1128. }\

  1129. \

  1130. static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1131.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1132.     uint8_t * const halfHV= temp;\

  1133.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1134.     assert(((int)temp & 7) == 0);\

  1135.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1136.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\

  1137. }\

  1138. \

  1139. static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1140.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1141.     uint8_t * const halfHV= temp;\

  1142.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1143.     assert(((int)temp & 7) == 0);\

  1144.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1145.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\

  1146. }\

  1147. \

  1148. static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1149.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1150.     uint8_t * const halfHV= temp;\

  1151.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1152.     assert(((int)temp & 7) == 0);\

  1153.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1154.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\

  1155. }\

  1156. 

  1157. #define H264_MC_4816(MMX)\

  1158. H264_MC(put_, 4, MMX, 8)\

  1159. H264_MC(put_, 8, MMX, 8)\

  1160. H264_MC(put_, 16,MMX, 8)\

  1161. H264_MC(avg_, 4, MMX, 8)\

  1162. H264_MC(avg_, 8, MMX, 8)\

  1163. H264_MC(avg_, 16,MMX, 8)\

  1164. 

  1165. #define H264_MC_816(QPEL, XMM)\

  1166. QPEL(put_, 8, XMM, 16)\

  1167. QPEL(put_, 16,XMM, 16)\

  1168. QPEL(avg_, 8, XMM, 16)\

  1169. QPEL(avg_, 16,XMM, 16)\

  1170. 

  1171. 

  1172. #define AVG_3DNOW_OP(a,b,temp, size) \

  1173. "mov" #size " " #b ", " #temp "   \n\t"\

  1174. "pavgusb " #temp ", " #a "        \n\t"\

  1175. "mov" #size " " #a ", " #b "      \n\t"

  1176. #define AVG_MMX2_OP(a,b,temp, size) \

  1177. "mov" #size " " #b ", " #temp "   \n\t"\

  1178. "pavgb " #temp ", " #a "          \n\t"\

  1179. "mov" #size " " #a ", " #b "      \n\t"

  1180. 

  1181. #define PAVGB "pavgusb"

  1182. QPEL_H264(put_,       PUT_OP, 3dnow)

  1183. QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow)

  1184. #undef PAVGB

  1185. #define PAVGB "pavgb"

  1186. QPEL_H264(put_,       PUT_OP, mmx2)

  1187. QPEL_H264(avg_,  AVG_MMX2_OP, mmx2)

  1188. QPEL_H264_V_XMM(put_,       PUT_OP, sse2)

  1189. QPEL_H264_V_XMM(avg_,  AVG_MMX2_OP, sse2)

  1190. QPEL_H264_HV_XMM(put_,       PUT_OP, sse2)

  1191. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, sse2)

  1192. #if HAVE_SSSE3

  1193. QPEL_H264_H_XMM(put_,       PUT_OP, ssse3)

  1194. QPEL_H264_H_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1195. QPEL_H264_HV2_XMM(put_,       PUT_OP, ssse3)

  1196. QPEL_H264_HV2_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1197. QPEL_H264_HV_XMM(put_,       PUT_OP, ssse3)

  1198. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1199. #endif

  1200. #undef PAVGB

  1201. 

  1202. H264_MC_4816(3dnow)

  1203. H264_MC_4816(mmx2)

  1204. H264_MC_816(H264_MC_V, sse2)

  1205. H264_MC_816(H264_MC_HV, sse2)

  1206. #if HAVE_SSSE3

  1207. H264_MC_816(H264_MC_H, ssse3)

  1208. H264_MC_816(H264_MC_HV, ssse3)

  1209. #endif