falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34093 Commits
32 Branches
649MB
Tree: 5ed032fb8b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5ed032fb8b'
${ noResults }
FFmpeg/libavcodec/x86/h264_qpel_mmx.c

1290 lines
56KB
Raw Blame History 

  1. /*

  2.  * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt

  3.  * Copyright (c) 2011 Daniel Kang

  4.  *

  5.  * This file is part of Libav.

  6.  *

  7.  * Libav is free software; you can redistribute it and/or

  8.  * modify it under the terms of the GNU Lesser General Public

  9.  * License as published by the Free Software Foundation; either

  10.  * version 2.1 of the License, or (at your option) any later version.

  11.  *

  12.  * Libav is distributed in the hope that it will be useful,

  13.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  14.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  15.  * Lesser General Public License for more details.

  16.  *

  17.  * You should have received a copy of the GNU Lesser General Public

  18.  * License along with Libav; if not, write to the Free Software

  19.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  20.  */

  21. 

  22. #include "dsputil_mmx.h"

  23. 

  24. /***********************************/

  25. /* motion compensation */

  26. 

  27. #define QPEL_H264V_MM(A,B,C,D,E,F,OP,T,Z,d,q)\

  28.         "mov"#q" "#C", "#T"         \n\t"\

  29.         "mov"#d" (%0), "#F"         \n\t"\

  30.         "paddw "#D", "#T"           \n\t"\

  31.         "psllw $2, "#T"             \n\t"\

  32.         "psubw "#B", "#T"           \n\t"\

  33.         "psubw "#E", "#T"           \n\t"\

  34.         "punpcklbw "#Z", "#F"       \n\t"\

  35.         "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\

  36.         "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\

  37.         "add %2, %0                 \n\t"\

  38.         "paddw "#F", "#A"           \n\t"\

  39.         "paddw "#A", "#T"           \n\t"\

  40.         "psraw $5, "#T"             \n\t"\

  41.         "packuswb "#T", "#T"        \n\t"\

  42.         OP(T, (%1), A, d)\

  43.         "add %3, %1                 \n\t"

  44. 

  45. #define QPEL_H264HV_MM(A,B,C,D,E,F,OF,T,Z,d,q)\

  46.         "mov"#q" "#C", "#T"         \n\t"\

  47.         "mov"#d" (%0), "#F"         \n\t"\

  48.         "paddw "#D", "#T"           \n\t"\

  49.         "psllw $2, "#T"             \n\t"\

  50.         "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\

  51.         "psubw "#B", "#T"           \n\t"\

  52.         "psubw "#E", "#T"           \n\t"\

  53.         "punpcklbw "#Z", "#F"       \n\t"\

  54.         "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\

  55.         "paddw "#F", "#A"           \n\t"\

  56.         "add %2, %0                 \n\t"\

  57.         "paddw "#A", "#T"           \n\t"\

  58.         "mov"#q" "#T", "#OF"(%1)    \n\t"

  59. 

  60. #define QPEL_H264V(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%mm6,%%mm7,d,q)

  61. #define QPEL_H264HV(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%mm6,%%mm7,d,q)

  62. #define QPEL_H264V_XMM(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%xmm6,%%xmm7,q,dqa)

  63. #define QPEL_H264HV_XMM(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%xmm6,%%xmm7,q,dqa)

  64. 

  65. 

  66. #define QPEL_H264(OPNAME, OP, MMX)\

  67. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  68.     int h=4;\

  69. \

  70.     __asm__ volatile(\

  71.         "pxor %%mm7, %%mm7          \n\t"\

  72.         "movq "MANGLE(ff_pw_5) ", %%mm4\n\t"\

  73.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  74.         "1:                         \n\t"\

  75.         "movd  -1(%0), %%mm1        \n\t"\

  76.         "movd    (%0), %%mm2        \n\t"\

  77.         "movd   1(%0), %%mm3        \n\t"\

  78.         "movd   2(%0), %%mm0        \n\t"\

  79.         "punpcklbw %%mm7, %%mm1     \n\t"\

  80.         "punpcklbw %%mm7, %%mm2     \n\t"\

  81.         "punpcklbw %%mm7, %%mm3     \n\t"\

  82.         "punpcklbw %%mm7, %%mm0     \n\t"\

  83.         "paddw %%mm0, %%mm1         \n\t"\

  84.         "paddw %%mm3, %%mm2         \n\t"\

  85.         "movd  -2(%0), %%mm0        \n\t"\

  86.         "movd   3(%0), %%mm3        \n\t"\

  87.         "punpcklbw %%mm7, %%mm0     \n\t"\

  88.         "punpcklbw %%mm7, %%mm3     \n\t"\

  89.         "paddw %%mm3, %%mm0         \n\t"\

  90.         "psllw $2, %%mm2            \n\t"\

  91.         "psubw %%mm1, %%mm2         \n\t"\

  92.         "pmullw %%mm4, %%mm2        \n\t"\

  93.         "paddw %%mm5, %%mm0         \n\t"\

  94.         "paddw %%mm2, %%mm0         \n\t"\

  95.         "psraw $5, %%mm0            \n\t"\

  96.         "packuswb %%mm0, %%mm0      \n\t"\

  97.         OP(%%mm0, (%1),%%mm6, d)\

  98.         "add %3, %0                 \n\t"\

  99.         "add %4, %1                 \n\t"\

  100.         "decl %2                    \n\t"\

  101.         " jnz 1b                    \n\t"\

  102.         : "+a"(src), "+c"(dst), "+g"(h)\

  103.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  104.         : "memory"\

  105.     );\

  106. }\

  107. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  108.     int h=4;\

  109.     __asm__ volatile(\

  110.         "pxor %%mm7, %%mm7          \n\t"\

  111.         "movq %0, %%mm4             \n\t"\

  112.         "movq %1, %%mm5             \n\t"\

  113.         :: "m"(ff_pw_5), "m"(ff_pw_16)\

  114.     );\

  115.     do{\

  116.     __asm__ volatile(\

  117.         "movd  -1(%0), %%mm1        \n\t"\

  118.         "movd    (%0), %%mm2        \n\t"\

  119.         "movd   1(%0), %%mm3        \n\t"\

  120.         "movd   2(%0), %%mm0        \n\t"\

  121.         "punpcklbw %%mm7, %%mm1     \n\t"\

  122.         "punpcklbw %%mm7, %%mm2     \n\t"\

  123.         "punpcklbw %%mm7, %%mm3     \n\t"\

  124.         "punpcklbw %%mm7, %%mm0     \n\t"\

  125.         "paddw %%mm0, %%mm1         \n\t"\

  126.         "paddw %%mm3, %%mm2         \n\t"\

  127.         "movd  -2(%0), %%mm0        \n\t"\

  128.         "movd   3(%0), %%mm3        \n\t"\

  129.         "punpcklbw %%mm7, %%mm0     \n\t"\

  130.         "punpcklbw %%mm7, %%mm3     \n\t"\

  131.         "paddw %%mm3, %%mm0         \n\t"\

  132.         "psllw $2, %%mm2            \n\t"\

  133.         "psubw %%mm1, %%mm2         \n\t"\

  134.         "pmullw %%mm4, %%mm2        \n\t"\

  135.         "paddw %%mm5, %%mm0         \n\t"\

  136.         "paddw %%mm2, %%mm0         \n\t"\

  137.         "movd   (%2), %%mm3         \n\t"\

  138.         "psraw $5, %%mm0            \n\t"\

  139.         "packuswb %%mm0, %%mm0      \n\t"\

  140.         PAVGB" %%mm3, %%mm0         \n\t"\

  141.         OP(%%mm0, (%1),%%mm6, d)\

  142.         "add %4, %0                 \n\t"\

  143.         "add %4, %1                 \n\t"\

  144.         "add %3, %2                 \n\t"\

  145.         : "+a"(src), "+c"(dst), "+d"(src2)\

  146.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  147.         : "memory"\

  148.     );\

  149.     }while(--h);\

  150. }\

  151. static av_noinline void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  152.     src -= 2*srcStride;\

  153.     __asm__ volatile(\

  154.         "pxor %%mm7, %%mm7          \n\t"\

  155.         "movd (%0), %%mm0           \n\t"\

  156.         "add %2, %0                 \n\t"\

  157.         "movd (%0), %%mm1           \n\t"\

  158.         "add %2, %0                 \n\t"\

  159.         "movd (%0), %%mm2           \n\t"\

  160.         "add %2, %0                 \n\t"\

  161.         "movd (%0), %%mm3           \n\t"\

  162.         "add %2, %0                 \n\t"\

  163.         "movd (%0), %%mm4           \n\t"\

  164.         "add %2, %0                 \n\t"\

  165.         "punpcklbw %%mm7, %%mm0     \n\t"\

  166.         "punpcklbw %%mm7, %%mm1     \n\t"\

  167.         "punpcklbw %%mm7, %%mm2     \n\t"\

  168.         "punpcklbw %%mm7, %%mm3     \n\t"\

  169.         "punpcklbw %%mm7, %%mm4     \n\t"\

  170.         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  171.         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  172.         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  173.         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  174.          \

  175.         : "+a"(src), "+c"(dst)\

  176.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  177.         : "memory"\

  178.     );\

  179. }\

  180. static av_noinline void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  181.     int h=4;\

  182.     int w=3;\

  183.     src -= 2*srcStride+2;\

  184.     while(w--){\

  185.         __asm__ volatile(\

  186.             "pxor %%mm7, %%mm7      \n\t"\

  187.             "movd (%0), %%mm0       \n\t"\

  188.             "add %2, %0             \n\t"\

  189.             "movd (%0), %%mm1       \n\t"\

  190.             "add %2, %0             \n\t"\

  191.             "movd (%0), %%mm2       \n\t"\

  192.             "add %2, %0             \n\t"\

  193.             "movd (%0), %%mm3       \n\t"\

  194.             "add %2, %0             \n\t"\

  195.             "movd (%0), %%mm4       \n\t"\

  196.             "add %2, %0             \n\t"\

  197.             "punpcklbw %%mm7, %%mm0 \n\t"\

  198.             "punpcklbw %%mm7, %%mm1 \n\t"\

  199.             "punpcklbw %%mm7, %%mm2 \n\t"\

  200.             "punpcklbw %%mm7, %%mm3 \n\t"\

  201.             "punpcklbw %%mm7, %%mm4 \n\t"\

  202.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\

  203.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\

  204.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\

  205.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\

  206.              \

  207.             : "+a"(src)\

  208.             : "c"(tmp), "S"((x86_reg)srcStride)\

  209.             : "memory"\

  210.         );\

  211.         tmp += 4;\

  212.         src += 4 - 9*srcStride;\

  213.     }\

  214.     tmp -= 3*4;\

  215.     __asm__ volatile(\

  216.         "1:                         \n\t"\

  217.         "movq     (%0), %%mm0       \n\t"\

  218.         "paddw  10(%0), %%mm0       \n\t"\

  219.         "movq    2(%0), %%mm1       \n\t"\

  220.         "paddw   8(%0), %%mm1       \n\t"\

  221.         "movq    4(%0), %%mm2       \n\t"\

  222.         "paddw   6(%0), %%mm2       \n\t"\

  223.         "psubw %%mm1, %%mm0         \n\t"/*a-b   (abccba)*/\

  224.         "psraw $2, %%mm0            \n\t"/*(a-b)/4 */\

  225.         "psubw %%mm1, %%mm0         \n\t"/*(a-b)/4-b */\

  226.         "paddsw %%mm2, %%mm0        \n\t"\

  227.         "psraw $2, %%mm0            \n\t"/*((a-b)/4-b+c)/4 */\

  228.         "paddw %%mm2, %%mm0         \n\t"/*(a-5*b+20*c)/16 */\

  229.         "psraw $6, %%mm0            \n\t"\

  230.         "packuswb %%mm0, %%mm0      \n\t"\

  231.         OP(%%mm0, (%1),%%mm7, d)\

  232.         "add $24, %0                \n\t"\

  233.         "add %3, %1                 \n\t"\

  234.         "decl %2                    \n\t"\

  235.         " jnz 1b                    \n\t"\

  236.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  237.         : "S"((x86_reg)dstStride)\

  238.         : "memory"\

  239.     );\

  240. }\

  241. \

  242. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  243.     int h=8;\

  244.     __asm__ volatile(\

  245.         "pxor %%mm7, %%mm7          \n\t"\

  246.         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\

  247.         "1:                         \n\t"\

  248.         "movq    (%0), %%mm0        \n\t"\

  249.         "movq   1(%0), %%mm2        \n\t"\

  250.         "movq %%mm0, %%mm1          \n\t"\

  251.         "movq %%mm2, %%mm3          \n\t"\

  252.         "punpcklbw %%mm7, %%mm0     \n\t"\

  253.         "punpckhbw %%mm7, %%mm1     \n\t"\

  254.         "punpcklbw %%mm7, %%mm2     \n\t"\

  255.         "punpckhbw %%mm7, %%mm3     \n\t"\

  256.         "paddw %%mm2, %%mm0         \n\t"\

  257.         "paddw %%mm3, %%mm1         \n\t"\

  258.         "psllw $2, %%mm0            \n\t"\

  259.         "psllw $2, %%mm1            \n\t"\

  260.         "movq   -1(%0), %%mm2       \n\t"\

  261.         "movq    2(%0), %%mm4       \n\t"\

  262.         "movq %%mm2, %%mm3          \n\t"\

  263.         "movq %%mm4, %%mm5          \n\t"\

  264.         "punpcklbw %%mm7, %%mm2     \n\t"\

  265.         "punpckhbw %%mm7, %%mm3     \n\t"\

  266.         "punpcklbw %%mm7, %%mm4     \n\t"\

  267.         "punpckhbw %%mm7, %%mm5     \n\t"\

  268.         "paddw %%mm4, %%mm2         \n\t"\

  269.         "paddw %%mm3, %%mm5         \n\t"\

  270.         "psubw %%mm2, %%mm0         \n\t"\

  271.         "psubw %%mm5, %%mm1         \n\t"\

  272.         "pmullw %%mm6, %%mm0        \n\t"\

  273.         "pmullw %%mm6, %%mm1        \n\t"\

  274.         "movd   -2(%0), %%mm2       \n\t"\

  275.         "movd    7(%0), %%mm5       \n\t"\

  276.         "punpcklbw %%mm7, %%mm2     \n\t"\

  277.         "punpcklbw %%mm7, %%mm5     \n\t"\

  278.         "paddw %%mm3, %%mm2         \n\t"\

  279.         "paddw %%mm5, %%mm4         \n\t"\

  280.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  281.         "paddw %%mm5, %%mm2         \n\t"\

  282.         "paddw %%mm5, %%mm4         \n\t"\

  283.         "paddw %%mm2, %%mm0         \n\t"\

  284.         "paddw %%mm4, %%mm1         \n\t"\

  285.         "psraw $5, %%mm0            \n\t"\

  286.         "psraw $5, %%mm1            \n\t"\

  287.         "packuswb %%mm1, %%mm0      \n\t"\

  288.         OP(%%mm0, (%1),%%mm5, q)\

  289.         "add %3, %0                 \n\t"\

  290.         "add %4, %1                 \n\t"\

  291.         "decl %2                    \n\t"\

  292.         " jnz 1b                    \n\t"\

  293.         : "+a"(src), "+c"(dst), "+g"(h)\

  294.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  295.         : "memory"\

  296.     );\

  297. }\

  298. \

  299. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  300.     int h=8;\

  301.     __asm__ volatile(\

  302.         "pxor %%mm7, %%mm7          \n\t"\

  303.         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\

  304.         "1:                         \n\t"\

  305.         "movq    (%0), %%mm0        \n\t"\

  306.         "movq   1(%0), %%mm2        \n\t"\

  307.         "movq %%mm0, %%mm1          \n\t"\

  308.         "movq %%mm2, %%mm3          \n\t"\

  309.         "punpcklbw %%mm7, %%mm0     \n\t"\

  310.         "punpckhbw %%mm7, %%mm1     \n\t"\

  311.         "punpcklbw %%mm7, %%mm2     \n\t"\

  312.         "punpckhbw %%mm7, %%mm3     \n\t"\

  313.         "paddw %%mm2, %%mm0         \n\t"\

  314.         "paddw %%mm3, %%mm1         \n\t"\

  315.         "psllw $2, %%mm0            \n\t"\

  316.         "psllw $2, %%mm1            \n\t"\

  317.         "movq   -1(%0), %%mm2       \n\t"\

  318.         "movq    2(%0), %%mm4       \n\t"\

  319.         "movq %%mm2, %%mm3          \n\t"\

  320.         "movq %%mm4, %%mm5          \n\t"\

  321.         "punpcklbw %%mm7, %%mm2     \n\t"\

  322.         "punpckhbw %%mm7, %%mm3     \n\t"\

  323.         "punpcklbw %%mm7, %%mm4     \n\t"\

  324.         "punpckhbw %%mm7, %%mm5     \n\t"\

  325.         "paddw %%mm4, %%mm2         \n\t"\

  326.         "paddw %%mm3, %%mm5         \n\t"\

  327.         "psubw %%mm2, %%mm0         \n\t"\

  328.         "psubw %%mm5, %%mm1         \n\t"\

  329.         "pmullw %%mm6, %%mm0        \n\t"\

  330.         "pmullw %%mm6, %%mm1        \n\t"\

  331.         "movd   -2(%0), %%mm2       \n\t"\

  332.         "movd    7(%0), %%mm5       \n\t"\

  333.         "punpcklbw %%mm7, %%mm2     \n\t"\

  334.         "punpcklbw %%mm7, %%mm5     \n\t"\

  335.         "paddw %%mm3, %%mm2         \n\t"\

  336.         "paddw %%mm5, %%mm4         \n\t"\

  337.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  338.         "paddw %%mm5, %%mm2         \n\t"\

  339.         "paddw %%mm5, %%mm4         \n\t"\

  340.         "paddw %%mm2, %%mm0         \n\t"\

  341.         "paddw %%mm4, %%mm1         \n\t"\

  342.         "psraw $5, %%mm0            \n\t"\

  343.         "psraw $5, %%mm1            \n\t"\

  344.         "movq (%2), %%mm4           \n\t"\

  345.         "packuswb %%mm1, %%mm0      \n\t"\

  346.         PAVGB" %%mm4, %%mm0         \n\t"\

  347.         OP(%%mm0, (%1),%%mm5, q)\

  348.         "add %5, %0                 \n\t"\

  349.         "add %5, %1                 \n\t"\

  350.         "add %4, %2                 \n\t"\

  351.         "decl %3                    \n\t"\

  352.         "jg 1b                      \n\t"\

  353.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  354.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  355.         : "memory"\

  356.     );\

  357. }\

  358. \

  359. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  360.     int w= 2;\

  361.     src -= 2*srcStride;\

  362.     \

  363.     while(w--){\

  364.         __asm__ volatile(\

  365.             "pxor %%mm7, %%mm7          \n\t"\

  366.             "movd (%0), %%mm0           \n\t"\

  367.             "add %2, %0                 \n\t"\

  368.             "movd (%0), %%mm1           \n\t"\

  369.             "add %2, %0                 \n\t"\

  370.             "movd (%0), %%mm2           \n\t"\

  371.             "add %2, %0                 \n\t"\

  372.             "movd (%0), %%mm3           \n\t"\

  373.             "add %2, %0                 \n\t"\

  374.             "movd (%0), %%mm4           \n\t"\

  375.             "add %2, %0                 \n\t"\

  376.             "punpcklbw %%mm7, %%mm0     \n\t"\

  377.             "punpcklbw %%mm7, %%mm1     \n\t"\

  378.             "punpcklbw %%mm7, %%mm2     \n\t"\

  379.             "punpcklbw %%mm7, %%mm3     \n\t"\

  380.             "punpcklbw %%mm7, %%mm4     \n\t"\

  381.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  382.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  383.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  384.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  385.             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  386.             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  387.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  388.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  389.             "cmpl $16, %4               \n\t"\

  390.             "jne 2f                     \n\t"\

  391.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  392.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  393.             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  394.             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  395.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  396.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  397.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  398.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  399.             "2:                         \n\t"\

  400.             \

  401.             : "+a"(src), "+c"(dst)\

  402.             : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\

  403.             : "memory"\

  404.         );\

  405.         src += 4-(h+5)*srcStride;\

  406.         dst += 4-h*dstStride;\

  407.     }\

  408. }\

  409. static av_always_inline void OPNAME ## h264_qpel8or16_hv1_lowpass_ ## MMX(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){\

  410.     int w = (size+8)>>2;\

  411.     src -= 2*srcStride+2;\

  412.     while(w--){\

  413.         __asm__ volatile(\

  414.             "pxor %%mm7, %%mm7      \n\t"\

  415.             "movd (%0), %%mm0       \n\t"\

  416.             "add %2, %0             \n\t"\

  417.             "movd (%0), %%mm1       \n\t"\

  418.             "add %2, %0             \n\t"\

  419.             "movd (%0), %%mm2       \n\t"\

  420.             "add %2, %0             \n\t"\

  421.             "movd (%0), %%mm3       \n\t"\

  422.             "add %2, %0             \n\t"\

  423.             "movd (%0), %%mm4       \n\t"\

  424.             "add %2, %0             \n\t"\

  425.             "punpcklbw %%mm7, %%mm0 \n\t"\

  426.             "punpcklbw %%mm7, %%mm1 \n\t"\

  427.             "punpcklbw %%mm7, %%mm2 \n\t"\

  428.             "punpcklbw %%mm7, %%mm3 \n\t"\

  429.             "punpcklbw %%mm7, %%mm4 \n\t"\

  430.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\

  431.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\

  432.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\

  433.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\

  434.             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\

  435.             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\

  436.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\

  437.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\

  438.             "cmpl $16, %3           \n\t"\

  439.             "jne 2f                 \n\t"\

  440.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1,  8*48)\

  441.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2,  9*48)\

  442.             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\

  443.             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\

  444.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\

  445.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\

  446.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\

  447.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\

  448.             "2:                     \n\t"\

  449.             : "+a"(src)\

  450.             : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size)\

  451.             : "memory"\

  452.             );\

  453.         tmp += 4;\

  454.         src += 4 - (size+5)*srcStride;\

  455.     }\

  456. }\

  457. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  458.     int w = size>>4;\

  459.     do{\

  460.     int h = size;\

  461.     __asm__ volatile(\

  462.         "1:                         \n\t"\

  463.         "movq     (%0), %%mm0       \n\t"\

  464.         "movq    8(%0), %%mm3       \n\t"\

  465.         "movq    2(%0), %%mm1       \n\t"\

  466.         "movq   10(%0), %%mm4       \n\t"\

  467.         "paddw   %%mm4, %%mm0       \n\t"\

  468.         "paddw   %%mm3, %%mm1       \n\t"\

  469.         "paddw  18(%0), %%mm3       \n\t"\

  470.         "paddw  16(%0), %%mm4       \n\t"\

  471.         "movq    4(%0), %%mm2       \n\t"\

  472.         "movq   12(%0), %%mm5       \n\t"\

  473.         "paddw   6(%0), %%mm2       \n\t"\

  474.         "paddw  14(%0), %%mm5       \n\t"\

  475.         "psubw %%mm1, %%mm0         \n\t"\

  476.         "psubw %%mm4, %%mm3         \n\t"\

  477.         "psraw $2, %%mm0            \n\t"\

  478.         "psraw $2, %%mm3            \n\t"\

  479.         "psubw %%mm1, %%mm0         \n\t"\

  480.         "psubw %%mm4, %%mm3         \n\t"\

  481.         "paddsw %%mm2, %%mm0        \n\t"\

  482.         "paddsw %%mm5, %%mm3        \n\t"\

  483.         "psraw $2, %%mm0            \n\t"\

  484.         "psraw $2, %%mm3            \n\t"\

  485.         "paddw %%mm2, %%mm0         \n\t"\

  486.         "paddw %%mm5, %%mm3         \n\t"\

  487.         "psraw $6, %%mm0            \n\t"\

  488.         "psraw $6, %%mm3            \n\t"\

  489.         "packuswb %%mm3, %%mm0      \n\t"\

  490.         OP(%%mm0, (%1),%%mm7, q)\

  491.         "add $48, %0                \n\t"\

  492.         "add %3, %1                 \n\t"\

  493.         "decl %2                    \n\t"\

  494.         " jnz 1b                    \n\t"\

  495.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  496.         : "S"((x86_reg)dstStride)\

  497.         : "memory"\

  498.     );\

  499.     tmp += 8 - size*24;\

  500.     dst += 8 - size*dstStride;\

  501.     }while(w--);\

  502. }\

  503. \

  504. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  505.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  506. }\

  507. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  508.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  509.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  510. }\

  511. \

  512. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  513.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  514.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  515.     src += 8*srcStride;\

  516.     dst += 8*dstStride;\

  517.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  518.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  519. }\

  520. \

  521. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  522.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  523.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  524.     src += 8*dstStride;\

  525.     dst += 8*dstStride;\

  526.     src2 += 8*src2Stride;\

  527.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  528.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  529. }\

  530. \

  531. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  532.           put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\

  533.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  534. }\

  535. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  536.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 8);\

  537. }\

  538. \

  539. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  540.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 16);\

  541. }\

  542. \

  543. static av_noinline void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  544. {\

  545.     __asm__ volatile(\

  546.         "movq      (%1), %%mm0          \n\t"\

  547.         "movq    24(%1), %%mm1          \n\t"\

  548.         "psraw      $5,  %%mm0          \n\t"\

  549.         "psraw      $5,  %%mm1          \n\t"\

  550.         "packuswb %%mm0, %%mm0          \n\t"\

  551.         "packuswb %%mm1, %%mm1          \n\t"\

  552.         PAVGB"     (%0), %%mm0          \n\t"\

  553.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  554.         OP(%%mm0, (%2),    %%mm4, d)\

  555.         OP(%%mm1, (%2,%4), %%mm5, d)\

  556.         "lea  (%0,%3,2), %0             \n\t"\

  557.         "lea  (%2,%4,2), %2             \n\t"\

  558.         "movq    48(%1), %%mm0          \n\t"\

  559.         "movq    72(%1), %%mm1          \n\t"\

  560.         "psraw      $5,  %%mm0          \n\t"\

  561.         "psraw      $5,  %%mm1          \n\t"\

  562.         "packuswb %%mm0, %%mm0          \n\t"\

  563.         "packuswb %%mm1, %%mm1          \n\t"\

  564.         PAVGB"     (%0), %%mm0          \n\t"\

  565.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  566.         OP(%%mm0, (%2),    %%mm4, d)\

  567.         OP(%%mm1, (%2,%4), %%mm5, d)\

  568.         :"+a"(src8), "+c"(src16), "+d"(dst)\

  569.         :"S"((x86_reg)src8Stride), "D"((x86_reg)dstStride)\

  570.         :"memory");\

  571. }\

  572. static av_noinline void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  573. {\

  574.     do{\

  575.     __asm__ volatile(\

  576.         "movq      (%1), %%mm0          \n\t"\

  577.         "movq     8(%1), %%mm1          \n\t"\

  578.         "movq    48(%1), %%mm2          \n\t"\

  579.         "movq  8+48(%1), %%mm3          \n\t"\

  580.         "psraw      $5,  %%mm0          \n\t"\

  581.         "psraw      $5,  %%mm1          \n\t"\

  582.         "psraw      $5,  %%mm2          \n\t"\

  583.         "psraw      $5,  %%mm3          \n\t"\

  584.         "packuswb %%mm1, %%mm0          \n\t"\

  585.         "packuswb %%mm3, %%mm2          \n\t"\

  586.         PAVGB"     (%0), %%mm0          \n\t"\

  587.         PAVGB"  (%0,%3), %%mm2          \n\t"\

  588.         OP(%%mm0, (%2), %%mm5, q)\

  589.         OP(%%mm2, (%2,%4), %%mm5, q)\

  590.         ::"a"(src8), "c"(src16), "d"(dst),\

  591.           "r"((x86_reg)src8Stride), "r"((x86_reg)dstStride)\

  592.         :"memory");\

  593.         src8 += 2L*src8Stride;\

  594.         src16 += 48;\

  595.         dst += 2L*dstStride;\

  596.     }while(h-=2);\

  597. }\

  598. static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  599. {\

  600.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst  , src16  , src8  , dstStride, src8Stride, h);\

  601.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\

  602. }\

  603. 

  604. 

  605. #if ARCH_X86_64

  606. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  607. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  608.     int h=16;\

  609.     __asm__ volatile(\

  610.         "pxor %%xmm15, %%xmm15      \n\t"\

  611.         "movdqa %6, %%xmm14         \n\t"\

  612.         "movdqa %7, %%xmm13         \n\t"\

  613.         "1:                         \n\t"\

  614.         "lddqu    6(%0), %%xmm1     \n\t"\

  615.         "lddqu   -2(%0), %%xmm7     \n\t"\

  616.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  617.         "punpckhbw %%xmm15, %%xmm1  \n\t"\

  618.         "punpcklbw %%xmm15, %%xmm0  \n\t"\

  619.         "punpcklbw %%xmm15, %%xmm7  \n\t"\

  620.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  621.         "movdqa  %%xmm0, %%xmm6     \n\t"\

  622.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  623.         "movdqa  %%xmm0, %%xmm8     \n\t"\

  624.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  625.         "movdqa  %%xmm0, %%xmm9     \n\t"\

  626.         "movdqa  %%xmm0, %%xmm12    \n\t"\

  627.         "movdqa  %%xmm1, %%xmm11    \n\t"\

  628.         "palignr $10,%%xmm0, %%xmm11\n\t"\

  629.         "palignr $10,%%xmm7, %%xmm12\n\t"\

  630.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  631.         "palignr $2, %%xmm7, %%xmm9 \n\t"\

  632.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  633.         "palignr $4, %%xmm7, %%xmm8 \n\t"\

  634.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  635.         "palignr $6, %%xmm7, %%xmm6 \n\t"\

  636.         "paddw   %%xmm0 ,%%xmm11    \n\t"\

  637.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  638.         "palignr $8, %%xmm7, %%xmm0 \n\t"\

  639.         "paddw   %%xmm12,%%xmm7     \n\t"\

  640.         "paddw   %%xmm3, %%xmm2     \n\t"\

  641.         "paddw   %%xmm8, %%xmm6     \n\t"\

  642.         "paddw   %%xmm4, %%xmm1     \n\t"\

  643.         "paddw   %%xmm9, %%xmm0     \n\t"\

  644.         "psllw   $2,     %%xmm2     \n\t"\

  645.         "psllw   $2,     %%xmm6     \n\t"\

  646.         "psubw   %%xmm1, %%xmm2     \n\t"\

  647.         "psubw   %%xmm0, %%xmm6     \n\t"\

  648.         "paddw   %%xmm13,%%xmm11    \n\t"\

  649.         "paddw   %%xmm13,%%xmm7     \n\t"\

  650.         "pmullw  %%xmm14,%%xmm2     \n\t"\

  651.         "pmullw  %%xmm14,%%xmm6     \n\t"\

  652.         "lddqu   (%2),   %%xmm3     \n\t"\

  653.         "paddw   %%xmm11,%%xmm2     \n\t"\

  654.         "paddw   %%xmm7, %%xmm6     \n\t"\

  655.         "psraw   $5,     %%xmm2     \n\t"\

  656.         "psraw   $5,     %%xmm6     \n\t"\

  657.         "packuswb %%xmm2,%%xmm6     \n\t"\

  658.         "pavgb   %%xmm3, %%xmm6     \n\t"\

  659.         OP(%%xmm6, (%1), %%xmm4, dqa)\

  660.         "add %5, %0                 \n\t"\

  661.         "add %5, %1                 \n\t"\

  662.         "add %4, %2                 \n\t"\

  663.         "decl %3                    \n\t"\

  664.         "jg 1b                      \n\t"\

  665.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  666.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\

  667.           "m"(ff_pw_5), "m"(ff_pw_16)\

  668.         : XMM_CLOBBERS("%xmm0" , "%xmm1" , "%xmm2" , "%xmm3" , \

  669.                        "%xmm4" , "%xmm5" , "%xmm6" , "%xmm7" , \

  670.                        "%xmm8" , "%xmm9" , "%xmm10", "%xmm11", \

  671.                        "%xmm12", "%xmm13", "%xmm14", "%xmm15",)\

  672.           "memory"\

  673.     );\

  674. }

  675. #else // ARCH_X86_64

  676. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  677. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  678.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  679.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  680.     src += 8*dstStride;\

  681.     dst += 8*dstStride;\

  682.     src2 += 8*src2Stride;\

  683.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  684.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  685. }

  686. #endif // ARCH_X86_64

  687. 

  688. #define QPEL_H264_H_XMM(OPNAME, OP, MMX)\

  689. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  690.     int h=8;\

  691.     __asm__ volatile(\

  692.         "pxor %%xmm7, %%xmm7        \n\t"\

  693.         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\

  694.         "1:                         \n\t"\

  695.         "lddqu   -2(%0), %%xmm1     \n\t"\

  696.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  697.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  698.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  699.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  700.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  701.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  702.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  703.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  704.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  705.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  706.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  707.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  708.         "paddw   %%xmm5, %%xmm0     \n\t"\

  709.         "paddw   %%xmm3, %%xmm2     \n\t"\

  710.         "paddw   %%xmm4, %%xmm1     \n\t"\

  711.         "psllw   $2,     %%xmm2     \n\t"\

  712.         "movq    (%2),   %%xmm3     \n\t"\

  713.         "psubw   %%xmm1, %%xmm2     \n\t"\

  714.         "paddw "MANGLE(ff_pw_16)", %%xmm0\n\t"\

  715.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  716.         "paddw   %%xmm0, %%xmm2     \n\t"\

  717.         "psraw   $5,     %%xmm2     \n\t"\

  718.         "packuswb %%xmm2, %%xmm2    \n\t"\

  719.         "pavgb   %%xmm3, %%xmm2     \n\t"\

  720.         OP(%%xmm2, (%1), %%xmm4, q)\

  721.         "add %5, %0                 \n\t"\

  722.         "add %5, %1                 \n\t"\

  723.         "add %4, %2                 \n\t"\

  724.         "decl %3                    \n\t"\

  725.         "jg 1b                      \n\t"\

  726.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  727.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  728.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  729.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  730.           "memory"\

  731.     );\

  732. }\

  733. QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  734. \

  735. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  736.     int h=8;\

  737.     __asm__ volatile(\

  738.         "pxor %%xmm7, %%xmm7        \n\t"\

  739.         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\

  740.         "1:                         \n\t"\

  741.         "lddqu   -2(%0), %%xmm1     \n\t"\

  742.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  743.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  744.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  745.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  746.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  747.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  748.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  749.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  750.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  751.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  752.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  753.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  754.         "paddw   %%xmm5, %%xmm0     \n\t"\

  755.         "paddw   %%xmm3, %%xmm2     \n\t"\

  756.         "paddw   %%xmm4, %%xmm1     \n\t"\

  757.         "psllw   $2,     %%xmm2     \n\t"\

  758.         "psubw   %%xmm1, %%xmm2     \n\t"\

  759.         "paddw   "MANGLE(ff_pw_16)", %%xmm0\n\t"\

  760.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  761.         "paddw   %%xmm0, %%xmm2     \n\t"\

  762.         "psraw   $5,     %%xmm2     \n\t"\

  763.         "packuswb %%xmm2, %%xmm2    \n\t"\

  764.         OP(%%xmm2, (%1), %%xmm4, q)\

  765.         "add %3, %0                 \n\t"\

  766.         "add %4, %1                 \n\t"\

  767.         "decl %2                    \n\t"\

  768.         " jnz 1b                    \n\t"\

  769.         : "+a"(src), "+c"(dst), "+g"(h)\

  770.         : "D"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  771.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  772.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  773.           "memory"\

  774.     );\

  775. }\

  776. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  777.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  778.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  779.     src += 8*srcStride;\

  780.     dst += 8*dstStride;\

  781.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  782.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  783. }\

  784. 

  785. #define QPEL_H264_V_XMM(OPNAME, OP, MMX)\

  786. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  787.     src -= 2*srcStride;\

  788.     \

  789.     __asm__ volatile(\

  790.         "pxor %%xmm7, %%xmm7        \n\t"\

  791.         "movq (%0), %%xmm0          \n\t"\

  792.         "add %2, %0                 \n\t"\

  793.         "movq (%0), %%xmm1          \n\t"\

  794.         "add %2, %0                 \n\t"\

  795.         "movq (%0), %%xmm2          \n\t"\

  796.         "add %2, %0                 \n\t"\

  797.         "movq (%0), %%xmm3          \n\t"\

  798.         "add %2, %0                 \n\t"\

  799.         "movq (%0), %%xmm4          \n\t"\

  800.         "add %2, %0                 \n\t"\

  801.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  802.         "punpcklbw %%xmm7, %%xmm1   \n\t"\

  803.         "punpcklbw %%xmm7, %%xmm2   \n\t"\

  804.         "punpcklbw %%xmm7, %%xmm3   \n\t"\

  805.         "punpcklbw %%xmm7, %%xmm4   \n\t"\

  806.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  807.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  808.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  809.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  810.         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  811.         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  812.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  813.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  814.         "cmpl $16, %4               \n\t"\

  815.         "jne 2f                     \n\t"\

  816.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  817.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  818.         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  819.         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  820.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  821.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  822.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  823.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  824.         "2:                          \n\t"\

  825.         \

  826.         : "+a"(src), "+c"(dst)\

  827.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\

  828.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  829.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  830.           "memory"\

  831.     );\

  832. }\

  833. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  834.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  835. }\

  836. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  837.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  838.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  839. }

  840. 

  841. static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){

  842.     int w = (size+8)>>3;

  843.     src -= 2*srcStride+2;

  844.     while(w--){

  845.         __asm__ volatile(

  846.             "pxor %%xmm7, %%xmm7        \n\t"

  847.             "movq (%0), %%xmm0          \n\t"

  848.             "add %2, %0                 \n\t"

  849.             "movq (%0), %%xmm1          \n\t"

  850.             "add %2, %0                 \n\t"

  851.             "movq (%0), %%xmm2          \n\t"

  852.             "add %2, %0                 \n\t"

  853.             "movq (%0), %%xmm3          \n\t"

  854.             "add %2, %0                 \n\t"

  855.             "movq (%0), %%xmm4          \n\t"

  856.             "add %2, %0                 \n\t"

  857.             "punpcklbw %%xmm7, %%xmm0   \n\t"

  858.             "punpcklbw %%xmm7, %%xmm1   \n\t"

  859.             "punpcklbw %%xmm7, %%xmm2   \n\t"

  860.             "punpcklbw %%xmm7, %%xmm3   \n\t"

  861.             "punpcklbw %%xmm7, %%xmm4   \n\t"

  862.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 0*48)

  863.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 1*48)

  864.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 2*48)

  865.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 3*48)

  866.             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 4*48)

  867.             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 5*48)

  868.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 6*48)

  869.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 7*48)

  870.             "cmpl $16, %3               \n\t"

  871.             "jne 2f                     \n\t"

  872.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1,  8*48)

  873.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2,  9*48)

  874.             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 10*48)

  875.             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 11*48)

  876.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 12*48)

  877.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 13*48)

  878.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 14*48)

  879.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 15*48)

  880.             "2:                         \n\t"

  881.             : "+a"(src)

  882.             : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size)

  883.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3",

  884.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)

  885.               "memory"

  886.         );

  887.         tmp += 8;

  888.         src += 8 - (size+5)*srcStride;

  889.     }

  890. }

  891. 

  892. #define QPEL_H264_HV2_XMM(OPNAME, OP, MMX)\

  893. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  894.     int h = size;\

  895.     if(size == 16){\

  896.         __asm__ volatile(\

  897.             "1:                         \n\t"\

  898.             "movdqa 32(%0), %%xmm4      \n\t"\

  899.             "movdqa 16(%0), %%xmm5      \n\t"\

  900.             "movdqa   (%0), %%xmm7      \n\t"\

  901.             "movdqa %%xmm4, %%xmm3      \n\t"\

  902.             "movdqa %%xmm4, %%xmm2      \n\t"\

  903.             "movdqa %%xmm4, %%xmm1      \n\t"\

  904.             "movdqa %%xmm4, %%xmm0      \n\t"\

  905.             "palignr $10, %%xmm5, %%xmm0 \n\t"\

  906.             "palignr  $8, %%xmm5, %%xmm1 \n\t"\

  907.             "palignr  $6, %%xmm5, %%xmm2 \n\t"\

  908.             "palignr  $4, %%xmm5, %%xmm3 \n\t"\

  909.             "palignr  $2, %%xmm5, %%xmm4 \n\t"\

  910.             "paddw  %%xmm5, %%xmm0      \n\t"\

  911.             "paddw  %%xmm4, %%xmm1      \n\t"\

  912.             "paddw  %%xmm3, %%xmm2      \n\t"\

  913.             "movdqa %%xmm5, %%xmm6      \n\t"\

  914.             "movdqa %%xmm5, %%xmm4      \n\t"\

  915.             "movdqa %%xmm5, %%xmm3      \n\t"\

  916.             "palignr  $8, %%xmm7, %%xmm4 \n\t"\

  917.             "palignr  $2, %%xmm7, %%xmm6 \n\t"\

  918.             "palignr $10, %%xmm7, %%xmm3 \n\t"\

  919.             "paddw  %%xmm6, %%xmm4      \n\t"\

  920.             "movdqa %%xmm5, %%xmm6      \n\t"\

  921.             "palignr  $6, %%xmm7, %%xmm5 \n\t"\

  922.             "palignr  $4, %%xmm7, %%xmm6 \n\t"\

  923.             "paddw  %%xmm7, %%xmm3      \n\t"\

  924.             "paddw  %%xmm6, %%xmm5      \n\t"\

  925.             \

  926.             "psubw  %%xmm1, %%xmm0      \n\t"\

  927.             "psubw  %%xmm4, %%xmm3      \n\t"\

  928.             "psraw      $2, %%xmm0      \n\t"\

  929.             "psraw      $2, %%xmm3      \n\t"\

  930.             "psubw  %%xmm1, %%xmm0      \n\t"\

  931.             "psubw  %%xmm4, %%xmm3      \n\t"\

  932.             "paddw  %%xmm2, %%xmm0      \n\t"\

  933.             "paddw  %%xmm5, %%xmm3      \n\t"\

  934.             "psraw      $2, %%xmm0      \n\t"\

  935.             "psraw      $2, %%xmm3      \n\t"\

  936.             "paddw  %%xmm2, %%xmm0      \n\t"\

  937.             "paddw  %%xmm5, %%xmm3      \n\t"\

  938.             "psraw      $6, %%xmm0      \n\t"\

  939.             "psraw      $6, %%xmm3      \n\t"\

  940.             "packuswb %%xmm0, %%xmm3    \n\t"\

  941.             OP(%%xmm3, (%1), %%xmm7, dqa)\

  942.             "add $48, %0                \n\t"\

  943.             "add %3, %1                 \n\t"\

  944.             "decl %2                    \n\t"\

  945.             " jnz 1b                    \n\t"\

  946.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  947.             : "S"((x86_reg)dstStride)\

  948.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  949.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  950.               "memory"\

  951.         );\

  952.     }else{\

  953.         __asm__ volatile(\

  954.             "1:                         \n\t"\

  955.             "movdqa 16(%0), %%xmm1      \n\t"\

  956.             "movdqa   (%0), %%xmm0      \n\t"\

  957.             "movdqa %%xmm1, %%xmm2      \n\t"\

  958.             "movdqa %%xmm1, %%xmm3      \n\t"\

  959.             "movdqa %%xmm1, %%xmm4      \n\t"\

  960.             "movdqa %%xmm1, %%xmm5      \n\t"\

  961.             "palignr $10, %%xmm0, %%xmm5 \n\t"\

  962.             "palignr  $8, %%xmm0, %%xmm4 \n\t"\

  963.             "palignr  $6, %%xmm0, %%xmm3 \n\t"\

  964.             "palignr  $4, %%xmm0, %%xmm2 \n\t"\

  965.             "palignr  $2, %%xmm0, %%xmm1 \n\t"\

  966.             "paddw  %%xmm5, %%xmm0      \n\t"\

  967.             "paddw  %%xmm4, %%xmm1      \n\t"\

  968.             "paddw  %%xmm3, %%xmm2      \n\t"\

  969.             "psubw  %%xmm1, %%xmm0      \n\t"\

  970.             "psraw      $2, %%xmm0      \n\t"\

  971.             "psubw  %%xmm1, %%xmm0      \n\t"\

  972.             "paddw  %%xmm2, %%xmm0      \n\t"\

  973.             "psraw      $2, %%xmm0      \n\t"\

  974.             "paddw  %%xmm2, %%xmm0      \n\t"\

  975.             "psraw      $6, %%xmm0      \n\t"\

  976.             "packuswb %%xmm0, %%xmm0    \n\t"\

  977.             OP(%%xmm0, (%1), %%xmm7, q)\

  978.             "add $48, %0                \n\t"\

  979.             "add %3, %1                 \n\t"\

  980.             "decl %2                    \n\t"\

  981.             " jnz 1b                    \n\t"\

  982.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  983.             : "S"((x86_reg)dstStride)\

  984.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  985.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  986.               "memory"\

  987.         );\

  988.     }\

  989. }

  990. 

  991. #define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\

  992. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  993.           put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\

  994.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  995. }\

  996. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  997.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\

  998. }\

  999. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  1000.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\

  1001. }\

  1002. 

  1003. #define put_pixels8_l2_sse2 put_pixels8_l2_mmx2

  1004. #define avg_pixels8_l2_sse2 avg_pixels8_l2_mmx2

  1005. #define put_pixels16_l2_sse2 put_pixels16_l2_mmx2

  1006. #define avg_pixels16_l2_sse2 avg_pixels16_l2_mmx2

  1007. #define put_pixels8_l2_ssse3 put_pixels8_l2_mmx2

  1008. #define avg_pixels8_l2_ssse3 avg_pixels8_l2_mmx2

  1009. #define put_pixels16_l2_ssse3 put_pixels16_l2_mmx2

  1010. #define avg_pixels16_l2_ssse3 avg_pixels16_l2_mmx2

  1011. 

  1012. #define put_pixels8_l2_shift5_sse2 put_pixels8_l2_shift5_mmx2

  1013. #define avg_pixels8_l2_shift5_sse2 avg_pixels8_l2_shift5_mmx2

  1014. #define put_pixels16_l2_shift5_sse2 put_pixels16_l2_shift5_mmx2

  1015. #define avg_pixels16_l2_shift5_sse2 avg_pixels16_l2_shift5_mmx2

  1016. #define put_pixels8_l2_shift5_ssse3 put_pixels8_l2_shift5_mmx2

  1017. #define avg_pixels8_l2_shift5_ssse3 avg_pixels8_l2_shift5_mmx2

  1018. #define put_pixels16_l2_shift5_ssse3 put_pixels16_l2_shift5_mmx2

  1019. #define avg_pixels16_l2_shift5_ssse3 avg_pixels16_l2_shift5_mmx2

  1020. 

  1021. #define put_h264_qpel8_h_lowpass_l2_sse2 put_h264_qpel8_h_lowpass_l2_mmx2

  1022. #define avg_h264_qpel8_h_lowpass_l2_sse2 avg_h264_qpel8_h_lowpass_l2_mmx2

  1023. #define put_h264_qpel16_h_lowpass_l2_sse2 put_h264_qpel16_h_lowpass_l2_mmx2

  1024. #define avg_h264_qpel16_h_lowpass_l2_sse2 avg_h264_qpel16_h_lowpass_l2_mmx2

  1025. 

  1026. #define put_h264_qpel8_v_lowpass_ssse3 put_h264_qpel8_v_lowpass_sse2

  1027. #define avg_h264_qpel8_v_lowpass_ssse3 avg_h264_qpel8_v_lowpass_sse2

  1028. #define put_h264_qpel16_v_lowpass_ssse3 put_h264_qpel16_v_lowpass_sse2

  1029. #define avg_h264_qpel16_v_lowpass_ssse3 avg_h264_qpel16_v_lowpass_sse2

  1030. 

  1031. #define put_h264_qpel8or16_hv2_lowpass_sse2 put_h264_qpel8or16_hv2_lowpass_mmx2

  1032. #define avg_h264_qpel8or16_hv2_lowpass_sse2 avg_h264_qpel8or16_hv2_lowpass_mmx2

  1033. 

  1034. #define H264_MC(OPNAME, SIZE, MMX, ALIGN) \

  1035. H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\

  1036. H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\

  1037. H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\

  1038. H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\

  1039. 

  1040. static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1041.     put_pixels16_sse2(dst, src, stride, 16);

  1042. }

  1043. static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1044.     avg_pixels16_sse2(dst, src, stride, 16);

  1045. }

  1046. #define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmx2

  1047. #define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmx2

  1048. 

  1049. #define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \

  1050. static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\

  1051.     OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\

  1052. }\

  1053. 

  1054. #define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \

  1055. static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1056.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\

  1057. }\

  1058. \

  1059. static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1060.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\

  1061. }\

  1062. \

  1063. static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1064.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\

  1065. }\

  1066. 

  1067. #define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \

  1068. static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1069.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1070.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1071.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\

  1072. }\

  1073. \

  1074. static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1075.     OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\

  1076. }\

  1077. \

  1078. static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1079.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1080.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1081.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\

  1082. }\

  1083. 

  1084. #define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \

  1085. static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1086.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1087.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1088.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1089. }\

  1090. \

  1091. static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1092.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1093.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1094.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1095. }\

  1096. \

  1097. static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1098.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1099.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1100.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1101. }\

  1102. \

  1103. static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1104.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1105.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1106.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1107. }\

  1108. \

  1109. static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1110.     DECLARE_ALIGNED(ALIGN, uint16_t, temp)[SIZE*(SIZE<8?12:24)];\

  1111.     OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\

  1112. }\

  1113. \

  1114. static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1115.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1116.     uint8_t * const halfHV= temp;\

  1117.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1118.     assert(((int)temp & 7) == 0);\

  1119.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1120.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\

  1121. }\

  1122. \

  1123. static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1124.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1125.     uint8_t * const halfHV= temp;\

  1126.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1127.     assert(((int)temp & 7) == 0);\

  1128.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1129.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\

  1130. }\

  1131. \

  1132. static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1133.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1134.     uint8_t * const halfHV= temp;\

  1135.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1136.     assert(((int)temp & 7) == 0);\

  1137.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1138.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\

  1139. }\

  1140. \

  1141. static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1142.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1143.     uint8_t * const halfHV= temp;\

  1144.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1145.     assert(((int)temp & 7) == 0);\

  1146.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1147.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\

  1148. }\

  1149. 

  1150. #define H264_MC_4816(MMX)\

  1151. H264_MC(put_, 4, MMX, 8)\

  1152. H264_MC(put_, 8, MMX, 8)\

  1153. H264_MC(put_, 16,MMX, 8)\

  1154. H264_MC(avg_, 4, MMX, 8)\

  1155. H264_MC(avg_, 8, MMX, 8)\

  1156. H264_MC(avg_, 16,MMX, 8)\

  1157. 

  1158. #define H264_MC_816(QPEL, XMM)\

  1159. QPEL(put_, 8, XMM, 16)\

  1160. QPEL(put_, 16,XMM, 16)\

  1161. QPEL(avg_, 8, XMM, 16)\

  1162. QPEL(avg_, 16,XMM, 16)\

  1163. 

  1164. #define PAVGB "pavgusb"

  1165. QPEL_H264(put_,       PUT_OP, 3dnow)

  1166. QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow)

  1167. #undef PAVGB

  1168. #define PAVGB "pavgb"

  1169. QPEL_H264(put_,       PUT_OP, mmx2)

  1170. QPEL_H264(avg_,  AVG_MMX2_OP, mmx2)

  1171. QPEL_H264_V_XMM(put_,       PUT_OP, sse2)

  1172. QPEL_H264_V_XMM(avg_,  AVG_MMX2_OP, sse2)

  1173. QPEL_H264_HV_XMM(put_,       PUT_OP, sse2)

  1174. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, sse2)

  1175. #if HAVE_SSSE3

  1176. QPEL_H264_H_XMM(put_,       PUT_OP, ssse3)

  1177. QPEL_H264_H_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1178. QPEL_H264_HV2_XMM(put_,       PUT_OP, ssse3)

  1179. QPEL_H264_HV2_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1180. QPEL_H264_HV_XMM(put_,       PUT_OP, ssse3)

  1181. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1182. #endif

  1183. #undef PAVGB

  1184. 

  1185. H264_MC_4816(3dnow)

  1186. H264_MC_4816(mmx2)

  1187. H264_MC_816(H264_MC_V, sse2)

  1188. H264_MC_816(H264_MC_HV, sse2)

  1189. #if HAVE_SSSE3

  1190. H264_MC_816(H264_MC_H, ssse3)

  1191. H264_MC_816(H264_MC_HV, ssse3)

  1192. #endif

  1193. 

  1194. 

  1195. 

  1196. //10bit

  1197. #define LUMA_MC_OP(OP, NUM, DEPTH, TYPE, OPT) \

  1198. void ff_ ## OP ## _h264_qpel ## NUM ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT \

  1199.     (uint8_t *dst, uint8_t *src, int stride);

  1200. 

  1201. #define LUMA_MC_ALL(DEPTH, TYPE, OPT) \

  1202.     LUMA_MC_OP(put,  4, DEPTH, TYPE, OPT) \

  1203.     LUMA_MC_OP(avg,  4, DEPTH, TYPE, OPT) \

  1204.     LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \

  1205.     LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \

  1206.     LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \

  1207.     LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)

  1208. 

  1209. #define LUMA_MC_816(DEPTH, TYPE, OPT) \

  1210.     LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \

  1211.     LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \

  1212.     LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \

  1213.     LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)

  1214. 

  1215. LUMA_MC_ALL(10, mc00, mmxext)

  1216. LUMA_MC_ALL(10, mc10, mmxext)

  1217. LUMA_MC_ALL(10, mc20, mmxext)

  1218. LUMA_MC_ALL(10, mc30, mmxext)

  1219. LUMA_MC_ALL(10, mc01, mmxext)

  1220. LUMA_MC_ALL(10, mc11, mmxext)

  1221. LUMA_MC_ALL(10, mc21, mmxext)

  1222. LUMA_MC_ALL(10, mc31, mmxext)

  1223. LUMA_MC_ALL(10, mc02, mmxext)

  1224. LUMA_MC_ALL(10, mc12, mmxext)

  1225. LUMA_MC_ALL(10, mc22, mmxext)

  1226. LUMA_MC_ALL(10, mc32, mmxext)

  1227. LUMA_MC_ALL(10, mc03, mmxext)

  1228. LUMA_MC_ALL(10, mc13, mmxext)

  1229. LUMA_MC_ALL(10, mc23, mmxext)

  1230. LUMA_MC_ALL(10, mc33, mmxext)

  1231. 

  1232. LUMA_MC_816(10, mc00, sse2)

  1233. LUMA_MC_816(10, mc10, sse2)

  1234. LUMA_MC_816(10, mc10, sse2_cache64)

  1235. LUMA_MC_816(10, mc10, ssse3_cache64)

  1236. LUMA_MC_816(10, mc20, sse2)

  1237. LUMA_MC_816(10, mc20, sse2_cache64)

  1238. LUMA_MC_816(10, mc20, ssse3_cache64)

  1239. LUMA_MC_816(10, mc30, sse2)

  1240. LUMA_MC_816(10, mc30, sse2_cache64)

  1241. LUMA_MC_816(10, mc30, ssse3_cache64)

  1242. LUMA_MC_816(10, mc01, sse2)

  1243. LUMA_MC_816(10, mc11, sse2)

  1244. LUMA_MC_816(10, mc21, sse2)

  1245. LUMA_MC_816(10, mc31, sse2)

  1246. LUMA_MC_816(10, mc02, sse2)

  1247. LUMA_MC_816(10, mc12, sse2)

  1248. LUMA_MC_816(10, mc22, sse2)

  1249. LUMA_MC_816(10, mc32, sse2)

  1250. LUMA_MC_816(10, mc03, sse2)

  1251. LUMA_MC_816(10, mc13, sse2)

  1252. LUMA_MC_816(10, mc23, sse2)

  1253. LUMA_MC_816(10, mc33, sse2)

  1254. 

  1255. #define QPEL16_OPMC(OP, MC, MMX)\

  1256. void ff_ ## OP ## _h264_qpel16_ ## MC ## _10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1257.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\

  1258.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\

  1259.     src += 8*stride;\

  1260.     dst += 8*stride;\

  1261.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\

  1262.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\

  1263. }

  1264. 

  1265. #define QPEL16_OP(MC, MMX)\

  1266. QPEL16_OPMC(put, MC, MMX)\

  1267. QPEL16_OPMC(avg, MC, MMX)

  1268. 

  1269. #define QPEL16(MMX)\

  1270. QPEL16_OP(mc00, MMX)\

  1271. QPEL16_OP(mc01, MMX)\

  1272. QPEL16_OP(mc02, MMX)\

  1273. QPEL16_OP(mc03, MMX)\

  1274. QPEL16_OP(mc10, MMX)\

  1275. QPEL16_OP(mc11, MMX)\

  1276. QPEL16_OP(mc12, MMX)\

  1277. QPEL16_OP(mc13, MMX)\

  1278. QPEL16_OP(mc20, MMX)\

  1279. QPEL16_OP(mc21, MMX)\

  1280. QPEL16_OP(mc22, MMX)\

  1281. QPEL16_OP(mc23, MMX)\

  1282. QPEL16_OP(mc30, MMX)\

  1283. QPEL16_OP(mc31, MMX)\

  1284. QPEL16_OP(mc32, MMX)\

  1285. QPEL16_OP(mc33, MMX)

  1286. 

  1287. #if ARCH_X86_32 && HAVE_YASM // ARCH_X86_64 implies sse2+

  1288. QPEL16(mmxext)

  1289. #endif