falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
35151 Commits
32 Branches
436MB
Tree: 473b297f26
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '473b297f26'
${ noResults }
FFmpeg/libavcodec/x86/h264_qpel.c

1292 lines
56KB
Raw Blame History 

  1. /*

  2.  * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt

  3.  * Copyright (c) 2011 Daniel Kang

  4.  *

  5.  * This file is part of Libav.

  6.  *

  7.  * Libav is free software; you can redistribute it and/or

  8.  * modify it under the terms of the GNU Lesser General Public

  9.  * License as published by the Free Software Foundation; either

  10.  * version 2.1 of the License, or (at your option) any later version.

  11.  *

  12.  * Libav is distributed in the hope that it will be useful,

  13.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  14.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  15.  * Lesser General Public License for more details.

  16.  *

  17.  * You should have received a copy of the GNU Lesser General Public

  18.  * License along with Libav; if not, write to the Free Software

  19.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  20.  */

  21. 

  22. #include "dsputil_mmx.h"

  23. 

  24. #if HAVE_INLINE_ASM

  25. 

  26. /***********************************/

  27. /* motion compensation */

  28. 

  29. #define QPEL_H264V_MM(A,B,C,D,E,F,OP,T,Z,d,q)\

  30.         "mov"#q" "#C", "#T"         \n\t"\

  31.         "mov"#d" (%0), "#F"         \n\t"\

  32.         "paddw "#D", "#T"           \n\t"\

  33.         "psllw $2, "#T"             \n\t"\

  34.         "psubw "#B", "#T"           \n\t"\

  35.         "psubw "#E", "#T"           \n\t"\

  36.         "punpcklbw "#Z", "#F"       \n\t"\

  37.         "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\

  38.         "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\

  39.         "add %2, %0                 \n\t"\

  40.         "paddw "#F", "#A"           \n\t"\

  41.         "paddw "#A", "#T"           \n\t"\

  42.         "psraw $5, "#T"             \n\t"\

  43.         "packuswb "#T", "#T"        \n\t"\

  44.         OP(T, (%1), A, d)\

  45.         "add %3, %1                 \n\t"

  46. 

  47. #define QPEL_H264HV_MM(A,B,C,D,E,F,OF,T,Z,d,q)\

  48.         "mov"#q" "#C", "#T"         \n\t"\

  49.         "mov"#d" (%0), "#F"         \n\t"\

  50.         "paddw "#D", "#T"           \n\t"\

  51.         "psllw $2, "#T"             \n\t"\

  52.         "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\

  53.         "psubw "#B", "#T"           \n\t"\

  54.         "psubw "#E", "#T"           \n\t"\

  55.         "punpcklbw "#Z", "#F"       \n\t"\

  56.         "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\

  57.         "paddw "#F", "#A"           \n\t"\

  58.         "add %2, %0                 \n\t"\

  59.         "paddw "#A", "#T"           \n\t"\

  60.         "mov"#q" "#T", "#OF"(%1)    \n\t"

  61. 

  62. #define QPEL_H264V(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%mm6,%%mm7,d,q)

  63. #define QPEL_H264HV(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%mm6,%%mm7,d,q)

  64. #define QPEL_H264V_XMM(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%xmm6,%%xmm7,q,dqa)

  65. #define QPEL_H264HV_XMM(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%xmm6,%%xmm7,q,dqa)

  66. 

  67. 

  68. #define QPEL_H264(OPNAME, OP, MMX)\

  69. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  70.     int h=4;\

  71. \

  72.     __asm__ volatile(\

  73.         "pxor %%mm7, %%mm7          \n\t"\

  74.         "movq "MANGLE(ff_pw_5) ", %%mm4\n\t"\

  75.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  76.         "1:                         \n\t"\

  77.         "movd  -1(%0), %%mm1        \n\t"\

  78.         "movd    (%0), %%mm2        \n\t"\

  79.         "movd   1(%0), %%mm3        \n\t"\

  80.         "movd   2(%0), %%mm0        \n\t"\

  81.         "punpcklbw %%mm7, %%mm1     \n\t"\

  82.         "punpcklbw %%mm7, %%mm2     \n\t"\

  83.         "punpcklbw %%mm7, %%mm3     \n\t"\

  84.         "punpcklbw %%mm7, %%mm0     \n\t"\

  85.         "paddw %%mm0, %%mm1         \n\t"\

  86.         "paddw %%mm3, %%mm2         \n\t"\

  87.         "movd  -2(%0), %%mm0        \n\t"\

  88.         "movd   3(%0), %%mm3        \n\t"\

  89.         "punpcklbw %%mm7, %%mm0     \n\t"\

  90.         "punpcklbw %%mm7, %%mm3     \n\t"\

  91.         "paddw %%mm3, %%mm0         \n\t"\

  92.         "psllw $2, %%mm2            \n\t"\

  93.         "psubw %%mm1, %%mm2         \n\t"\

  94.         "pmullw %%mm4, %%mm2        \n\t"\

  95.         "paddw %%mm5, %%mm0         \n\t"\

  96.         "paddw %%mm2, %%mm0         \n\t"\

  97.         "psraw $5, %%mm0            \n\t"\

  98.         "packuswb %%mm0, %%mm0      \n\t"\

  99.         OP(%%mm0, (%1),%%mm6, d)\

  100.         "add %3, %0                 \n\t"\

  101.         "add %4, %1                 \n\t"\

  102.         "decl %2                    \n\t"\

  103.         " jnz 1b                    \n\t"\

  104.         : "+a"(src), "+c"(dst), "+g"(h)\

  105.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  106.         : "memory"\

  107.     );\

  108. }\

  109. static av_noinline void OPNAME ## h264_qpel4_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  110.     int h=4;\

  111.     __asm__ volatile(\

  112.         "pxor %%mm7, %%mm7          \n\t"\

  113.         "movq %0, %%mm4             \n\t"\

  114.         "movq %1, %%mm5             \n\t"\

  115.         :: "m"(ff_pw_5), "m"(ff_pw_16)\

  116.     );\

  117.     do{\

  118.     __asm__ volatile(\

  119.         "movd  -1(%0), %%mm1        \n\t"\

  120.         "movd    (%0), %%mm2        \n\t"\

  121.         "movd   1(%0), %%mm3        \n\t"\

  122.         "movd   2(%0), %%mm0        \n\t"\

  123.         "punpcklbw %%mm7, %%mm1     \n\t"\

  124.         "punpcklbw %%mm7, %%mm2     \n\t"\

  125.         "punpcklbw %%mm7, %%mm3     \n\t"\

  126.         "punpcklbw %%mm7, %%mm0     \n\t"\

  127.         "paddw %%mm0, %%mm1         \n\t"\

  128.         "paddw %%mm3, %%mm2         \n\t"\

  129.         "movd  -2(%0), %%mm0        \n\t"\

  130.         "movd   3(%0), %%mm3        \n\t"\

  131.         "punpcklbw %%mm7, %%mm0     \n\t"\

  132.         "punpcklbw %%mm7, %%mm3     \n\t"\

  133.         "paddw %%mm3, %%mm0         \n\t"\

  134.         "psllw $2, %%mm2            \n\t"\

  135.         "psubw %%mm1, %%mm2         \n\t"\

  136.         "pmullw %%mm4, %%mm2        \n\t"\

  137.         "paddw %%mm5, %%mm0         \n\t"\

  138.         "paddw %%mm2, %%mm0         \n\t"\

  139.         "movd   (%2), %%mm3         \n\t"\

  140.         "psraw $5, %%mm0            \n\t"\

  141.         "packuswb %%mm0, %%mm0      \n\t"\

  142.         PAVGB" %%mm3, %%mm0         \n\t"\

  143.         OP(%%mm0, (%1),%%mm6, d)\

  144.         "add %4, %0                 \n\t"\

  145.         "add %4, %1                 \n\t"\

  146.         "add %3, %2                 \n\t"\

  147.         : "+a"(src), "+c"(dst), "+d"(src2)\

  148.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  149.         : "memory"\

  150.     );\

  151.     }while(--h);\

  152. }\

  153. static av_noinline void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  154.     src -= 2*srcStride;\

  155.     __asm__ volatile(\

  156.         "pxor %%mm7, %%mm7          \n\t"\

  157.         "movd (%0), %%mm0           \n\t"\

  158.         "add %2, %0                 \n\t"\

  159.         "movd (%0), %%mm1           \n\t"\

  160.         "add %2, %0                 \n\t"\

  161.         "movd (%0), %%mm2           \n\t"\

  162.         "add %2, %0                 \n\t"\

  163.         "movd (%0), %%mm3           \n\t"\

  164.         "add %2, %0                 \n\t"\

  165.         "movd (%0), %%mm4           \n\t"\

  166.         "add %2, %0                 \n\t"\

  167.         "punpcklbw %%mm7, %%mm0     \n\t"\

  168.         "punpcklbw %%mm7, %%mm1     \n\t"\

  169.         "punpcklbw %%mm7, %%mm2     \n\t"\

  170.         "punpcklbw %%mm7, %%mm3     \n\t"\

  171.         "punpcklbw %%mm7, %%mm4     \n\t"\

  172.         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  173.         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  174.         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  175.         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  176.          \

  177.         : "+a"(src), "+c"(dst)\

  178.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\

  179.         : "memory"\

  180.     );\

  181. }\

  182. static av_noinline void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  183.     int h=4;\

  184.     int w=3;\

  185.     src -= 2*srcStride+2;\

  186.     while(w--){\

  187.         __asm__ volatile(\

  188.             "pxor %%mm7, %%mm7      \n\t"\

  189.             "movd (%0), %%mm0       \n\t"\

  190.             "add %2, %0             \n\t"\

  191.             "movd (%0), %%mm1       \n\t"\

  192.             "add %2, %0             \n\t"\

  193.             "movd (%0), %%mm2       \n\t"\

  194.             "add %2, %0             \n\t"\

  195.             "movd (%0), %%mm3       \n\t"\

  196.             "add %2, %0             \n\t"\

  197.             "movd (%0), %%mm4       \n\t"\

  198.             "add %2, %0             \n\t"\

  199.             "punpcklbw %%mm7, %%mm0 \n\t"\

  200.             "punpcklbw %%mm7, %%mm1 \n\t"\

  201.             "punpcklbw %%mm7, %%mm2 \n\t"\

  202.             "punpcklbw %%mm7, %%mm3 \n\t"\

  203.             "punpcklbw %%mm7, %%mm4 \n\t"\

  204.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\

  205.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\

  206.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\

  207.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\

  208.              \

  209.             : "+a"(src)\

  210.             : "c"(tmp), "S"((x86_reg)srcStride)\

  211.             : "memory"\

  212.         );\

  213.         tmp += 4;\

  214.         src += 4 - 9*srcStride;\

  215.     }\

  216.     tmp -= 3*4;\

  217.     __asm__ volatile(\

  218.         "1:                         \n\t"\

  219.         "movq     (%0), %%mm0       \n\t"\

  220.         "paddw  10(%0), %%mm0       \n\t"\

  221.         "movq    2(%0), %%mm1       \n\t"\

  222.         "paddw   8(%0), %%mm1       \n\t"\

  223.         "movq    4(%0), %%mm2       \n\t"\

  224.         "paddw   6(%0), %%mm2       \n\t"\

  225.         "psubw %%mm1, %%mm0         \n\t"/*a-b   (abccba)*/\

  226.         "psraw $2, %%mm0            \n\t"/*(a-b)/4 */\

  227.         "psubw %%mm1, %%mm0         \n\t"/*(a-b)/4-b */\

  228.         "paddsw %%mm2, %%mm0        \n\t"\

  229.         "psraw $2, %%mm0            \n\t"/*((a-b)/4-b+c)/4 */\

  230.         "paddw %%mm2, %%mm0         \n\t"/*(a-5*b+20*c)/16 */\

  231.         "psraw $6, %%mm0            \n\t"\

  232.         "packuswb %%mm0, %%mm0      \n\t"\

  233.         OP(%%mm0, (%1),%%mm7, d)\

  234.         "add $24, %0                \n\t"\

  235.         "add %3, %1                 \n\t"\

  236.         "decl %2                    \n\t"\

  237.         " jnz 1b                    \n\t"\

  238.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  239.         : "S"((x86_reg)dstStride)\

  240.         : "memory"\

  241.     );\

  242. }\

  243. \

  244. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  245.     int h=8;\

  246.     __asm__ volatile(\

  247.         "pxor %%mm7, %%mm7          \n\t"\

  248.         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\

  249.         "1:                         \n\t"\

  250.         "movq    (%0), %%mm0        \n\t"\

  251.         "movq   1(%0), %%mm2        \n\t"\

  252.         "movq %%mm0, %%mm1          \n\t"\

  253.         "movq %%mm2, %%mm3          \n\t"\

  254.         "punpcklbw %%mm7, %%mm0     \n\t"\

  255.         "punpckhbw %%mm7, %%mm1     \n\t"\

  256.         "punpcklbw %%mm7, %%mm2     \n\t"\

  257.         "punpckhbw %%mm7, %%mm3     \n\t"\

  258.         "paddw %%mm2, %%mm0         \n\t"\

  259.         "paddw %%mm3, %%mm1         \n\t"\

  260.         "psllw $2, %%mm0            \n\t"\

  261.         "psllw $2, %%mm1            \n\t"\

  262.         "movq   -1(%0), %%mm2       \n\t"\

  263.         "movq    2(%0), %%mm4       \n\t"\

  264.         "movq %%mm2, %%mm3          \n\t"\

  265.         "movq %%mm4, %%mm5          \n\t"\

  266.         "punpcklbw %%mm7, %%mm2     \n\t"\

  267.         "punpckhbw %%mm7, %%mm3     \n\t"\

  268.         "punpcklbw %%mm7, %%mm4     \n\t"\

  269.         "punpckhbw %%mm7, %%mm5     \n\t"\

  270.         "paddw %%mm4, %%mm2         \n\t"\

  271.         "paddw %%mm3, %%mm5         \n\t"\

  272.         "psubw %%mm2, %%mm0         \n\t"\

  273.         "psubw %%mm5, %%mm1         \n\t"\

  274.         "pmullw %%mm6, %%mm0        \n\t"\

  275.         "pmullw %%mm6, %%mm1        \n\t"\

  276.         "movd   -2(%0), %%mm2       \n\t"\

  277.         "movd    7(%0), %%mm5       \n\t"\

  278.         "punpcklbw %%mm7, %%mm2     \n\t"\

  279.         "punpcklbw %%mm7, %%mm5     \n\t"\

  280.         "paddw %%mm3, %%mm2         \n\t"\

  281.         "paddw %%mm5, %%mm4         \n\t"\

  282.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  283.         "paddw %%mm5, %%mm2         \n\t"\

  284.         "paddw %%mm5, %%mm4         \n\t"\

  285.         "paddw %%mm2, %%mm0         \n\t"\

  286.         "paddw %%mm4, %%mm1         \n\t"\

  287.         "psraw $5, %%mm0            \n\t"\

  288.         "psraw $5, %%mm1            \n\t"\

  289.         "packuswb %%mm1, %%mm0      \n\t"\

  290.         OP(%%mm0, (%1),%%mm5, q)\

  291.         "add %3, %0                 \n\t"\

  292.         "add %4, %1                 \n\t"\

  293.         "decl %2                    \n\t"\

  294.         " jnz 1b                    \n\t"\

  295.         : "+a"(src), "+c"(dst), "+g"(h)\

  296.         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  297.         : "memory"\

  298.     );\

  299. }\

  300. \

  301. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  302.     int h=8;\

  303.     __asm__ volatile(\

  304.         "pxor %%mm7, %%mm7          \n\t"\

  305.         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\

  306.         "1:                         \n\t"\

  307.         "movq    (%0), %%mm0        \n\t"\

  308.         "movq   1(%0), %%mm2        \n\t"\

  309.         "movq %%mm0, %%mm1          \n\t"\

  310.         "movq %%mm2, %%mm3          \n\t"\

  311.         "punpcklbw %%mm7, %%mm0     \n\t"\

  312.         "punpckhbw %%mm7, %%mm1     \n\t"\

  313.         "punpcklbw %%mm7, %%mm2     \n\t"\

  314.         "punpckhbw %%mm7, %%mm3     \n\t"\

  315.         "paddw %%mm2, %%mm0         \n\t"\

  316.         "paddw %%mm3, %%mm1         \n\t"\

  317.         "psllw $2, %%mm0            \n\t"\

  318.         "psllw $2, %%mm1            \n\t"\

  319.         "movq   -1(%0), %%mm2       \n\t"\

  320.         "movq    2(%0), %%mm4       \n\t"\

  321.         "movq %%mm2, %%mm3          \n\t"\

  322.         "movq %%mm4, %%mm5          \n\t"\

  323.         "punpcklbw %%mm7, %%mm2     \n\t"\

  324.         "punpckhbw %%mm7, %%mm3     \n\t"\

  325.         "punpcklbw %%mm7, %%mm4     \n\t"\

  326.         "punpckhbw %%mm7, %%mm5     \n\t"\

  327.         "paddw %%mm4, %%mm2         \n\t"\

  328.         "paddw %%mm3, %%mm5         \n\t"\

  329.         "psubw %%mm2, %%mm0         \n\t"\

  330.         "psubw %%mm5, %%mm1         \n\t"\

  331.         "pmullw %%mm6, %%mm0        \n\t"\

  332.         "pmullw %%mm6, %%mm1        \n\t"\

  333.         "movd   -2(%0), %%mm2       \n\t"\

  334.         "movd    7(%0), %%mm5       \n\t"\

  335.         "punpcklbw %%mm7, %%mm2     \n\t"\

  336.         "punpcklbw %%mm7, %%mm5     \n\t"\

  337.         "paddw %%mm3, %%mm2         \n\t"\

  338.         "paddw %%mm5, %%mm4         \n\t"\

  339.         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\

  340.         "paddw %%mm5, %%mm2         \n\t"\

  341.         "paddw %%mm5, %%mm4         \n\t"\

  342.         "paddw %%mm2, %%mm0         \n\t"\

  343.         "paddw %%mm4, %%mm1         \n\t"\

  344.         "psraw $5, %%mm0            \n\t"\

  345.         "psraw $5, %%mm1            \n\t"\

  346.         "movq (%2), %%mm4           \n\t"\

  347.         "packuswb %%mm1, %%mm0      \n\t"\

  348.         PAVGB" %%mm4, %%mm0         \n\t"\

  349.         OP(%%mm0, (%1),%%mm5, q)\

  350.         "add %5, %0                 \n\t"\

  351.         "add %5, %1                 \n\t"\

  352.         "add %4, %2                 \n\t"\

  353.         "decl %3                    \n\t"\

  354.         "jg 1b                      \n\t"\

  355.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  356.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  357.         : "memory"\

  358.     );\

  359. }\

  360. \

  361. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  362.     int w= 2;\

  363.     src -= 2*srcStride;\

  364.     \

  365.     while(w--){\

  366.         __asm__ volatile(\

  367.             "pxor %%mm7, %%mm7          \n\t"\

  368.             "movd (%0), %%mm0           \n\t"\

  369.             "add %2, %0                 \n\t"\

  370.             "movd (%0), %%mm1           \n\t"\

  371.             "add %2, %0                 \n\t"\

  372.             "movd (%0), %%mm2           \n\t"\

  373.             "add %2, %0                 \n\t"\

  374.             "movd (%0), %%mm3           \n\t"\

  375.             "add %2, %0                 \n\t"\

  376.             "movd (%0), %%mm4           \n\t"\

  377.             "add %2, %0                 \n\t"\

  378.             "punpcklbw %%mm7, %%mm0     \n\t"\

  379.             "punpcklbw %%mm7, %%mm1     \n\t"\

  380.             "punpcklbw %%mm7, %%mm2     \n\t"\

  381.             "punpcklbw %%mm7, %%mm3     \n\t"\

  382.             "punpcklbw %%mm7, %%mm4     \n\t"\

  383.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  384.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  385.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  386.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  387.             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  388.             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  389.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  390.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  391.             "cmpl $16, %4               \n\t"\

  392.             "jne 2f                     \n\t"\

  393.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  394.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  395.             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\

  396.             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\

  397.             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\

  398.             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\

  399.             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\

  400.             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\

  401.             "2:                         \n\t"\

  402.             \

  403.             : "+a"(src), "+c"(dst)\

  404.             : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\

  405.             : "memory"\

  406.         );\

  407.         src += 4-(h+5)*srcStride;\

  408.         dst += 4-h*dstStride;\

  409.     }\

  410. }\

  411. static av_always_inline void OPNAME ## h264_qpel8or16_hv1_lowpass_ ## MMX(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){\

  412.     int w = (size+8)>>2;\

  413.     src -= 2*srcStride+2;\

  414.     while(w--){\

  415.         __asm__ volatile(\

  416.             "pxor %%mm7, %%mm7      \n\t"\

  417.             "movd (%0), %%mm0       \n\t"\

  418.             "add %2, %0             \n\t"\

  419.             "movd (%0), %%mm1       \n\t"\

  420.             "add %2, %0             \n\t"\

  421.             "movd (%0), %%mm2       \n\t"\

  422.             "add %2, %0             \n\t"\

  423.             "movd (%0), %%mm3       \n\t"\

  424.             "add %2, %0             \n\t"\

  425.             "movd (%0), %%mm4       \n\t"\

  426.             "add %2, %0             \n\t"\

  427.             "punpcklbw %%mm7, %%mm0 \n\t"\

  428.             "punpcklbw %%mm7, %%mm1 \n\t"\

  429.             "punpcklbw %%mm7, %%mm2 \n\t"\

  430.             "punpcklbw %%mm7, %%mm3 \n\t"\

  431.             "punpcklbw %%mm7, %%mm4 \n\t"\

  432.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\

  433.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\

  434.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\

  435.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\

  436.             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\

  437.             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\

  438.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\

  439.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\

  440.             "cmpl $16, %3           \n\t"\

  441.             "jne 2f                 \n\t"\

  442.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1,  8*48)\

  443.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2,  9*48)\

  444.             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\

  445.             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\

  446.             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\

  447.             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\

  448.             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\

  449.             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\

  450.             "2:                     \n\t"\

  451.             : "+a"(src)\

  452.             : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size)\

  453.             : "memory"\

  454.             );\

  455.         tmp += 4;\

  456.         src += 4 - (size+5)*srcStride;\

  457.     }\

  458. }\

  459. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  460.     int w = size>>4;\

  461.     do{\

  462.     int h = size;\

  463.     __asm__ volatile(\

  464.         "1:                         \n\t"\

  465.         "movq     (%0), %%mm0       \n\t"\

  466.         "movq    8(%0), %%mm3       \n\t"\

  467.         "movq    2(%0), %%mm1       \n\t"\

  468.         "movq   10(%0), %%mm4       \n\t"\

  469.         "paddw   %%mm4, %%mm0       \n\t"\

  470.         "paddw   %%mm3, %%mm1       \n\t"\

  471.         "paddw  18(%0), %%mm3       \n\t"\

  472.         "paddw  16(%0), %%mm4       \n\t"\

  473.         "movq    4(%0), %%mm2       \n\t"\

  474.         "movq   12(%0), %%mm5       \n\t"\

  475.         "paddw   6(%0), %%mm2       \n\t"\

  476.         "paddw  14(%0), %%mm5       \n\t"\

  477.         "psubw %%mm1, %%mm0         \n\t"\

  478.         "psubw %%mm4, %%mm3         \n\t"\

  479.         "psraw $2, %%mm0            \n\t"\

  480.         "psraw $2, %%mm3            \n\t"\

  481.         "psubw %%mm1, %%mm0         \n\t"\

  482.         "psubw %%mm4, %%mm3         \n\t"\

  483.         "paddsw %%mm2, %%mm0        \n\t"\

  484.         "paddsw %%mm5, %%mm3        \n\t"\

  485.         "psraw $2, %%mm0            \n\t"\

  486.         "psraw $2, %%mm3            \n\t"\

  487.         "paddw %%mm2, %%mm0         \n\t"\

  488.         "paddw %%mm5, %%mm3         \n\t"\

  489.         "psraw $6, %%mm0            \n\t"\

  490.         "psraw $6, %%mm3            \n\t"\

  491.         "packuswb %%mm3, %%mm0      \n\t"\

  492.         OP(%%mm0, (%1),%%mm7, q)\

  493.         "add $48, %0                \n\t"\

  494.         "add %3, %1                 \n\t"\

  495.         "decl %2                    \n\t"\

  496.         " jnz 1b                    \n\t"\

  497.         : "+a"(tmp), "+c"(dst), "+g"(h)\

  498.         : "S"((x86_reg)dstStride)\

  499.         : "memory"\

  500.     );\

  501.     tmp += 8 - size*24;\

  502.     dst += 8 - size*dstStride;\

  503.     }while(w--);\

  504. }\

  505. \

  506. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  507.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  508. }\

  509. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  510.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  511.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  512. }\

  513. \

  514. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  515.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  516.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  517.     src += 8*srcStride;\

  518.     dst += 8*dstStride;\

  519.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  520.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  521. }\

  522. \

  523. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  524.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  525.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  526.     src += 8*dstStride;\

  527.     dst += 8*dstStride;\

  528.     src2 += 8*src2Stride;\

  529.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  530.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  531. }\

  532. \

  533. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  534.           put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\

  535.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  536. }\

  537. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  538.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 8);\

  539. }\

  540. \

  541. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  542.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 16);\

  543. }\

  544. \

  545. static av_noinline void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  546. {\

  547.     __asm__ volatile(\

  548.         "movq      (%1), %%mm0          \n\t"\

  549.         "movq    24(%1), %%mm1          \n\t"\

  550.         "psraw      $5,  %%mm0          \n\t"\

  551.         "psraw      $5,  %%mm1          \n\t"\

  552.         "packuswb %%mm0, %%mm0          \n\t"\

  553.         "packuswb %%mm1, %%mm1          \n\t"\

  554.         PAVGB"     (%0), %%mm0          \n\t"\

  555.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  556.         OP(%%mm0, (%2),    %%mm4, d)\

  557.         OP(%%mm1, (%2,%4), %%mm5, d)\

  558.         "lea  (%0,%3,2), %0             \n\t"\

  559.         "lea  (%2,%4,2), %2             \n\t"\

  560.         "movq    48(%1), %%mm0          \n\t"\

  561.         "movq    72(%1), %%mm1          \n\t"\

  562.         "psraw      $5,  %%mm0          \n\t"\

  563.         "psraw      $5,  %%mm1          \n\t"\

  564.         "packuswb %%mm0, %%mm0          \n\t"\

  565.         "packuswb %%mm1, %%mm1          \n\t"\

  566.         PAVGB"     (%0), %%mm0          \n\t"\

  567.         PAVGB"  (%0,%3), %%mm1          \n\t"\

  568.         OP(%%mm0, (%2),    %%mm4, d)\

  569.         OP(%%mm1, (%2,%4), %%mm5, d)\

  570.         :"+a"(src8), "+c"(src16), "+d"(dst)\

  571.         :"S"((x86_reg)src8Stride), "D"((x86_reg)dstStride)\

  572.         :"memory");\

  573. }\

  574. static av_noinline void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  575. {\

  576.     do{\

  577.     __asm__ volatile(\

  578.         "movq      (%1), %%mm0          \n\t"\

  579.         "movq     8(%1), %%mm1          \n\t"\

  580.         "movq    48(%1), %%mm2          \n\t"\

  581.         "movq  8+48(%1), %%mm3          \n\t"\

  582.         "psraw      $5,  %%mm0          \n\t"\

  583.         "psraw      $5,  %%mm1          \n\t"\

  584.         "psraw      $5,  %%mm2          \n\t"\

  585.         "psraw      $5,  %%mm3          \n\t"\

  586.         "packuswb %%mm1, %%mm0          \n\t"\

  587.         "packuswb %%mm3, %%mm2          \n\t"\

  588.         PAVGB"     (%0), %%mm0          \n\t"\

  589.         PAVGB"  (%0,%3), %%mm2          \n\t"\

  590.         OP(%%mm0, (%2), %%mm5, q)\

  591.         OP(%%mm2, (%2,%4), %%mm5, q)\

  592.         ::"a"(src8), "c"(src16), "d"(dst),\

  593.           "r"((x86_reg)src8Stride), "r"((x86_reg)dstStride)\

  594.         :"memory");\

  595.         src8 += 2L*src8Stride;\

  596.         src16 += 48;\

  597.         dst += 2L*dstStride;\

  598.     }while(h-=2);\

  599. }\

  600. static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\

  601. {\

  602.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst  , src16  , src8  , dstStride, src8Stride, h);\

  603.     OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\

  604. }\

  605. 

  606. 

  607. #if ARCH_X86_64

  608. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  609. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  610.     int h=16;\

  611.     __asm__ volatile(\

  612.         "pxor %%xmm15, %%xmm15      \n\t"\

  613.         "movdqa %6, %%xmm14         \n\t"\

  614.         "movdqa %7, %%xmm13         \n\t"\

  615.         "1:                         \n\t"\

  616.         "lddqu    6(%0), %%xmm1     \n\t"\

  617.         "lddqu   -2(%0), %%xmm7     \n\t"\

  618.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  619.         "punpckhbw %%xmm15, %%xmm1  \n\t"\

  620.         "punpcklbw %%xmm15, %%xmm0  \n\t"\

  621.         "punpcklbw %%xmm15, %%xmm7  \n\t"\

  622.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  623.         "movdqa  %%xmm0, %%xmm6     \n\t"\

  624.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  625.         "movdqa  %%xmm0, %%xmm8     \n\t"\

  626.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  627.         "movdqa  %%xmm0, %%xmm9     \n\t"\

  628.         "movdqa  %%xmm0, %%xmm12    \n\t"\

  629.         "movdqa  %%xmm1, %%xmm11    \n\t"\

  630.         "palignr $10,%%xmm0, %%xmm11\n\t"\

  631.         "palignr $10,%%xmm7, %%xmm12\n\t"\

  632.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  633.         "palignr $2, %%xmm7, %%xmm9 \n\t"\

  634.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  635.         "palignr $4, %%xmm7, %%xmm8 \n\t"\

  636.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  637.         "palignr $6, %%xmm7, %%xmm6 \n\t"\

  638.         "paddw   %%xmm0 ,%%xmm11    \n\t"\

  639.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  640.         "palignr $8, %%xmm7, %%xmm0 \n\t"\

  641.         "paddw   %%xmm12,%%xmm7     \n\t"\

  642.         "paddw   %%xmm3, %%xmm2     \n\t"\

  643.         "paddw   %%xmm8, %%xmm6     \n\t"\

  644.         "paddw   %%xmm4, %%xmm1     \n\t"\

  645.         "paddw   %%xmm9, %%xmm0     \n\t"\

  646.         "psllw   $2,     %%xmm2     \n\t"\

  647.         "psllw   $2,     %%xmm6     \n\t"\

  648.         "psubw   %%xmm1, %%xmm2     \n\t"\

  649.         "psubw   %%xmm0, %%xmm6     \n\t"\

  650.         "paddw   %%xmm13,%%xmm11    \n\t"\

  651.         "paddw   %%xmm13,%%xmm7     \n\t"\

  652.         "pmullw  %%xmm14,%%xmm2     \n\t"\

  653.         "pmullw  %%xmm14,%%xmm6     \n\t"\

  654.         "lddqu   (%2),   %%xmm3     \n\t"\

  655.         "paddw   %%xmm11,%%xmm2     \n\t"\

  656.         "paddw   %%xmm7, %%xmm6     \n\t"\

  657.         "psraw   $5,     %%xmm2     \n\t"\

  658.         "psraw   $5,     %%xmm6     \n\t"\

  659.         "packuswb %%xmm2,%%xmm6     \n\t"\

  660.         "pavgb   %%xmm3, %%xmm6     \n\t"\

  661.         OP(%%xmm6, (%1), %%xmm4, dqa)\

  662.         "add %5, %0                 \n\t"\

  663.         "add %5, %1                 \n\t"\

  664.         "add %4, %2                 \n\t"\

  665.         "decl %3                    \n\t"\

  666.         "jg 1b                      \n\t"\

  667.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  668.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\

  669.           "m"(ff_pw_5), "m"(ff_pw_16)\

  670.         : XMM_CLOBBERS("%xmm0" , "%xmm1" , "%xmm2" , "%xmm3" , \

  671.                        "%xmm4" , "%xmm5" , "%xmm6" , "%xmm7" , \

  672.                        "%xmm8" , "%xmm9" , "%xmm10", "%xmm11", \

  673.                        "%xmm12", "%xmm13", "%xmm14", "%xmm15",)\

  674.           "memory"\

  675.     );\

  676. }

  677. #else // ARCH_X86_64

  678. #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  679. static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  680.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  681.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  682.     src += 8*dstStride;\

  683.     dst += 8*dstStride;\

  684.     src2 += 8*src2Stride;\

  685.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\

  686.     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\

  687. }

  688. #endif // ARCH_X86_64

  689. 

  690. #define QPEL_H264_H_XMM(OPNAME, OP, MMX)\

  691. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\

  692.     int h=8;\

  693.     __asm__ volatile(\

  694.         "pxor %%xmm7, %%xmm7        \n\t"\

  695.         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\

  696.         "1:                         \n\t"\

  697.         "lddqu   -2(%0), %%xmm1     \n\t"\

  698.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  699.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  700.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  701.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  702.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  703.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  704.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  705.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  706.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  707.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  708.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  709.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  710.         "paddw   %%xmm5, %%xmm0     \n\t"\

  711.         "paddw   %%xmm3, %%xmm2     \n\t"\

  712.         "paddw   %%xmm4, %%xmm1     \n\t"\

  713.         "psllw   $2,     %%xmm2     \n\t"\

  714.         "movq    (%2),   %%xmm3     \n\t"\

  715.         "psubw   %%xmm1, %%xmm2     \n\t"\

  716.         "paddw "MANGLE(ff_pw_16)", %%xmm0\n\t"\

  717.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  718.         "paddw   %%xmm0, %%xmm2     \n\t"\

  719.         "psraw   $5,     %%xmm2     \n\t"\

  720.         "packuswb %%xmm2, %%xmm2    \n\t"\

  721.         "pavgb   %%xmm3, %%xmm2     \n\t"\

  722.         OP(%%xmm2, (%1), %%xmm4, q)\

  723.         "add %5, %0                 \n\t"\

  724.         "add %5, %1                 \n\t"\

  725.         "add %4, %2                 \n\t"\

  726.         "decl %3                    \n\t"\

  727.         "jg 1b                      \n\t"\

  728.         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\

  729.         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\

  730.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  731.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  732.           "memory"\

  733.     );\

  734. }\

  735. QPEL_H264_H16_XMM(OPNAME, OP, MMX)\

  736. \

  737. static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  738.     int h=8;\

  739.     __asm__ volatile(\

  740.         "pxor %%xmm7, %%xmm7        \n\t"\

  741.         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\

  742.         "1:                         \n\t"\

  743.         "lddqu   -2(%0), %%xmm1     \n\t"\

  744.         "movdqa  %%xmm1, %%xmm0     \n\t"\

  745.         "punpckhbw %%xmm7, %%xmm1   \n\t"\

  746.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  747.         "movdqa  %%xmm1, %%xmm2     \n\t"\

  748.         "movdqa  %%xmm1, %%xmm3     \n\t"\

  749.         "movdqa  %%xmm1, %%xmm4     \n\t"\

  750.         "movdqa  %%xmm1, %%xmm5     \n\t"\

  751.         "palignr $2, %%xmm0, %%xmm4 \n\t"\

  752.         "palignr $4, %%xmm0, %%xmm3 \n\t"\

  753.         "palignr $6, %%xmm0, %%xmm2 \n\t"\

  754.         "palignr $8, %%xmm0, %%xmm1 \n\t"\

  755.         "palignr $10,%%xmm0, %%xmm5 \n\t"\

  756.         "paddw   %%xmm5, %%xmm0     \n\t"\

  757.         "paddw   %%xmm3, %%xmm2     \n\t"\

  758.         "paddw   %%xmm4, %%xmm1     \n\t"\

  759.         "psllw   $2,     %%xmm2     \n\t"\

  760.         "psubw   %%xmm1, %%xmm2     \n\t"\

  761.         "paddw   "MANGLE(ff_pw_16)", %%xmm0\n\t"\

  762.         "pmullw  %%xmm6, %%xmm2     \n\t"\

  763.         "paddw   %%xmm0, %%xmm2     \n\t"\

  764.         "psraw   $5,     %%xmm2     \n\t"\

  765.         "packuswb %%xmm2, %%xmm2    \n\t"\

  766.         OP(%%xmm2, (%1), %%xmm4, q)\

  767.         "add %3, %0                 \n\t"\

  768.         "add %4, %1                 \n\t"\

  769.         "decl %2                    \n\t"\

  770.         " jnz 1b                    \n\t"\

  771.         : "+a"(src), "+c"(dst), "+g"(h)\

  772.         : "D"((x86_reg)srcStride), "S"((x86_reg)dstStride)\

  773.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  774.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  775.           "memory"\

  776.     );\

  777. }\

  778. static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  779.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  780.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  781.     src += 8*srcStride;\

  782.     dst += 8*dstStride;\

  783.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\

  784.     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\

  785. }\

  786. 

  787. #define QPEL_H264_V_XMM(OPNAME, OP, MMX)\

  788. static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\

  789.     src -= 2*srcStride;\

  790.     \

  791.     __asm__ volatile(\

  792.         "pxor %%xmm7, %%xmm7        \n\t"\

  793.         "movq (%0), %%xmm0          \n\t"\

  794.         "add %2, %0                 \n\t"\

  795.         "movq (%0), %%xmm1          \n\t"\

  796.         "add %2, %0                 \n\t"\

  797.         "movq (%0), %%xmm2          \n\t"\

  798.         "add %2, %0                 \n\t"\

  799.         "movq (%0), %%xmm3          \n\t"\

  800.         "add %2, %0                 \n\t"\

  801.         "movq (%0), %%xmm4          \n\t"\

  802.         "add %2, %0                 \n\t"\

  803.         "punpcklbw %%xmm7, %%xmm0   \n\t"\

  804.         "punpcklbw %%xmm7, %%xmm1   \n\t"\

  805.         "punpcklbw %%xmm7, %%xmm2   \n\t"\

  806.         "punpcklbw %%xmm7, %%xmm3   \n\t"\

  807.         "punpcklbw %%xmm7, %%xmm4   \n\t"\

  808.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  809.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  810.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  811.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  812.         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  813.         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  814.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  815.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  816.         "cmpl $16, %4               \n\t"\

  817.         "jne 2f                     \n\t"\

  818.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  819.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  820.         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\

  821.         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\

  822.         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\

  823.         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\

  824.         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\

  825.         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\

  826.         "2:                          \n\t"\

  827.         \

  828.         : "+a"(src), "+c"(dst)\

  829.         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\

  830.         : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  831.                        "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  832.           "memory"\

  833.     );\

  834. }\

  835. static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  836.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\

  837. }\

  838. static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\

  839.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\

  840.     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\

  841. }

  842. 

  843. static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){

  844.     int w = (size+8)>>3;

  845.     src -= 2*srcStride+2;

  846.     while(w--){

  847.         __asm__ volatile(

  848.             "pxor %%xmm7, %%xmm7        \n\t"

  849.             "movq (%0), %%xmm0          \n\t"

  850.             "add %2, %0                 \n\t"

  851.             "movq (%0), %%xmm1          \n\t"

  852.             "add %2, %0                 \n\t"

  853.             "movq (%0), %%xmm2          \n\t"

  854.             "add %2, %0                 \n\t"

  855.             "movq (%0), %%xmm3          \n\t"

  856.             "add %2, %0                 \n\t"

  857.             "movq (%0), %%xmm4          \n\t"

  858.             "add %2, %0                 \n\t"

  859.             "punpcklbw %%xmm7, %%xmm0   \n\t"

  860.             "punpcklbw %%xmm7, %%xmm1   \n\t"

  861.             "punpcklbw %%xmm7, %%xmm2   \n\t"

  862.             "punpcklbw %%xmm7, %%xmm3   \n\t"

  863.             "punpcklbw %%xmm7, %%xmm4   \n\t"

  864.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 0*48)

  865.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 1*48)

  866.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 2*48)

  867.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 3*48)

  868.             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 4*48)

  869.             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 5*48)

  870.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 6*48)

  871.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 7*48)

  872.             "cmpl $16, %3               \n\t"

  873.             "jne 2f                     \n\t"

  874.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1,  8*48)

  875.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2,  9*48)

  876.             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 10*48)

  877.             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 11*48)

  878.             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 12*48)

  879.             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 13*48)

  880.             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 14*48)

  881.             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 15*48)

  882.             "2:                         \n\t"

  883.             : "+a"(src)

  884.             : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size)

  885.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3",

  886.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)

  887.               "memory"

  888.         );

  889.         tmp += 8;

  890.         src += 8 - (size+5)*srcStride;

  891.     }

  892. }

  893. 

  894. #define QPEL_H264_HV2_XMM(OPNAME, OP, MMX)\

  895. static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\

  896.     int h = size;\

  897.     if(size == 16){\

  898.         __asm__ volatile(\

  899.             "1:                         \n\t"\

  900.             "movdqa 32(%0), %%xmm4      \n\t"\

  901.             "movdqa 16(%0), %%xmm5      \n\t"\

  902.             "movdqa   (%0), %%xmm7      \n\t"\

  903.             "movdqa %%xmm4, %%xmm3      \n\t"\

  904.             "movdqa %%xmm4, %%xmm2      \n\t"\

  905.             "movdqa %%xmm4, %%xmm1      \n\t"\

  906.             "movdqa %%xmm4, %%xmm0      \n\t"\

  907.             "palignr $10, %%xmm5, %%xmm0 \n\t"\

  908.             "palignr  $8, %%xmm5, %%xmm1 \n\t"\

  909.             "palignr  $6, %%xmm5, %%xmm2 \n\t"\

  910.             "palignr  $4, %%xmm5, %%xmm3 \n\t"\

  911.             "palignr  $2, %%xmm5, %%xmm4 \n\t"\

  912.             "paddw  %%xmm5, %%xmm0      \n\t"\

  913.             "paddw  %%xmm4, %%xmm1      \n\t"\

  914.             "paddw  %%xmm3, %%xmm2      \n\t"\

  915.             "movdqa %%xmm5, %%xmm6      \n\t"\

  916.             "movdqa %%xmm5, %%xmm4      \n\t"\

  917.             "movdqa %%xmm5, %%xmm3      \n\t"\

  918.             "palignr  $8, %%xmm7, %%xmm4 \n\t"\

  919.             "palignr  $2, %%xmm7, %%xmm6 \n\t"\

  920.             "palignr $10, %%xmm7, %%xmm3 \n\t"\

  921.             "paddw  %%xmm6, %%xmm4      \n\t"\

  922.             "movdqa %%xmm5, %%xmm6      \n\t"\

  923.             "palignr  $6, %%xmm7, %%xmm5 \n\t"\

  924.             "palignr  $4, %%xmm7, %%xmm6 \n\t"\

  925.             "paddw  %%xmm7, %%xmm3      \n\t"\

  926.             "paddw  %%xmm6, %%xmm5      \n\t"\

  927.             \

  928.             "psubw  %%xmm1, %%xmm0      \n\t"\

  929.             "psubw  %%xmm4, %%xmm3      \n\t"\

  930.             "psraw      $2, %%xmm0      \n\t"\

  931.             "psraw      $2, %%xmm3      \n\t"\

  932.             "psubw  %%xmm1, %%xmm0      \n\t"\

  933.             "psubw  %%xmm4, %%xmm3      \n\t"\

  934.             "paddw  %%xmm2, %%xmm0      \n\t"\

  935.             "paddw  %%xmm5, %%xmm3      \n\t"\

  936.             "psraw      $2, %%xmm0      \n\t"\

  937.             "psraw      $2, %%xmm3      \n\t"\

  938.             "paddw  %%xmm2, %%xmm0      \n\t"\

  939.             "paddw  %%xmm5, %%xmm3      \n\t"\

  940.             "psraw      $6, %%xmm0      \n\t"\

  941.             "psraw      $6, %%xmm3      \n\t"\

  942.             "packuswb %%xmm0, %%xmm3    \n\t"\

  943.             OP(%%xmm3, (%1), %%xmm7, dqa)\

  944.             "add $48, %0                \n\t"\

  945.             "add %3, %1                 \n\t"\

  946.             "decl %2                    \n\t"\

  947.             " jnz 1b                    \n\t"\

  948.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  949.             : "S"((x86_reg)dstStride)\

  950.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  951.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  952.               "memory"\

  953.         );\

  954.     }else{\

  955.         __asm__ volatile(\

  956.             "1:                         \n\t"\

  957.             "movdqa 16(%0), %%xmm1      \n\t"\

  958.             "movdqa   (%0), %%xmm0      \n\t"\

  959.             "movdqa %%xmm1, %%xmm2      \n\t"\

  960.             "movdqa %%xmm1, %%xmm3      \n\t"\

  961.             "movdqa %%xmm1, %%xmm4      \n\t"\

  962.             "movdqa %%xmm1, %%xmm5      \n\t"\

  963.             "palignr $10, %%xmm0, %%xmm5 \n\t"\

  964.             "palignr  $8, %%xmm0, %%xmm4 \n\t"\

  965.             "palignr  $6, %%xmm0, %%xmm3 \n\t"\

  966.             "palignr  $4, %%xmm0, %%xmm2 \n\t"\

  967.             "palignr  $2, %%xmm0, %%xmm1 \n\t"\

  968.             "paddw  %%xmm5, %%xmm0      \n\t"\

  969.             "paddw  %%xmm4, %%xmm1      \n\t"\

  970.             "paddw  %%xmm3, %%xmm2      \n\t"\

  971.             "psubw  %%xmm1, %%xmm0      \n\t"\

  972.             "psraw      $2, %%xmm0      \n\t"\

  973.             "psubw  %%xmm1, %%xmm0      \n\t"\

  974.             "paddw  %%xmm2, %%xmm0      \n\t"\

  975.             "psraw      $2, %%xmm0      \n\t"\

  976.             "paddw  %%xmm2, %%xmm0      \n\t"\

  977.             "psraw      $6, %%xmm0      \n\t"\

  978.             "packuswb %%xmm0, %%xmm0    \n\t"\

  979.             OP(%%xmm0, (%1), %%xmm7, q)\

  980.             "add $48, %0                \n\t"\

  981.             "add %3, %1                 \n\t"\

  982.             "decl %2                    \n\t"\

  983.             " jnz 1b                    \n\t"\

  984.             : "+a"(tmp), "+c"(dst), "+g"(h)\

  985.             : "S"((x86_reg)dstStride)\

  986.             : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \

  987.                            "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\

  988.               "memory"\

  989.         );\

  990.     }\

  991. }

  992. 

  993. #define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\

  994. static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\

  995.           put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\

  996.     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\

  997. }\

  998. static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  999.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\

  1000. }\

  1001. static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\

  1002.     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\

  1003. }\

  1004. 

  1005. #define put_pixels8_l2_sse2 put_pixels8_l2_mmx2

  1006. #define avg_pixels8_l2_sse2 avg_pixels8_l2_mmx2

  1007. #define put_pixels16_l2_sse2 put_pixels16_l2_mmx2

  1008. #define avg_pixels16_l2_sse2 avg_pixels16_l2_mmx2

  1009. #define put_pixels8_l2_ssse3 put_pixels8_l2_mmx2

  1010. #define avg_pixels8_l2_ssse3 avg_pixels8_l2_mmx2

  1011. #define put_pixels16_l2_ssse3 put_pixels16_l2_mmx2

  1012. #define avg_pixels16_l2_ssse3 avg_pixels16_l2_mmx2

  1013. 

  1014. #define put_pixels8_l2_shift5_sse2 put_pixels8_l2_shift5_mmx2

  1015. #define avg_pixels8_l2_shift5_sse2 avg_pixels8_l2_shift5_mmx2

  1016. #define put_pixels16_l2_shift5_sse2 put_pixels16_l2_shift5_mmx2

  1017. #define avg_pixels16_l2_shift5_sse2 avg_pixels16_l2_shift5_mmx2

  1018. #define put_pixels8_l2_shift5_ssse3 put_pixels8_l2_shift5_mmx2

  1019. #define avg_pixels8_l2_shift5_ssse3 avg_pixels8_l2_shift5_mmx2

  1020. #define put_pixels16_l2_shift5_ssse3 put_pixels16_l2_shift5_mmx2

  1021. #define avg_pixels16_l2_shift5_ssse3 avg_pixels16_l2_shift5_mmx2

  1022. 

  1023. #define put_h264_qpel8_h_lowpass_l2_sse2 put_h264_qpel8_h_lowpass_l2_mmx2

  1024. #define avg_h264_qpel8_h_lowpass_l2_sse2 avg_h264_qpel8_h_lowpass_l2_mmx2

  1025. #define put_h264_qpel16_h_lowpass_l2_sse2 put_h264_qpel16_h_lowpass_l2_mmx2

  1026. #define avg_h264_qpel16_h_lowpass_l2_sse2 avg_h264_qpel16_h_lowpass_l2_mmx2

  1027. 

  1028. #define put_h264_qpel8_v_lowpass_ssse3 put_h264_qpel8_v_lowpass_sse2

  1029. #define avg_h264_qpel8_v_lowpass_ssse3 avg_h264_qpel8_v_lowpass_sse2

  1030. #define put_h264_qpel16_v_lowpass_ssse3 put_h264_qpel16_v_lowpass_sse2

  1031. #define avg_h264_qpel16_v_lowpass_ssse3 avg_h264_qpel16_v_lowpass_sse2

  1032. 

  1033. #define put_h264_qpel8or16_hv2_lowpass_sse2 put_h264_qpel8or16_hv2_lowpass_mmx2

  1034. #define avg_h264_qpel8or16_hv2_lowpass_sse2 avg_h264_qpel8or16_hv2_lowpass_mmx2

  1035. 

  1036. #define H264_MC(OPNAME, SIZE, MMX, ALIGN) \

  1037. H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\

  1038. H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\

  1039. H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\

  1040. H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\

  1041. 

  1042. static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1043.     put_pixels16_sse2(dst, src, stride, 16);

  1044. }

  1045. static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){

  1046.     avg_pixels16_sse2(dst, src, stride, 16);

  1047. }

  1048. #define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmx2

  1049. #define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmx2

  1050. 

  1051. #define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \

  1052. static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\

  1053.     OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\

  1054. }\

  1055. 

  1056. #define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \

  1057. static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1058.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\

  1059. }\

  1060. \

  1061. static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1062.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\

  1063. }\

  1064. \

  1065. static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1066.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\

  1067. }\

  1068. 

  1069. #define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \

  1070. static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1071.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1072.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1073.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\

  1074. }\

  1075. \

  1076. static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1077.     OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\

  1078. }\

  1079. \

  1080. static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1081.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1082.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1083.     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\

  1084. }\

  1085. 

  1086. #define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \

  1087. static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1088.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1089.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1090.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1091. }\

  1092. \

  1093. static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1094.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1095.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1096.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\

  1097. }\

  1098. \

  1099. static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1100.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1101.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\

  1102.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1103. }\

  1104. \

  1105. static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1106.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\

  1107.     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\

  1108.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\

  1109. }\

  1110. \

  1111. static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1112.     DECLARE_ALIGNED(ALIGN, uint16_t, temp)[SIZE*(SIZE<8?12:24)];\

  1113.     OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\

  1114. }\

  1115. \

  1116. static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1117.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1118.     uint8_t * const halfHV= temp;\

  1119.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1120.     assert(((int)temp & 7) == 0);\

  1121.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1122.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\

  1123. }\

  1124. \

  1125. static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1126.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1127.     uint8_t * const halfHV= temp;\

  1128.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1129.     assert(((int)temp & 7) == 0);\

  1130.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1131.     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\

  1132. }\

  1133. \

  1134. static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1135.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1136.     uint8_t * const halfHV= temp;\

  1137.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1138.     assert(((int)temp & 7) == 0);\

  1139.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1140.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\

  1141. }\

  1142. \

  1143. static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1144.     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\

  1145.     uint8_t * const halfHV= temp;\

  1146.     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\

  1147.     assert(((int)temp & 7) == 0);\

  1148.     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\

  1149.     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\

  1150. }\

  1151. 

  1152. #define H264_MC_4816(MMX)\

  1153. H264_MC(put_, 4, MMX, 8)\

  1154. H264_MC(put_, 8, MMX, 8)\

  1155. H264_MC(put_, 16,MMX, 8)\

  1156. H264_MC(avg_, 4, MMX, 8)\

  1157. H264_MC(avg_, 8, MMX, 8)\

  1158. H264_MC(avg_, 16,MMX, 8)\

  1159. 

  1160. #define H264_MC_816(QPEL, XMM)\

  1161. QPEL(put_, 8, XMM, 16)\

  1162. QPEL(put_, 16,XMM, 16)\

  1163. QPEL(avg_, 8, XMM, 16)\

  1164. QPEL(avg_, 16,XMM, 16)\

  1165. 

  1166. #define PAVGB "pavgusb"

  1167. QPEL_H264(put_,       PUT_OP, 3dnow)

  1168. QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow)

  1169. #undef PAVGB

  1170. #define PAVGB "pavgb"

  1171. QPEL_H264(put_,       PUT_OP, mmx2)

  1172. QPEL_H264(avg_,  AVG_MMX2_OP, mmx2)

  1173. QPEL_H264_V_XMM(put_,       PUT_OP, sse2)

  1174. QPEL_H264_V_XMM(avg_,  AVG_MMX2_OP, sse2)

  1175. QPEL_H264_HV_XMM(put_,       PUT_OP, sse2)

  1176. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, sse2)

  1177. #if HAVE_SSSE3_INLINE

  1178. QPEL_H264_H_XMM(put_,       PUT_OP, ssse3)

  1179. QPEL_H264_H_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1180. QPEL_H264_HV2_XMM(put_,       PUT_OP, ssse3)

  1181. QPEL_H264_HV2_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1182. QPEL_H264_HV_XMM(put_,       PUT_OP, ssse3)

  1183. QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, ssse3)

  1184. #endif

  1185. #undef PAVGB

  1186. 

  1187. H264_MC_4816(3dnow)

  1188. H264_MC_4816(mmx2)

  1189. H264_MC_816(H264_MC_V, sse2)

  1190. H264_MC_816(H264_MC_HV, sse2)

  1191. #if HAVE_SSSE3_INLINE

  1192. H264_MC_816(H264_MC_H, ssse3)

  1193. H264_MC_816(H264_MC_HV, ssse3)

  1194. #endif

  1195. 

  1196. #endif /* HAVE_INLINE_ASM */

  1197. 

  1198. //10bit

  1199. #define LUMA_MC_OP(OP, NUM, DEPTH, TYPE, OPT) \

  1200. void ff_ ## OP ## _h264_qpel ## NUM ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT \

  1201.     (uint8_t *dst, uint8_t *src, int stride);

  1202. 

  1203. #define LUMA_MC_ALL(DEPTH, TYPE, OPT) \

  1204.     LUMA_MC_OP(put,  4, DEPTH, TYPE, OPT) \

  1205.     LUMA_MC_OP(avg,  4, DEPTH, TYPE, OPT) \

  1206.     LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \

  1207.     LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \

  1208.     LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \

  1209.     LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)

  1210. 

  1211. #define LUMA_MC_816(DEPTH, TYPE, OPT) \

  1212.     LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \

  1213.     LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \

  1214.     LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \

  1215.     LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)

  1216. 

  1217. LUMA_MC_ALL(10, mc00, mmxext)

  1218. LUMA_MC_ALL(10, mc10, mmxext)

  1219. LUMA_MC_ALL(10, mc20, mmxext)

  1220. LUMA_MC_ALL(10, mc30, mmxext)

  1221. LUMA_MC_ALL(10, mc01, mmxext)

  1222. LUMA_MC_ALL(10, mc11, mmxext)

  1223. LUMA_MC_ALL(10, mc21, mmxext)

  1224. LUMA_MC_ALL(10, mc31, mmxext)

  1225. LUMA_MC_ALL(10, mc02, mmxext)

  1226. LUMA_MC_ALL(10, mc12, mmxext)

  1227. LUMA_MC_ALL(10, mc22, mmxext)

  1228. LUMA_MC_ALL(10, mc32, mmxext)

  1229. LUMA_MC_ALL(10, mc03, mmxext)

  1230. LUMA_MC_ALL(10, mc13, mmxext)

  1231. LUMA_MC_ALL(10, mc23, mmxext)

  1232. LUMA_MC_ALL(10, mc33, mmxext)

  1233. 

  1234. LUMA_MC_816(10, mc00, sse2)

  1235. LUMA_MC_816(10, mc10, sse2)

  1236. LUMA_MC_816(10, mc10, sse2_cache64)

  1237. LUMA_MC_816(10, mc10, ssse3_cache64)

  1238. LUMA_MC_816(10, mc20, sse2)

  1239. LUMA_MC_816(10, mc20, sse2_cache64)

  1240. LUMA_MC_816(10, mc20, ssse3_cache64)

  1241. LUMA_MC_816(10, mc30, sse2)

  1242. LUMA_MC_816(10, mc30, sse2_cache64)

  1243. LUMA_MC_816(10, mc30, ssse3_cache64)

  1244. LUMA_MC_816(10, mc01, sse2)

  1245. LUMA_MC_816(10, mc11, sse2)

  1246. LUMA_MC_816(10, mc21, sse2)

  1247. LUMA_MC_816(10, mc31, sse2)

  1248. LUMA_MC_816(10, mc02, sse2)

  1249. LUMA_MC_816(10, mc12, sse2)

  1250. LUMA_MC_816(10, mc22, sse2)

  1251. LUMA_MC_816(10, mc32, sse2)

  1252. LUMA_MC_816(10, mc03, sse2)

  1253. LUMA_MC_816(10, mc13, sse2)

  1254. LUMA_MC_816(10, mc23, sse2)

  1255. LUMA_MC_816(10, mc33, sse2)

  1256. 

  1257. #define QPEL16_OPMC(OP, MC, MMX)\

  1258. void ff_ ## OP ## _h264_qpel16_ ## MC ## _10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\

  1259.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\

  1260.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\

  1261.     src += 8*stride;\

  1262.     dst += 8*stride;\

  1263.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\

  1264.     ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\

  1265. }

  1266. 

  1267. #define QPEL16_OP(MC, MMX)\

  1268. QPEL16_OPMC(put, MC, MMX)\

  1269. QPEL16_OPMC(avg, MC, MMX)

  1270. 

  1271. #define QPEL16(MMX)\

  1272. QPEL16_OP(mc00, MMX)\

  1273. QPEL16_OP(mc01, MMX)\

  1274. QPEL16_OP(mc02, MMX)\

  1275. QPEL16_OP(mc03, MMX)\

  1276. QPEL16_OP(mc10, MMX)\

  1277. QPEL16_OP(mc11, MMX)\

  1278. QPEL16_OP(mc12, MMX)\

  1279. QPEL16_OP(mc13, MMX)\

  1280. QPEL16_OP(mc20, MMX)\

  1281. QPEL16_OP(mc21, MMX)\

  1282. QPEL16_OP(mc22, MMX)\

  1283. QPEL16_OP(mc23, MMX)\

  1284. QPEL16_OP(mc30, MMX)\

  1285. QPEL16_OP(mc31, MMX)\

  1286. QPEL16_OP(mc32, MMX)\

  1287. QPEL16_OP(mc33, MMX)

  1288. 

  1289. #if ARCH_X86_32 && HAVE_YASM // ARCH_X86_64 implies sse2+

  1290. QPEL16(mmxext)

  1291. #endif