falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36452 Commits
32 Branches
944MB
Tree: 76e74e4831
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '76e74e4831'
${ noResults }
FFmpeg/libavcodec/x86/dsputil_rnd_template.c

595 lines
23KB
Raw Blame History 

  1. /*

  2.  * DSP utils mmx functions are compiled twice for rnd/no_rnd

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2003-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  * mostly rewritten by Michael Niedermayer <michaelni@gmx.at>

  8.  * and improved by Zdenek Kabelac <kabi@users.sf.net>

  9.  *

  10.  * This file is part of Libav.

  11.  *

  12.  * Libav is free software; you can redistribute it and/or

  13.  * modify it under the terms of the GNU Lesser General Public

  14.  * License as published by the Free Software Foundation; either

  15.  * version 2.1 of the License, or (at your option) any later version.

  16.  *

  17.  * Libav is distributed in the hope that it will be useful,

  18.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  19.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  20.  * Lesser General Public License for more details.

  21.  *

  22.  * You should have received a copy of the GNU Lesser General Public

  23.  * License along with Libav; if not, write to the Free Software

  24.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  25.  */

  26. 

  27. // put_pixels

  28. static void DEF(put, pixels8_x2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  29. {

  30.     MOVQ_BFE(mm6);

  31.     __asm__ volatile(

  32.         "lea    (%3, %3), %%"REG_a"     \n\t"

  33.         ".p2align 3                     \n\t"

  34.         "1:                             \n\t"

  35.         "movq   (%1), %%mm0             \n\t"

  36.         "movq   1(%1), %%mm1            \n\t"

  37.         "movq   (%1, %3), %%mm2         \n\t"

  38.         "movq   1(%1, %3), %%mm3        \n\t"

  39.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  40.         "movq   %%mm4, (%2)             \n\t"

  41.         "movq   %%mm5, (%2, %3)         \n\t"

  42.         "add    %%"REG_a", %1           \n\t"

  43.         "add    %%"REG_a", %2           \n\t"

  44.         "movq   (%1), %%mm0             \n\t"

  45.         "movq   1(%1), %%mm1            \n\t"

  46.         "movq   (%1, %3), %%mm2         \n\t"

  47.         "movq   1(%1, %3), %%mm3        \n\t"

  48.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  49.         "movq   %%mm4, (%2)             \n\t"

  50.         "movq   %%mm5, (%2, %3)         \n\t"

  51.         "add    %%"REG_a", %1           \n\t"

  52.         "add    %%"REG_a", %2           \n\t"

  53.         "subl   $4, %0                  \n\t"

  54.         "jnz    1b                      \n\t"

  55.         :"+g"(h), "+S"(pixels), "+D"(block)

  56.         :"r"((x86_reg)line_size)

  57.         :REG_a, "memory");

  58. }

  59. 

  60. static void av_unused DEF(put, pixels8_l2)(uint8_t *dst, uint8_t *src1, uint8_t *src2, int dstStride, int src1Stride, int h)

  61. {

  62.     MOVQ_BFE(mm6);

  63.     __asm__ volatile(

  64.         "testl $1, %0                   \n\t"

  65.         " jz 1f                         \n\t"

  66.         "movq   (%1), %%mm0             \n\t"

  67.         "movq   (%2), %%mm1             \n\t"

  68.         "add    %4, %1                  \n\t"

  69.         "add    $8, %2                  \n\t"

  70.         PAVGB(%%mm0, %%mm1, %%mm4, %%mm6)

  71.         "movq   %%mm4, (%3)             \n\t"

  72.         "add    %5, %3                  \n\t"

  73.         "decl   %0                      \n\t"

  74.         ".p2align 3                     \n\t"

  75.         "1:                             \n\t"

  76.         "movq   (%1), %%mm0             \n\t"

  77.         "movq   (%2), %%mm1             \n\t"

  78.         "add    %4, %1                  \n\t"

  79.         "movq   (%1), %%mm2             \n\t"

  80.         "movq   8(%2), %%mm3            \n\t"

  81.         "add    %4, %1                  \n\t"

  82.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  83.         "movq   %%mm4, (%3)             \n\t"

  84.         "add    %5, %3                  \n\t"

  85.         "movq   %%mm5, (%3)             \n\t"

  86.         "add    %5, %3                  \n\t"

  87.         "movq   (%1), %%mm0             \n\t"

  88.         "movq   16(%2), %%mm1           \n\t"

  89.         "add    %4, %1                  \n\t"

  90.         "movq   (%1), %%mm2             \n\t"

  91.         "movq   24(%2), %%mm3           \n\t"

  92.         "add    %4, %1                  \n\t"

  93.         "add    $32, %2                 \n\t"

  94.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  95.         "movq   %%mm4, (%3)             \n\t"

  96.         "add    %5, %3                  \n\t"

  97.         "movq   %%mm5, (%3)             \n\t"

  98.         "add    %5, %3                  \n\t"

  99.         "subl   $4, %0                  \n\t"

  100.         "jnz    1b                      \n\t"

  101. #if !HAVE_EBX_AVAILABLE //Note "+bm" and "+mb" are buggy too (with gcc 3.2.2 at least) and cannot be used

  102.         :"+m"(h), "+a"(src1), "+c"(src2), "+d"(dst)

  103. #else

  104.         :"+b"(h), "+a"(src1), "+c"(src2), "+d"(dst)

  105. #endif

  106.         :"S"((x86_reg)src1Stride), "D"((x86_reg)dstStride)

  107.         :"memory");

  108. }

  109. 

  110. static void DEF(put, pixels16_x2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  111. {

  112.     MOVQ_BFE(mm6);

  113.     __asm__ volatile(

  114.         "lea        (%3, %3), %%"REG_a" \n\t"

  115.         ".p2align 3                     \n\t"

  116.         "1:                             \n\t"

  117.         "movq   (%1), %%mm0             \n\t"

  118.         "movq   1(%1), %%mm1            \n\t"

  119.         "movq   (%1, %3), %%mm2         \n\t"

  120.         "movq   1(%1, %3), %%mm3        \n\t"

  121.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  122.         "movq   %%mm4, (%2)             \n\t"

  123.         "movq   %%mm5, (%2, %3)         \n\t"

  124.         "movq   8(%1), %%mm0            \n\t"

  125.         "movq   9(%1), %%mm1            \n\t"

  126.         "movq   8(%1, %3), %%mm2        \n\t"

  127.         "movq   9(%1, %3), %%mm3        \n\t"

  128.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  129.         "movq   %%mm4, 8(%2)            \n\t"

  130.         "movq   %%mm5, 8(%2, %3)        \n\t"

  131.         "add    %%"REG_a", %1           \n\t"

  132.         "add    %%"REG_a", %2           \n\t"

  133.         "movq   (%1), %%mm0             \n\t"

  134.         "movq   1(%1), %%mm1            \n\t"

  135.         "movq   (%1, %3), %%mm2         \n\t"

  136.         "movq   1(%1, %3), %%mm3        \n\t"

  137.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  138.         "movq   %%mm4, (%2)             \n\t"

  139.         "movq   %%mm5, (%2, %3)         \n\t"

  140.         "movq   8(%1), %%mm0            \n\t"

  141.         "movq   9(%1), %%mm1            \n\t"

  142.         "movq   8(%1, %3), %%mm2        \n\t"

  143.         "movq   9(%1, %3), %%mm3        \n\t"

  144.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  145.         "movq   %%mm4, 8(%2)            \n\t"

  146.         "movq   %%mm5, 8(%2, %3)        \n\t"

  147.         "add    %%"REG_a", %1           \n\t"

  148.         "add    %%"REG_a", %2           \n\t"

  149.         "subl   $4, %0                  \n\t"

  150.         "jnz    1b                      \n\t"

  151.         :"+g"(h), "+S"(pixels), "+D"(block)

  152.         :"r"((x86_reg)line_size)

  153.         :REG_a, "memory");

  154. }

  155. 

  156. static void av_unused DEF(put, pixels16_l2)(uint8_t *dst, uint8_t *src1, uint8_t *src2, int dstStride, int src1Stride, int h)

  157. {

  158.     MOVQ_BFE(mm6);

  159.     __asm__ volatile(

  160.         "testl $1, %0                   \n\t"

  161.         " jz 1f                         \n\t"

  162.         "movq   (%1), %%mm0             \n\t"

  163.         "movq   (%2), %%mm1             \n\t"

  164.         "movq   8(%1), %%mm2            \n\t"

  165.         "movq   8(%2), %%mm3            \n\t"

  166.         "add    %4, %1                  \n\t"

  167.         "add    $16, %2                 \n\t"

  168.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  169.         "movq   %%mm4, (%3)             \n\t"

  170.         "movq   %%mm5, 8(%3)            \n\t"

  171.         "add    %5, %3                  \n\t"

  172.         "decl   %0                      \n\t"

  173.         ".p2align 3                     \n\t"

  174.         "1:                             \n\t"

  175.         "movq   (%1), %%mm0             \n\t"

  176.         "movq   (%2), %%mm1             \n\t"

  177.         "movq   8(%1), %%mm2            \n\t"

  178.         "movq   8(%2), %%mm3            \n\t"

  179.         "add    %4, %1                  \n\t"

  180.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  181.         "movq   %%mm4, (%3)             \n\t"

  182.         "movq   %%mm5, 8(%3)            \n\t"

  183.         "add    %5, %3                  \n\t"

  184.         "movq   (%1), %%mm0             \n\t"

  185.         "movq   16(%2), %%mm1           \n\t"

  186.         "movq   8(%1), %%mm2            \n\t"

  187.         "movq   24(%2), %%mm3           \n\t"

  188.         "add    %4, %1                  \n\t"

  189.         PAVGBP(%%mm0, %%mm1, %%mm4,   %%mm2, %%mm3, %%mm5)

  190.         "movq   %%mm4, (%3)             \n\t"

  191.         "movq   %%mm5, 8(%3)            \n\t"

  192.         "add    %5, %3                  \n\t"

  193.         "add    $32, %2                 \n\t"

  194.         "subl   $2, %0                  \n\t"

  195.         "jnz    1b                      \n\t"

  196. #if !HAVE_EBX_AVAILABLE  //Note "+bm" and "+mb" are buggy too (with gcc 3.2.2 at least) and cannot be used

  197.         :"+m"(h), "+a"(src1), "+c"(src2), "+d"(dst)

  198. #else

  199.         :"+b"(h), "+a"(src1), "+c"(src2), "+d"(dst)

  200. #endif

  201.         :"S"((x86_reg)src1Stride), "D"((x86_reg)dstStride)

  202.         :"memory");

  203. }

  204. 

  205. static void DEF(put, pixels8_y2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  206. {

  207.     MOVQ_BFE(mm6);

  208.     __asm__ volatile(

  209.         "lea (%3, %3), %%"REG_a"        \n\t"

  210.         "movq (%1), %%mm0               \n\t"

  211.         ".p2align 3                     \n\t"

  212.         "1:                             \n\t"

  213.         "movq   (%1, %3), %%mm1         \n\t"

  214.         "movq   (%1, %%"REG_a"),%%mm2   \n\t"

  215.         PAVGBP(%%mm1, %%mm0, %%mm4,   %%mm2, %%mm1, %%mm5)

  216.         "movq   %%mm4, (%2)             \n\t"

  217.         "movq   %%mm5, (%2, %3)         \n\t"

  218.         "add    %%"REG_a", %1           \n\t"

  219.         "add    %%"REG_a", %2           \n\t"

  220.         "movq   (%1, %3), %%mm1         \n\t"

  221.         "movq   (%1, %%"REG_a"),%%mm0   \n\t"

  222.         PAVGBP(%%mm1, %%mm2, %%mm4,   %%mm0, %%mm1, %%mm5)

  223.         "movq   %%mm4, (%2)             \n\t"

  224.         "movq   %%mm5, (%2, %3)         \n\t"

  225.         "add    %%"REG_a", %1           \n\t"

  226.         "add    %%"REG_a", %2           \n\t"

  227.         "subl   $4, %0                  \n\t"

  228.         "jnz    1b                      \n\t"

  229.         :"+g"(h), "+S"(pixels), "+D"(block)

  230.         :"r"((x86_reg)line_size)

  231.         :REG_a, "memory");

  232. }

  233. 

  234. static void DEF(put, pixels8_xy2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  235. {

  236.     MOVQ_ZERO(mm7);

  237.     SET_RND(mm6); // =2 for rnd  and  =1 for no_rnd version

  238.     __asm__ volatile(

  239.         "movq   (%1), %%mm0             \n\t"

  240.         "movq   1(%1), %%mm4            \n\t"

  241.         "movq   %%mm0, %%mm1            \n\t"

  242.         "movq   %%mm4, %%mm5            \n\t"

  243.         "punpcklbw %%mm7, %%mm0         \n\t"

  244.         "punpcklbw %%mm7, %%mm4         \n\t"

  245.         "punpckhbw %%mm7, %%mm1         \n\t"

  246.         "punpckhbw %%mm7, %%mm5         \n\t"

  247.         "paddusw %%mm0, %%mm4           \n\t"

  248.         "paddusw %%mm1, %%mm5           \n\t"

  249.         "xor    %%"REG_a", %%"REG_a"    \n\t"

  250.         "add    %3, %1                  \n\t"

  251.         ".p2align 3                     \n\t"

  252.         "1:                             \n\t"

  253.         "movq   (%1, %%"REG_a"), %%mm0  \n\t"

  254.         "movq   1(%1, %%"REG_a"), %%mm2 \n\t"

  255.         "movq   %%mm0, %%mm1            \n\t"

  256.         "movq   %%mm2, %%mm3            \n\t"

  257.         "punpcklbw %%mm7, %%mm0         \n\t"

  258.         "punpcklbw %%mm7, %%mm2         \n\t"

  259.         "punpckhbw %%mm7, %%mm1         \n\t"

  260.         "punpckhbw %%mm7, %%mm3         \n\t"

  261.         "paddusw %%mm2, %%mm0           \n\t"

  262.         "paddusw %%mm3, %%mm1           \n\t"

  263.         "paddusw %%mm6, %%mm4           \n\t"

  264.         "paddusw %%mm6, %%mm5           \n\t"

  265.         "paddusw %%mm0, %%mm4           \n\t"

  266.         "paddusw %%mm1, %%mm5           \n\t"

  267.         "psrlw  $2, %%mm4               \n\t"

  268.         "psrlw  $2, %%mm5               \n\t"

  269.         "packuswb  %%mm5, %%mm4         \n\t"

  270.         "movq   %%mm4, (%2, %%"REG_a")  \n\t"

  271.         "add    %3, %%"REG_a"           \n\t"

  272. 

  273.         "movq   (%1, %%"REG_a"), %%mm2  \n\t" // 0 <-> 2   1 <-> 3

  274.         "movq   1(%1, %%"REG_a"), %%mm4 \n\t"

  275.         "movq   %%mm2, %%mm3            \n\t"

  276.         "movq   %%mm4, %%mm5            \n\t"

  277.         "punpcklbw %%mm7, %%mm2         \n\t"

  278.         "punpcklbw %%mm7, %%mm4         \n\t"

  279.         "punpckhbw %%mm7, %%mm3         \n\t"

  280.         "punpckhbw %%mm7, %%mm5         \n\t"

  281.         "paddusw %%mm2, %%mm4           \n\t"

  282.         "paddusw %%mm3, %%mm5           \n\t"

  283.         "paddusw %%mm6, %%mm0           \n\t"

  284.         "paddusw %%mm6, %%mm1           \n\t"

  285.         "paddusw %%mm4, %%mm0           \n\t"

  286.         "paddusw %%mm5, %%mm1           \n\t"

  287.         "psrlw  $2, %%mm0               \n\t"

  288.         "psrlw  $2, %%mm1               \n\t"

  289.         "packuswb  %%mm1, %%mm0         \n\t"

  290.         "movq   %%mm0, (%2, %%"REG_a")  \n\t"

  291.         "add    %3, %%"REG_a"           \n\t"

  292. 

  293.         "subl   $2, %0                  \n\t"

  294.         "jnz    1b                      \n\t"

  295.         :"+g"(h), "+S"(pixels)

  296.         :"D"(block), "r"((x86_reg)line_size)

  297.         :REG_a, "memory");

  298. }

  299. 

  300. // avg_pixels

  301. static void av_unused DEF(avg, pixels4)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  302. {

  303.     MOVQ_BFE(mm6);

  304.     JUMPALIGN();

  305.     do {

  306.         __asm__ volatile(

  307.              "movd  %0, %%mm0           \n\t"

  308.              "movd  %1, %%mm1           \n\t"

  309.              OP_AVG(%%mm0, %%mm1, %%mm2, %%mm6)

  310.              "movd  %%mm2, %0           \n\t"

  311.              :"+m"(*block)

  312.              :"m"(*pixels)

  313.              :"memory");

  314.         pixels += line_size;

  315.         block += line_size;

  316.     }

  317.     while (--h);

  318. }

  319. 

  320. #ifndef NO_RND

  321. // in case more speed is needed - unroling would certainly help

  322. static void DEF(avg, pixels8)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  323. {

  324.     MOVQ_BFE(mm6);

  325.     JUMPALIGN();

  326.     do {

  327.         __asm__ volatile(

  328.              "movq  %0, %%mm0           \n\t"

  329.              "movq  %1, %%mm1           \n\t"

  330.              OP_AVG(%%mm0, %%mm1, %%mm2, %%mm6)

  331.              "movq  %%mm2, %0           \n\t"

  332.              :"+m"(*block)

  333.              :"m"(*pixels)

  334.              :"memory");

  335.         pixels += line_size;

  336.         block += line_size;

  337.     }

  338.     while (--h);

  339. }

  340. #endif // NO_RND

  341. 

  342. static void DEF(avg, pixels16)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  343. {

  344.     MOVQ_BFE(mm6);

  345.     JUMPALIGN();

  346.     do {

  347.         __asm__ volatile(

  348.              "movq  %0, %%mm0           \n\t"

  349.              "movq  %1, %%mm1           \n\t"

  350.              OP_AVG(%%mm0, %%mm1, %%mm2, %%mm6)

  351.              "movq  %%mm2, %0           \n\t"

  352.              "movq  8%0, %%mm0          \n\t"

  353.              "movq  8%1, %%mm1          \n\t"

  354.              OP_AVG(%%mm0, %%mm1, %%mm2, %%mm6)

  355.              "movq  %%mm2, 8%0          \n\t"

  356.              :"+m"(*block)

  357.              :"m"(*pixels)

  358.              :"memory");

  359.         pixels += line_size;

  360.         block += line_size;

  361.     }

  362.     while (--h);

  363. }

  364. 

  365. #ifndef NO_RND

  366. static void DEF(avg, pixels8_x2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  367. {

  368.     MOVQ_BFE(mm6);

  369.     JUMPALIGN();

  370.     do {

  371.         __asm__ volatile(

  372.             "movq  %1, %%mm0            \n\t"

  373.             "movq  1%1, %%mm1           \n\t"

  374.             "movq  %0, %%mm3            \n\t"

  375.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  376.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  377.             "movq  %%mm0, %0            \n\t"

  378.             :"+m"(*block)

  379.             :"m"(*pixels)

  380.             :"memory");

  381.         pixels += line_size;

  382.         block += line_size;

  383.     } while (--h);

  384. }

  385. #endif // NO_RND

  386. 

  387. static av_unused void DEF(avg, pixels8_l2)(uint8_t *dst, uint8_t *src1, uint8_t *src2, int dstStride, int src1Stride, int h)

  388. {

  389.     MOVQ_BFE(mm6);

  390.     JUMPALIGN();

  391.     do {

  392.         __asm__ volatile(

  393.             "movq  %1, %%mm0            \n\t"

  394.             "movq  %2, %%mm1            \n\t"

  395.             "movq  %0, %%mm3            \n\t"

  396.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  397.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  398.             "movq  %%mm0, %0            \n\t"

  399.             :"+m"(*dst)

  400.             :"m"(*src1), "m"(*src2)

  401.             :"memory");

  402.         dst += dstStride;

  403.         src1 += src1Stride;

  404.         src2 += 8;

  405.     } while (--h);

  406. }

  407. 

  408. static void DEF(avg, pixels16_x2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  409. {

  410.     MOVQ_BFE(mm6);

  411.     JUMPALIGN();

  412.     do {

  413.         __asm__ volatile(

  414.             "movq  %1, %%mm0            \n\t"

  415.             "movq  1%1, %%mm1           \n\t"

  416.             "movq  %0, %%mm3            \n\t"

  417.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  418.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  419.             "movq  %%mm0, %0            \n\t"

  420.             "movq  8%1, %%mm0           \n\t"

  421.             "movq  9%1, %%mm1           \n\t"

  422.             "movq  8%0, %%mm3           \n\t"

  423.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  424.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  425.             "movq  %%mm0, 8%0           \n\t"

  426.             :"+m"(*block)

  427.             :"m"(*pixels)

  428.             :"memory");

  429.         pixels += line_size;

  430.         block += line_size;

  431.     } while (--h);

  432. }

  433. 

  434. static av_unused void DEF(avg, pixels16_l2)(uint8_t *dst, uint8_t *src1, uint8_t *src2, int dstStride, int src1Stride, int h)

  435. {

  436.     MOVQ_BFE(mm6);

  437.     JUMPALIGN();

  438.     do {

  439.         __asm__ volatile(

  440.             "movq  %1, %%mm0            \n\t"

  441.             "movq  %2, %%mm1            \n\t"

  442.             "movq  %0, %%mm3            \n\t"

  443.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  444.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  445.             "movq  %%mm0, %0            \n\t"

  446.             "movq  8%1, %%mm0           \n\t"

  447.             "movq  8%2, %%mm1           \n\t"

  448.             "movq  8%0, %%mm3           \n\t"

  449.             PAVGB(%%mm0, %%mm1, %%mm2, %%mm6)

  450.             OP_AVG(%%mm3, %%mm2, %%mm0, %%mm6)

  451.             "movq  %%mm0, 8%0           \n\t"

  452.             :"+m"(*dst)

  453.             :"m"(*src1), "m"(*src2)

  454.             :"memory");

  455.         dst += dstStride;

  456.         src1 += src1Stride;

  457.         src2 += 16;

  458.     } while (--h);

  459. }

  460. 

  461. static void DEF(avg, pixels8_y2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  462. {

  463.     MOVQ_BFE(mm6);

  464.     __asm__ volatile(

  465.         "lea    (%3, %3), %%"REG_a"     \n\t"

  466.         "movq   (%1), %%mm0             \n\t"

  467.         ".p2align 3                     \n\t"

  468.         "1:                             \n\t"

  469.         "movq   (%1, %3), %%mm1         \n\t"

  470.         "movq   (%1, %%"REG_a"), %%mm2  \n\t"

  471.         PAVGBP(%%mm1, %%mm0, %%mm4,   %%mm2, %%mm1, %%mm5)

  472.         "movq   (%2), %%mm3             \n\t"

  473.         OP_AVG(%%mm3, %%mm4, %%mm0, %%mm6)

  474.         "movq   (%2, %3), %%mm3         \n\t"

  475.         OP_AVG(%%mm3, %%mm5, %%mm1, %%mm6)

  476.         "movq   %%mm0, (%2)             \n\t"

  477.         "movq   %%mm1, (%2, %3)         \n\t"

  478.         "add    %%"REG_a", %1           \n\t"

  479.         "add    %%"REG_a", %2           \n\t"

  480. 

  481.         "movq   (%1, %3), %%mm1         \n\t"

  482.         "movq   (%1, %%"REG_a"), %%mm0  \n\t"

  483.         PAVGBP(%%mm1, %%mm2, %%mm4,   %%mm0, %%mm1, %%mm5)

  484.         "movq   (%2), %%mm3             \n\t"

  485.         OP_AVG(%%mm3, %%mm4, %%mm2, %%mm6)

  486.         "movq   (%2, %3), %%mm3         \n\t"

  487.         OP_AVG(%%mm3, %%mm5, %%mm1, %%mm6)

  488.         "movq   %%mm2, (%2)             \n\t"

  489.         "movq   %%mm1, (%2, %3)         \n\t"

  490.         "add    %%"REG_a", %1           \n\t"

  491.         "add    %%"REG_a", %2           \n\t"

  492. 

  493.         "subl   $4, %0                  \n\t"

  494.         "jnz    1b                      \n\t"

  495.         :"+g"(h), "+S"(pixels), "+D"(block)

  496.         :"r"((x86_reg)line_size)

  497.         :REG_a, "memory");

  498. }

  499. 

  500. // this routine is 'slightly' suboptimal but mostly unused

  501. static void DEF(avg, pixels8_xy2)(uint8_t *block, const uint8_t *pixels, int line_size, int h)

  502. {

  503.     MOVQ_ZERO(mm7);

  504.     SET_RND(mm6); // =2 for rnd  and  =1 for no_rnd version

  505.     __asm__ volatile(

  506.         "movq   (%1), %%mm0             \n\t"

  507.         "movq   1(%1), %%mm4            \n\t"

  508.         "movq   %%mm0, %%mm1            \n\t"

  509.         "movq   %%mm4, %%mm5            \n\t"

  510.         "punpcklbw %%mm7, %%mm0         \n\t"

  511.         "punpcklbw %%mm7, %%mm4         \n\t"

  512.         "punpckhbw %%mm7, %%mm1         \n\t"

  513.         "punpckhbw %%mm7, %%mm5         \n\t"

  514.         "paddusw %%mm0, %%mm4           \n\t"

  515.         "paddusw %%mm1, %%mm5           \n\t"

  516.         "xor    %%"REG_a", %%"REG_a"    \n\t"

  517.         "add    %3, %1                  \n\t"

  518.         ".p2align 3                     \n\t"

  519.         "1:                             \n\t"

  520.         "movq   (%1, %%"REG_a"), %%mm0  \n\t"

  521.         "movq   1(%1, %%"REG_a"), %%mm2 \n\t"

  522.         "movq   %%mm0, %%mm1            \n\t"

  523.         "movq   %%mm2, %%mm3            \n\t"

  524.         "punpcklbw %%mm7, %%mm0         \n\t"

  525.         "punpcklbw %%mm7, %%mm2         \n\t"

  526.         "punpckhbw %%mm7, %%mm1         \n\t"

  527.         "punpckhbw %%mm7, %%mm3         \n\t"

  528.         "paddusw %%mm2, %%mm0           \n\t"

  529.         "paddusw %%mm3, %%mm1           \n\t"

  530.         "paddusw %%mm6, %%mm4           \n\t"

  531.         "paddusw %%mm6, %%mm5           \n\t"

  532.         "paddusw %%mm0, %%mm4           \n\t"

  533.         "paddusw %%mm1, %%mm5           \n\t"

  534.         "psrlw  $2, %%mm4               \n\t"

  535.         "psrlw  $2, %%mm5               \n\t"

  536.                 "movq   (%2, %%"REG_a"), %%mm3  \n\t"

  537.         "packuswb  %%mm5, %%mm4         \n\t"

  538.                 "pcmpeqd %%mm2, %%mm2   \n\t"

  539.                 "paddb %%mm2, %%mm2     \n\t"

  540.                 OP_AVG(%%mm3, %%mm4, %%mm5, %%mm2)

  541.                 "movq   %%mm5, (%2, %%"REG_a")  \n\t"

  542.         "add    %3, %%"REG_a"                \n\t"

  543. 

  544.         "movq   (%1, %%"REG_a"), %%mm2  \n\t" // 0 <-> 2   1 <-> 3

  545.         "movq   1(%1, %%"REG_a"), %%mm4 \n\t"

  546.         "movq   %%mm2, %%mm3            \n\t"

  547.         "movq   %%mm4, %%mm5            \n\t"

  548.         "punpcklbw %%mm7, %%mm2         \n\t"

  549.         "punpcklbw %%mm7, %%mm4         \n\t"

  550.         "punpckhbw %%mm7, %%mm3         \n\t"

  551.         "punpckhbw %%mm7, %%mm5         \n\t"

  552.         "paddusw %%mm2, %%mm4           \n\t"

  553.         "paddusw %%mm3, %%mm5           \n\t"

  554.         "paddusw %%mm6, %%mm0           \n\t"

  555.         "paddusw %%mm6, %%mm1           \n\t"

  556.         "paddusw %%mm4, %%mm0           \n\t"

  557.         "paddusw %%mm5, %%mm1           \n\t"

  558.         "psrlw  $2, %%mm0               \n\t"

  559.         "psrlw  $2, %%mm1               \n\t"

  560.                 "movq   (%2, %%"REG_a"), %%mm3  \n\t"

  561.         "packuswb  %%mm1, %%mm0         \n\t"

  562.                 "pcmpeqd %%mm2, %%mm2   \n\t"

  563.                 "paddb %%mm2, %%mm2     \n\t"

  564.                 OP_AVG(%%mm3, %%mm0, %%mm1, %%mm2)

  565.                 "movq   %%mm1, (%2, %%"REG_a")  \n\t"

  566.         "add    %3, %%"REG_a"           \n\t"

  567. 

  568.         "subl   $2, %0                  \n\t"

  569.         "jnz    1b                      \n\t"

  570.         :"+g"(h), "+S"(pixels)

  571.         :"D"(block), "r"((x86_reg)line_size)

  572.         :REG_a, "memory");

  573. }

  574. 

  575. //FIXME optimize

  576. static void DEF(put, pixels16_y2)(uint8_t *block, const uint8_t *pixels, int line_size, int h){

  577.     DEF(put, pixels8_y2)(block  , pixels  , line_size, h);

  578.     DEF(put, pixels8_y2)(block+8, pixels+8, line_size, h);

  579. }

  580. 

  581. static void DEF(put, pixels16_xy2)(uint8_t *block, const uint8_t *pixels, int line_size, int h){

  582.     DEF(put, pixels8_xy2)(block  , pixels  , line_size, h);

  583.     DEF(put, pixels8_xy2)(block+8, pixels+8, line_size, h);

  584. }

  585. 

  586. static void DEF(avg, pixels16_y2)(uint8_t *block, const uint8_t *pixels, int line_size, int h){

  587.     DEF(avg, pixels8_y2)(block  , pixels  , line_size, h);

  588.     DEF(avg, pixels8_y2)(block+8, pixels+8, line_size, h);

  589. }

  590. 

  591. static void DEF(avg, pixels16_xy2)(uint8_t *block, const uint8_t *pixels, int line_size, int h){

  592.     DEF(avg, pixels8_xy2)(block  , pixels  , line_size, h);

  593.     DEF(avg, pixels8_xy2)(block+8, pixels+8, line_size, h);

  594. }