falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
30472 Commits
32 Branches
283MB
Tree: b932eb1be6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'b932eb1be6'
${ noResults }
FFmpeg/libswscale/x86/swscale_template.c

2552 lines
117KB
Raw Blame History 

  1. /*

  2.  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>

  3.  *

  4.  * This file is part of FFmpeg.

  5.  *

  6.  * FFmpeg is free software; you can redistribute it and/or

  7.  * modify it under the terms of the GNU Lesser General Public

  8.  * License as published by the Free Software Foundation; either

  9.  * version 2.1 of the License, or (at your option) any later version.

  10.  *

  11.  * FFmpeg is distributed in the hope that it will be useful,

  12.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  13.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  14.  * Lesser General Public License for more details.

  15.  *

  16.  * You should have received a copy of the GNU Lesser General Public

  17.  * License along with FFmpeg; if not, write to the Free Software

  18.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  19.  */

  20. 

  21. #include "swscale_template.h"

  22. 

  23. #undef REAL_MOVNTQ

  24. #undef MOVNTQ

  25. #undef PREFETCH

  26. 

  27. #if COMPILE_TEMPLATE_MMX2

  28. #define PREFETCH "prefetchnta"

  29. #else

  30. #define PREFETCH  " # nop"

  31. #endif

  32. 

  33. #if COMPILE_TEMPLATE_MMX2

  34. #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"

  35. #else

  36. #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"

  37. #endif

  38. #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)

  39. 

  40. #define YSCALEYUV2YV12X(offset, dest, end, pos) \

  41.     __asm__ volatile(\

  42.         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\

  43.         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\

  44.         "lea                     " offset "(%0), %%"REG_d"  \n\t"\

  45.         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\

  46.         ".p2align                             4             \n\t" /* FIXME Unroll? */\

  47.         "1:                                                 \n\t"\

  48.         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\

  49.         "movq                (%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\

  50.         "movq               8(%%"REG_S", %3, 2), %%mm5      \n\t" /* srcData */\

  51.         "add                                $16, %%"REG_d"  \n\t"\

  52.         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\

  53.         "test                         %%"REG_S", %%"REG_S"  \n\t"\

  54.         "pmulhw                           %%mm0, %%mm2      \n\t"\

  55.         "pmulhw                           %%mm0, %%mm5      \n\t"\

  56.         "paddw                            %%mm2, %%mm3      \n\t"\

  57.         "paddw                            %%mm5, %%mm4      \n\t"\

  58.         " jnz                                1b             \n\t"\

  59.         "psraw                               $3, %%mm3      \n\t"\

  60.         "psraw                               $3, %%mm4      \n\t"\

  61.         "packuswb                         %%mm4, %%mm3      \n\t"\

  62.         MOVNTQ(%%mm3, (%1, %3))\

  63.         "add                                 $8, %3         \n\t"\

  64.         "cmp                                 %2, %3         \n\t"\

  65.         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\

  66.         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\

  67.         "lea                     " offset "(%0), %%"REG_d"  \n\t"\

  68.         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\

  69.         "jb                                  1b             \n\t"\

  70.         :: "r" (&c->redDither),\

  71.            "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\

  72.         : "%"REG_d, "%"REG_S\

  73.     );

  74. 

  75. static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,

  76.                                     const int16_t **lumSrc, int lumFilterSize,

  77.                                     const int16_t *chrFilter, const int16_t **chrUSrc,

  78.                                     const int16_t **chrVSrc,

  79.                                     int chrFilterSize, const int16_t **alpSrc,

  80.                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,

  81.                                     uint8_t *aDest, int dstW, int chrDstW,

  82.                                     const uint8_t *lumDither, const uint8_t *chrDither)

  83. {

  84.     int i;

  85.     if (uDest) {

  86.         x86_reg uv_off = c->uv_off;

  87.         for(i=0; i<8; i++) c->dither16[i] = chrDither[i]>>4;

  88.         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)

  89.         for(i=0; i<8; i++) c->dither16[i] = chrDither[(i+3)&7]>>4;

  90.         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)

  91.     }

  92.     for(i=0; i<8; i++) c->dither16[i] = lumDither[i]>>4;

  93.     if (CONFIG_SWSCALE_ALPHA && aDest) {

  94.         YSCALEYUV2YV12X(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)

  95.     }

  96. 

  97.     YSCALEYUV2YV12X(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)

  98. }

  99. 

  100. #define YSCALEYUV2YV12X_ACCURATE(offset, dest, end, pos) \

  101.     __asm__ volatile(\

  102.         "lea                     " offset "(%0), %%"REG_d"  \n\t"\

  103.         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\

  104.         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\

  105.         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\

  106.         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\

  107.         "pxor                             %%mm4, %%mm4      \n\t"\

  108.         "pxor                             %%mm5, %%mm5      \n\t"\

  109.         "pxor                             %%mm6, %%mm6      \n\t"\

  110.         "pxor                             %%mm7, %%mm7      \n\t"\

  111.         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\

  112.         ".p2align                             4             \n\t"\

  113.         "1:                                                 \n\t"\

  114.         "movq                (%%"REG_S", %3, 2), %%mm0      \n\t" /* srcData */\

  115.         "movq               8(%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\

  116.         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\

  117.         "movq                (%%"REG_S", %3, 2), %%mm1      \n\t" /* srcData */\

  118.         "movq                             %%mm0, %%mm3      \n\t"\

  119.         "punpcklwd                        %%mm1, %%mm0      \n\t"\

  120.         "punpckhwd                        %%mm1, %%mm3      \n\t"\

  121.         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\

  122.         "pmaddwd                          %%mm1, %%mm0      \n\t"\

  123.         "pmaddwd                          %%mm1, %%mm3      \n\t"\

  124.         "paddd                            %%mm0, %%mm4      \n\t"\

  125.         "paddd                            %%mm3, %%mm5      \n\t"\

  126.         "movq               8(%%"REG_S", %3, 2), %%mm3      \n\t" /* srcData */\

  127.         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\

  128.         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\

  129.         "test                         %%"REG_S", %%"REG_S"  \n\t"\

  130.         "movq                             %%mm2, %%mm0      \n\t"\

  131.         "punpcklwd                        %%mm3, %%mm2      \n\t"\

  132.         "punpckhwd                        %%mm3, %%mm0      \n\t"\

  133.         "pmaddwd                          %%mm1, %%mm2      \n\t"\

  134.         "pmaddwd                          %%mm1, %%mm0      \n\t"\

  135.         "paddd                            %%mm2, %%mm6      \n\t"\

  136.         "paddd                            %%mm0, %%mm7      \n\t"\

  137.         " jnz                                1b             \n\t"\

  138.         "psrad                              $19, %%mm4      \n\t"\

  139.         "psrad                              $19, %%mm5      \n\t"\

  140.         "psrad                              $19, %%mm6      \n\t"\

  141.         "psrad                              $19, %%mm7      \n\t"\

  142.         "packssdw                         %%mm5, %%mm4      \n\t"\

  143.         "packssdw                         %%mm7, %%mm6      \n\t"\

  144.         "packuswb                         %%mm6, %%mm4      \n\t"\

  145.         MOVNTQ(%%mm4, (%1, %3))\

  146.         "add                                 $8, %3         \n\t"\

  147.         "cmp                                 %2, %3         \n\t"\

  148.         "lea                     " offset "(%0), %%"REG_d"  \n\t"\

  149.         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\

  150.         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\

  151.         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\

  152.         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\

  153.         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\

  154.         "jb                                  1b             \n\t"\

  155.         :: "r" (&c->redDither),\

  156.         "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\

  157.         : "%"REG_a, "%"REG_d, "%"REG_S\

  158.     );

  159. 

  160. static inline void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,

  161.                                        const int16_t **lumSrc, int lumFilterSize,

  162.                                        const int16_t *chrFilter, const int16_t **chrUSrc,

  163.                                        const int16_t **chrVSrc,

  164.                                        int chrFilterSize, const int16_t **alpSrc,

  165.                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,

  166.                                        uint8_t *aDest, int dstW, int chrDstW,

  167.                                        const uint8_t *lumDither, const uint8_t *chrDither)

  168. {

  169.     int i;

  170.     if (uDest) {

  171.         x86_reg uv_off = c->uv_off;

  172.         for(i=0; i<8; i++) c->dither32[i] = chrDither[i]<<12;

  173.         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)

  174.         for(i=0; i<8; i++) c->dither32[i] = chrDither[(i+3)&7]<<12;

  175.         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)

  176.     }

  177.     for(i=0; i<8; i++) c->dither32[i] = lumDither[i]<<12;

  178.     if (CONFIG_SWSCALE_ALPHA && aDest) {

  179.         YSCALEYUV2YV12X_ACCURATE(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)

  180.     }

  181. 

  182.     YSCALEYUV2YV12X_ACCURATE(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)

  183. }

  184. 

  185. #define YSCALEYUV2YV121 \

  186.     "mov %2, %%"REG_a"                    \n\t"\

  187.     ".p2align               4             \n\t" /* FIXME Unroll? */\

  188.     "1:                                   \n\t"\

  189.     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\

  190.     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\

  191.     "psraw                 $7, %%mm0      \n\t"\

  192.     "psraw                 $7, %%mm1      \n\t"\

  193.     "packuswb           %%mm1, %%mm0      \n\t"\

  194.     MOVNTQ(%%mm0, (%1, %%REGa))\

  195.     "add                   $8, %%"REG_a"  \n\t"\

  196.     "jnc                   1b             \n\t"

  197. 

  198. static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,

  199.                                     const int16_t *chrUSrc, const int16_t *chrVSrc,

  200.                                     const int16_t *alpSrc,

  201.                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,

  202.                                     uint8_t *aDest, int dstW, int chrDstW,

  203.                                     const uint8_t *lumDither, const uint8_t *chrDither)

  204. {

  205.     int p= 4;

  206.     const int16_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };

  207.     uint8_t *dst[4]= { aDest, dest, uDest, vDest };

  208.     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };

  209. 

  210.     while (p--) {

  211.         if (dst[p]) {

  212.             __asm__ volatile(

  213.                YSCALEYUV2YV121

  214.                :: "r" (src[p]), "r" (dst[p] + counter[p]),

  215.                   "g" (-counter[p])

  216.                : "%"REG_a

  217.             );

  218.         }

  219.     }

  220. }

  221. 

  222. #define YSCALEYUV2YV121_ACCURATE \

  223.     "mov %2, %%"REG_a"                    \n\t"\

  224.     "movq               0(%3), %%mm6      \n\t"\

  225.     "movq               8(%3), %%mm7      \n\t"\

  226.     ".p2align                4            \n\t" /* FIXME Unroll? */\

  227.     "1:                                   \n\t"\

  228.     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\

  229.     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\

  230.     "paddsw             %%mm6, %%mm0      \n\t"\

  231.     "paddsw             %%mm7, %%mm1      \n\t"\

  232.     "psraw                 $7, %%mm0      \n\t"\

  233.     "psraw                 $7, %%mm1      \n\t"\

  234.     "packuswb           %%mm1, %%mm0      \n\t"\

  235.     MOVNTQ(%%mm0, (%1, %%REGa))\

  236.     "add                   $8, %%"REG_a"  \n\t"\

  237.     "jnc                   1b             \n\t"

  238. 

  239. static inline void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,

  240.                                        const int16_t *chrUSrc, const int16_t *chrVSrc,

  241.                                        const int16_t *alpSrc,

  242.                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,

  243.                                        uint8_t *aDest, int dstW, int chrDstW,

  244.                                        const uint8_t *lumDither, const uint8_t *chrDither)

  245. {

  246.     int p= 4;

  247.     const int16_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };

  248.     uint8_t *dst[4]= { aDest, dest, uDest, vDest };

  249.     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };

  250. 

  251.     while (p--) {

  252.         if (dst[p]) {

  253.             int i;

  254.             for(i=0; i<8; i++) c->dither16[i] = i<2 ? lumDither[i] : chrDither[i];

  255.             __asm__ volatile(

  256.                 YSCALEYUV2YV121_ACCURATE

  257.                 :: "r" (src[p]), "r" (dst[p] + counter[p]),

  258.                    "g" (-counter[p]), "r"(c->dither16)

  259.                 : "%"REG_a

  260.             );

  261.         }

  262.     }

  263. }

  264. 

  265. #define YSCALEYUV2PACKEDX_UV \

  266.     __asm__ volatile(\

  267.         "xor                   %%"REG_a", %%"REG_a"     \n\t"\

  268.         ".p2align                      4                \n\t"\

  269.         "nop                                            \n\t"\

  270.         "1:                                             \n\t"\

  271.         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\

  272.         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  273.         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\

  274.         "movq                      %%mm3, %%mm4         \n\t"\

  275.         ".p2align                      4                \n\t"\

  276.         "2:                                             \n\t"\

  277.         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\

  278.         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\

  279.         "add                          %6, %%"REG_S"     \n\t" \

  280.         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\

  281.         "add                         $16, %%"REG_d"     \n\t"\

  282.         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  283.         "pmulhw                    %%mm0, %%mm2         \n\t"\

  284.         "pmulhw                    %%mm0, %%mm5         \n\t"\

  285.         "paddw                     %%mm2, %%mm3         \n\t"\

  286.         "paddw                     %%mm5, %%mm4         \n\t"\

  287.         "test                  %%"REG_S", %%"REG_S"     \n\t"\

  288.         " jnz                         2b                \n\t"\

  289. 

  290. #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \

  291.     "lea                "offset"(%0), %%"REG_d"     \n\t"\

  292.     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  293.     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\

  294.     "movq                    "#dst1", "#dst2"       \n\t"\

  295.     ".p2align                      4                \n\t"\

  296.     "2:                                             \n\t"\

  297.     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\

  298.     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\

  299.     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\

  300.     "add                         $16, %%"REG_d"            \n\t"\

  301.     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  302.     "pmulhw                 "#coeff", "#src1"       \n\t"\

  303.     "pmulhw                 "#coeff", "#src2"       \n\t"\

  304.     "paddw                   "#src1", "#dst1"       \n\t"\

  305.     "paddw                   "#src2", "#dst2"       \n\t"\

  306.     "test                  %%"REG_S", %%"REG_S"     \n\t"\

  307.     " jnz                         2b                \n\t"\

  308. 

  309. #define YSCALEYUV2PACKEDX \

  310.     YSCALEYUV2PACKEDX_UV \

  311.     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \

  312. 

  313. #define YSCALEYUV2PACKEDX_END                     \

  314.         :: "r" (&c->redDither),                   \

  315.             "m" (dummy), "m" (dummy), "m" (dummy),\

  316.             "r" (dest), "m" (dstW_reg), "m"(uv_off) \

  317.         : "%"REG_a, "%"REG_d, "%"REG_S            \

  318.     );

  319. 

  320. #define YSCALEYUV2PACKEDX_ACCURATE_UV \

  321.     __asm__ volatile(\

  322.         "xor %%"REG_a", %%"REG_a"                       \n\t"\

  323.         ".p2align                      4                \n\t"\

  324.         "nop                                            \n\t"\

  325.         "1:                                             \n\t"\

  326.         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\

  327.         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  328.         "pxor                      %%mm4, %%mm4         \n\t"\

  329.         "pxor                      %%mm5, %%mm5         \n\t"\

  330.         "pxor                      %%mm6, %%mm6         \n\t"\

  331.         "pxor                      %%mm7, %%mm7         \n\t"\

  332.         ".p2align                      4                \n\t"\

  333.         "2:                                             \n\t"\

  334.         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\

  335.         "add                          %6, %%"REG_S"      \n\t" \

  336.         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\

  337.         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\

  338.         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\

  339.         "movq                      %%mm0, %%mm3         \n\t"\

  340.         "punpcklwd                 %%mm1, %%mm0         \n\t"\

  341.         "punpckhwd                 %%mm1, %%mm3         \n\t"\

  342.         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\

  343.         "pmaddwd                   %%mm1, %%mm0         \n\t"\

  344.         "pmaddwd                   %%mm1, %%mm3         \n\t"\

  345.         "paddd                     %%mm0, %%mm4         \n\t"\

  346.         "paddd                     %%mm3, %%mm5         \n\t"\

  347.         "add                          %6, %%"REG_S"      \n\t" \

  348.         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\

  349.         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\

  350.         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\

  351.         "test                  %%"REG_S", %%"REG_S"     \n\t"\

  352.         "movq                      %%mm2, %%mm0         \n\t"\

  353.         "punpcklwd                 %%mm3, %%mm2         \n\t"\

  354.         "punpckhwd                 %%mm3, %%mm0         \n\t"\

  355.         "pmaddwd                   %%mm1, %%mm2         \n\t"\

  356.         "pmaddwd                   %%mm1, %%mm0         \n\t"\

  357.         "paddd                     %%mm2, %%mm6         \n\t"\

  358.         "paddd                     %%mm0, %%mm7         \n\t"\

  359.         " jnz                         2b                \n\t"\

  360.         "psrad                       $16, %%mm4         \n\t"\

  361.         "psrad                       $16, %%mm5         \n\t"\

  362.         "psrad                       $16, %%mm6         \n\t"\

  363.         "psrad                       $16, %%mm7         \n\t"\

  364.         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\

  365.         "packssdw                  %%mm5, %%mm4         \n\t"\

  366.         "packssdw                  %%mm7, %%mm6         \n\t"\

  367.         "paddw                     %%mm0, %%mm4         \n\t"\

  368.         "paddw                     %%mm0, %%mm6         \n\t"\

  369.         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\

  370.         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\

  371. 

  372. #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \

  373.     "lea                "offset"(%0), %%"REG_d"     \n\t"\

  374.     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\

  375.     "pxor                      %%mm1, %%mm1         \n\t"\

  376.     "pxor                      %%mm5, %%mm5         \n\t"\

  377.     "pxor                      %%mm7, %%mm7         \n\t"\

  378.     "pxor                      %%mm6, %%mm6         \n\t"\

  379.     ".p2align                      4                \n\t"\

  380.     "2:                                             \n\t"\

  381.     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\

  382.     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\

  383.     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\

  384.     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\

  385.     "movq                      %%mm0, %%mm3         \n\t"\

  386.     "punpcklwd                 %%mm4, %%mm0         \n\t"\

  387.     "punpckhwd                 %%mm4, %%mm3         \n\t"\

  388.     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\

  389.     "pmaddwd                   %%mm4, %%mm0         \n\t"\

  390.     "pmaddwd                   %%mm4, %%mm3         \n\t"\

  391.     "paddd                     %%mm0, %%mm1         \n\t"\

  392.     "paddd                     %%mm3, %%mm5         \n\t"\

  393.     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\

  394.     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\

  395.     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\

  396.     "test                  %%"REG_S", %%"REG_S"     \n\t"\

  397.     "movq                      %%mm2, %%mm0         \n\t"\

  398.     "punpcklwd                 %%mm3, %%mm2         \n\t"\

  399.     "punpckhwd                 %%mm3, %%mm0         \n\t"\

  400.     "pmaddwd                   %%mm4, %%mm2         \n\t"\

  401.     "pmaddwd                   %%mm4, %%mm0         \n\t"\

  402.     "paddd                     %%mm2, %%mm7         \n\t"\

  403.     "paddd                     %%mm0, %%mm6         \n\t"\

  404.     " jnz                         2b                \n\t"\

  405.     "psrad                       $16, %%mm1         \n\t"\

  406.     "psrad                       $16, %%mm5         \n\t"\

  407.     "psrad                       $16, %%mm7         \n\t"\

  408.     "psrad                       $16, %%mm6         \n\t"\

  409.     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\

  410.     "packssdw                  %%mm5, %%mm1         \n\t"\

  411.     "packssdw                  %%mm6, %%mm7         \n\t"\

  412.     "paddw                     %%mm0, %%mm1         \n\t"\

  413.     "paddw                     %%mm0, %%mm7         \n\t"\

  414.     "movq               "U_TEMP"(%0), %%mm3         \n\t"\

  415.     "movq               "V_TEMP"(%0), %%mm4         \n\t"\

  416. 

  417. #define YSCALEYUV2PACKEDX_ACCURATE \

  418.     YSCALEYUV2PACKEDX_ACCURATE_UV \

  419.     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)

  420. 

  421. #define YSCALEYUV2RGBX \

  422.     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\

  423.     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\

  424.     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\

  425.     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\

  426.     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\

  427.     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\

  428.     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\

  429.     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\

  430.     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\

  431.     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\

  432.     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\

  433.     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\

  434.     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\

  435.     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\

  436.     "paddw           %%mm3, %%mm4       \n\t"\

  437.     "movq            %%mm2, %%mm0       \n\t"\

  438.     "movq            %%mm5, %%mm6       \n\t"\

  439.     "movq            %%mm4, %%mm3       \n\t"\

  440.     "punpcklwd       %%mm2, %%mm2       \n\t"\

  441.     "punpcklwd       %%mm5, %%mm5       \n\t"\

  442.     "punpcklwd       %%mm4, %%mm4       \n\t"\

  443.     "paddw           %%mm1, %%mm2       \n\t"\

  444.     "paddw           %%mm1, %%mm5       \n\t"\

  445.     "paddw           %%mm1, %%mm4       \n\t"\

  446.     "punpckhwd       %%mm0, %%mm0       \n\t"\

  447.     "punpckhwd       %%mm6, %%mm6       \n\t"\

  448.     "punpckhwd       %%mm3, %%mm3       \n\t"\

  449.     "paddw           %%mm7, %%mm0       \n\t"\

  450.     "paddw           %%mm7, %%mm6       \n\t"\

  451.     "paddw           %%mm7, %%mm3       \n\t"\

  452.     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\

  453.     "packuswb        %%mm0, %%mm2       \n\t"\

  454.     "packuswb        %%mm6, %%mm5       \n\t"\

  455.     "packuswb        %%mm3, %%mm4       \n\t"\

  456. 

  457. #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \

  458.     "movq       "#b", "#q2"     \n\t" /* B */\

  459.     "movq       "#r", "#t"      \n\t" /* R */\

  460.     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\

  461.     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\

  462.     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\

  463.     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\

  464.     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\

  465.     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\

  466.     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\

  467.     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\

  468.     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\

  469.     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\

  470. \

  471.     MOVNTQ(   q0,   (dst, index, 4))\

  472.     MOVNTQ(    b,  8(dst, index, 4))\

  473.     MOVNTQ(   q2, 16(dst, index, 4))\

  474.     MOVNTQ(   q3, 24(dst, index, 4))\

  475. \

  476.     "add      $8, "#index"      \n\t"\

  477.     "cmp "#dstw", "#index"      \n\t"\

  478.     " jb      1b                \n\t"

  479. #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)

  480. 

  481. static inline void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,

  482.                                           const int16_t **lumSrc, int lumFilterSize,

  483.                                           const int16_t *chrFilter, const int16_t **chrUSrc,

  484.                                           const int16_t **chrVSrc,

  485.                                           int chrFilterSize, const int16_t **alpSrc,

  486.                                           uint8_t *dest, int dstW, int dstY)

  487. {

  488.     x86_reg dummy=0;

  489.     x86_reg dstW_reg = dstW;

  490.     x86_reg uv_off = c->uv_off << 1;

  491. 

  492.     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {

  493.         YSCALEYUV2PACKEDX_ACCURATE

  494.         YSCALEYUV2RGBX

  495.         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"

  496.         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"

  497.         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"

  498.         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)

  499.         "movq               "Y_TEMP"(%0), %%mm5         \n\t"

  500.         "psraw                        $3, %%mm1         \n\t"

  501.         "psraw                        $3, %%mm7         \n\t"

  502.         "packuswb                  %%mm7, %%mm1         \n\t"

  503.         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)

  504.         YSCALEYUV2PACKEDX_END

  505.     } else {

  506.         YSCALEYUV2PACKEDX_ACCURATE

  507.         YSCALEYUV2RGBX

  508.         "pcmpeqd %%mm7, %%mm7 \n\t"

  509.         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  510.         YSCALEYUV2PACKEDX_END

  511.     }

  512. }

  513. 

  514. static inline void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,

  515.                                        const int16_t **lumSrc, int lumFilterSize,

  516.                                        const int16_t *chrFilter, const int16_t **chrUSrc,

  517.                                        const int16_t **chrVSrc,

  518.                                        int chrFilterSize, const int16_t **alpSrc,

  519.                                        uint8_t *dest, int dstW, int dstY)

  520. {

  521.     x86_reg dummy=0;

  522.     x86_reg dstW_reg = dstW;

  523.     x86_reg uv_off = c->uv_off << 1;

  524. 

  525.     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {

  526.         YSCALEYUV2PACKEDX

  527.         YSCALEYUV2RGBX

  528.         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)

  529.         "psraw                        $3, %%mm1         \n\t"

  530.         "psraw                        $3, %%mm7         \n\t"

  531.         "packuswb                  %%mm7, %%mm1         \n\t"

  532.         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)

  533.         YSCALEYUV2PACKEDX_END

  534.     } else {

  535.         YSCALEYUV2PACKEDX

  536.         YSCALEYUV2RGBX

  537.         "pcmpeqd %%mm7, %%mm7 \n\t"

  538.         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  539.         YSCALEYUV2PACKEDX_END

  540.     }

  541. }

  542. 

  543. #define REAL_WRITERGB16(dst, dstw, index) \

  544.     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\

  545.     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\

  546.     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\

  547.     "psrlq           $3, %%mm2  \n\t"\

  548. \

  549.     "movq         %%mm2, %%mm1  \n\t"\

  550.     "movq         %%mm4, %%mm3  \n\t"\

  551. \

  552.     "punpcklbw    %%mm7, %%mm3  \n\t"\

  553.     "punpcklbw    %%mm5, %%mm2  \n\t"\

  554.     "punpckhbw    %%mm7, %%mm4  \n\t"\

  555.     "punpckhbw    %%mm5, %%mm1  \n\t"\

  556. \

  557.     "psllq           $3, %%mm3  \n\t"\

  558.     "psllq           $3, %%mm4  \n\t"\

  559. \

  560.     "por          %%mm3, %%mm2  \n\t"\

  561.     "por          %%mm4, %%mm1  \n\t"\

  562. \

  563.     MOVNTQ(%%mm2,  (dst, index, 2))\

  564.     MOVNTQ(%%mm1, 8(dst, index, 2))\

  565. \

  566.     "add             $8, "#index"   \n\t"\

  567.     "cmp        "#dstw", "#index"   \n\t"\

  568.     " jb             1b             \n\t"

  569. #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)

  570. 

  571. static inline void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,

  572.                                            const int16_t **lumSrc, int lumFilterSize,

  573.                                            const int16_t *chrFilter, const int16_t **chrUSrc,

  574.                                            const int16_t **chrVSrc,

  575.                                            int chrFilterSize, const int16_t **alpSrc,

  576.                                            uint8_t *dest, int dstW, int dstY)

  577. {

  578.     x86_reg dummy=0;

  579.     x86_reg dstW_reg = dstW;

  580.     x86_reg uv_off = c->uv_off << 1;

  581. 

  582.     YSCALEYUV2PACKEDX_ACCURATE

  583.     YSCALEYUV2RGBX

  584.     "pxor %%mm7, %%mm7 \n\t"

  585.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  586. #ifdef DITHER1XBPP

  587.     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"

  588.     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"

  589.     "paddusb "RED_DITHER"(%0), %%mm5\n\t"

  590. #endif

  591.     WRITERGB16(%4, %5, %%REGa)

  592.     YSCALEYUV2PACKEDX_END

  593. }

  594. 

  595. static inline void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,

  596.                                         const int16_t **lumSrc, int lumFilterSize,

  597.                                         const int16_t *chrFilter, const int16_t **chrUSrc,

  598.                                         const int16_t **chrVSrc,

  599.                                         int chrFilterSize, const int16_t **alpSrc,

  600.                                         uint8_t *dest, int dstW, int dstY)

  601. {

  602.     x86_reg dummy=0;

  603.     x86_reg dstW_reg = dstW;

  604.     x86_reg uv_off = c->uv_off << 1;

  605. 

  606.     YSCALEYUV2PACKEDX

  607.     YSCALEYUV2RGBX

  608.     "pxor %%mm7, %%mm7 \n\t"

  609.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  610. #ifdef DITHER1XBPP

  611.     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"

  612.     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"

  613.     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"

  614. #endif

  615.     WRITERGB16(%4, %5, %%REGa)

  616.     YSCALEYUV2PACKEDX_END

  617. }

  618. 

  619. #define REAL_WRITERGB15(dst, dstw, index) \

  620.     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\

  621.     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\

  622.     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\

  623.     "psrlq           $3, %%mm2  \n\t"\

  624.     "psrlq           $1, %%mm5  \n\t"\

  625. \

  626.     "movq         %%mm2, %%mm1  \n\t"\

  627.     "movq         %%mm4, %%mm3  \n\t"\

  628. \

  629.     "punpcklbw    %%mm7, %%mm3  \n\t"\

  630.     "punpcklbw    %%mm5, %%mm2  \n\t"\

  631.     "punpckhbw    %%mm7, %%mm4  \n\t"\

  632.     "punpckhbw    %%mm5, %%mm1  \n\t"\

  633. \

  634.     "psllq           $2, %%mm3  \n\t"\

  635.     "psllq           $2, %%mm4  \n\t"\

  636. \

  637.     "por          %%mm3, %%mm2  \n\t"\

  638.     "por          %%mm4, %%mm1  \n\t"\

  639. \

  640.     MOVNTQ(%%mm2,  (dst, index, 2))\

  641.     MOVNTQ(%%mm1, 8(dst, index, 2))\

  642. \

  643.     "add             $8, "#index"   \n\t"\

  644.     "cmp        "#dstw", "#index"   \n\t"\

  645.     " jb             1b             \n\t"

  646. #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)

  647. 

  648. static inline void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,

  649.                                            const int16_t **lumSrc, int lumFilterSize,

  650.                                            const int16_t *chrFilter, const int16_t **chrUSrc,

  651.                                            const int16_t **chrVSrc,

  652.                                            int chrFilterSize, const int16_t **alpSrc,

  653.                                            uint8_t *dest, int dstW, int dstY)

  654. {

  655.     x86_reg dummy=0;

  656.     x86_reg dstW_reg = dstW;

  657.     x86_reg uv_off = c->uv_off << 1;

  658. 

  659.     YSCALEYUV2PACKEDX_ACCURATE

  660.     YSCALEYUV2RGBX

  661.     "pxor %%mm7, %%mm7 \n\t"

  662.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  663. #ifdef DITHER1XBPP

  664.     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"

  665.     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"

  666.     "paddusb "RED_DITHER"(%0), %%mm5\n\t"

  667. #endif

  668.     WRITERGB15(%4, %5, %%REGa)

  669.     YSCALEYUV2PACKEDX_END

  670. }

  671. 

  672. static inline void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,

  673.                                         const int16_t **lumSrc, int lumFilterSize,

  674.                                         const int16_t *chrFilter, const int16_t **chrUSrc,

  675.                                         const int16_t **chrVSrc,

  676.                                         int chrFilterSize, const int16_t **alpSrc,

  677.                                         uint8_t *dest, int dstW, int dstY)

  678. {

  679.     x86_reg dummy=0;

  680.     x86_reg dstW_reg = dstW;

  681.     x86_reg uv_off = c->uv_off << 1;

  682. 

  683.     YSCALEYUV2PACKEDX

  684.     YSCALEYUV2RGBX

  685.     "pxor %%mm7, %%mm7 \n\t"

  686.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  687. #ifdef DITHER1XBPP

  688.     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"

  689.     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"

  690.     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"

  691. #endif

  692.     WRITERGB15(%4, %5, %%REGa)

  693.     YSCALEYUV2PACKEDX_END

  694. }

  695. 

  696. #define WRITEBGR24MMX(dst, dstw, index) \

  697.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\

  698.     "movq      %%mm2, %%mm1     \n\t" /* B */\

  699.     "movq      %%mm5, %%mm6     \n\t" /* R */\

  700.     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\

  701.     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\

  702.     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\

  703.     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\

  704.     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\

  705.     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\

  706.     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\

  707.     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\

  708.     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\

  709.     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\

  710. \

  711.     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\

  712.     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\

  713.     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\

  714.     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\

  715. \

  716.     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\

  717.     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\

  718.     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\

  719.     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\

  720. \

  721.     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\

  722.     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\

  723.     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\

  724.     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\

  725. \

  726.     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\

  727.     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\

  728.     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\

  729.     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\

  730.     MOVNTQ(%%mm0, (dst))\

  731. \

  732.     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\

  733.     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\

  734.     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\

  735.     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\

  736.     MOVNTQ(%%mm6, 8(dst))\

  737. \

  738.     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\

  739.     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\

  740.     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\

  741.     MOVNTQ(%%mm5, 16(dst))\

  742. \

  743.     "add         $24, "#dst"    \n\t"\

  744. \

  745.     "add          $8, "#index"  \n\t"\

  746.     "cmp     "#dstw", "#index"  \n\t"\

  747.     " jb          1b            \n\t"

  748. 

  749. #define WRITEBGR24MMX2(dst, dstw, index) \

  750.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\

  751.     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\

  752.     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\

  753.     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\

  754.     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\

  755.     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\

  756. \

  757.     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\

  758.     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\

  759.     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\

  760. \

  761.     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\

  762.     "por    %%mm1, %%mm6        \n\t"\

  763.     "por    %%mm3, %%mm6        \n\t"\

  764.     MOVNTQ(%%mm6, (dst))\

  765. \

  766.     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\

  767.     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\

  768.     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\

  769.     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\

  770. \

  771.     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\

  772.     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\

  773.     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\

  774. \

  775.     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\

  776.     "por    %%mm3, %%mm6        \n\t"\

  777.     MOVNTQ(%%mm6, 8(dst))\

  778. \

  779.     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\

  780.     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\

  781.     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\

  782. \

  783.     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\

  784.     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\

  785.     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\

  786. \

  787.     "por    %%mm1, %%mm3        \n\t"\

  788.     "por    %%mm3, %%mm6        \n\t"\

  789.     MOVNTQ(%%mm6, 16(dst))\

  790. \

  791.     "add      $24, "#dst"       \n\t"\

  792. \

  793.     "add       $8, "#index"     \n\t"\

  794.     "cmp  "#dstw", "#index"     \n\t"\

  795.     " jb       1b               \n\t"

  796. 

  797. #if COMPILE_TEMPLATE_MMX2

  798. #undef WRITEBGR24

  799. #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)

  800. #else

  801. #undef WRITEBGR24

  802. #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)

  803. #endif

  804. 

  805. static inline void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,

  806.                                           const int16_t **lumSrc, int lumFilterSize,

  807.                                           const int16_t *chrFilter, const int16_t **chrUSrc,

  808.                                           const int16_t **chrVSrc,

  809.                                           int chrFilterSize, const int16_t **alpSrc,

  810.                                           uint8_t *dest, int dstW, int dstY)

  811. {

  812.     x86_reg dummy=0;

  813.     x86_reg dstW_reg = dstW;

  814.     x86_reg uv_off = c->uv_off << 1;

  815. 

  816.     YSCALEYUV2PACKEDX_ACCURATE

  817.     YSCALEYUV2RGBX

  818.     "pxor %%mm7, %%mm7 \n\t"

  819.     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize

  820.     "add %4, %%"REG_c"                        \n\t"

  821.     WRITEBGR24(%%REGc, %5, %%REGa)

  822.     :: "r" (&c->redDither),

  823.        "m" (dummy), "m" (dummy), "m" (dummy),

  824.        "r" (dest), "m" (dstW_reg), "m"(uv_off)

  825.     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S

  826.     );

  827. }

  828. 

  829. static inline void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,

  830.                                        const int16_t **lumSrc, int lumFilterSize,

  831.                                        const int16_t *chrFilter, const int16_t **chrUSrc,

  832.                                        const int16_t **chrVSrc,

  833.                                        int chrFilterSize, const int16_t **alpSrc,

  834.                                        uint8_t *dest, int dstW, int dstY)

  835. {

  836.     x86_reg dummy=0;

  837.     x86_reg dstW_reg = dstW;

  838.     x86_reg uv_off = c->uv_off << 1;

  839. 

  840.     YSCALEYUV2PACKEDX

  841.     YSCALEYUV2RGBX

  842.     "pxor                    %%mm7, %%mm7       \n\t"

  843.     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize

  844.     "add                        %4, %%"REG_c"   \n\t"

  845.     WRITEBGR24(%%REGc, %5, %%REGa)

  846.     :: "r" (&c->redDither),

  847.        "m" (dummy), "m" (dummy), "m" (dummy),

  848.        "r" (dest),  "m" (dstW_reg), "m"(uv_off)

  849.     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S

  850.     );

  851. }

  852. 

  853. #define REAL_WRITEYUY2(dst, dstw, index) \

  854.     "packuswb  %%mm3, %%mm3     \n\t"\

  855.     "packuswb  %%mm4, %%mm4     \n\t"\

  856.     "packuswb  %%mm7, %%mm1     \n\t"\

  857.     "punpcklbw %%mm4, %%mm3     \n\t"\

  858.     "movq      %%mm1, %%mm7     \n\t"\

  859.     "punpcklbw %%mm3, %%mm1     \n\t"\

  860.     "punpckhbw %%mm3, %%mm7     \n\t"\

  861. \

  862.     MOVNTQ(%%mm1, (dst, index, 2))\

  863.     MOVNTQ(%%mm7, 8(dst, index, 2))\

  864. \

  865.     "add          $8, "#index"  \n\t"\

  866.     "cmp     "#dstw", "#index"  \n\t"\

  867.     " jb          1b            \n\t"

  868. #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)

  869. 

  870. static inline void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,

  871.                                             const int16_t **lumSrc, int lumFilterSize,

  872.                                             const int16_t *chrFilter, const int16_t **chrUSrc,

  873.                                             const int16_t **chrVSrc,

  874.                                             int chrFilterSize, const int16_t **alpSrc,

  875.                                             uint8_t *dest, int dstW, int dstY)

  876. {

  877.     x86_reg dummy=0;

  878.     x86_reg dstW_reg = dstW;

  879.     x86_reg uv_off = c->uv_off << 1;

  880. 

  881.     YSCALEYUV2PACKEDX_ACCURATE

  882.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  883.     "psraw $3, %%mm3    \n\t"

  884.     "psraw $3, %%mm4    \n\t"

  885.     "psraw $3, %%mm1    \n\t"

  886.     "psraw $3, %%mm7    \n\t"

  887.     WRITEYUY2(%4, %5, %%REGa)

  888.     YSCALEYUV2PACKEDX_END

  889. }

  890. 

  891. static inline void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,

  892.                                          const int16_t **lumSrc, int lumFilterSize,

  893.                                          const int16_t *chrFilter, const int16_t **chrUSrc,

  894.                                          const int16_t **chrVSrc,

  895.                                          int chrFilterSize, const int16_t **alpSrc,

  896.                                          uint8_t *dest, int dstW, int dstY)

  897. {

  898.     x86_reg dummy=0;

  899.     x86_reg dstW_reg = dstW;

  900.     x86_reg uv_off = c->uv_off << 1;

  901. 

  902.     YSCALEYUV2PACKEDX

  903.     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  904.     "psraw $3, %%mm3    \n\t"

  905.     "psraw $3, %%mm4    \n\t"

  906.     "psraw $3, %%mm1    \n\t"

  907.     "psraw $3, %%mm7    \n\t"

  908.     WRITEYUY2(%4, %5, %%REGa)

  909.     YSCALEYUV2PACKEDX_END

  910. }

  911. 

  912. #define REAL_YSCALEYUV2RGB_UV(index, c) \

  913.     "xor            "#index", "#index"  \n\t"\

  914.     ".p2align              4            \n\t"\

  915.     "1:                                 \n\t"\

  916.     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\

  917.     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\

  918.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  919.     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\

  920.     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\

  921.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  922.     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\

  923.     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\

  924.     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\

  925.     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\

  926.     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\

  927.     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\

  928.     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\

  929.     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\

  930.     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\

  931.     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\

  932.     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\

  933.     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\

  934.     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\

  935.     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\

  936.     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\

  937.     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\

  938. 

  939. #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \

  940.     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\

  941.     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\

  942.     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\

  943.     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\

  944.     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\

  945.     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\

  946.     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\

  947.     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\

  948.     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  949.     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  950.     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\

  951.     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\

  952. 

  953. #define REAL_YSCALEYUV2RGB_COEFF(c) \

  954.     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\

  955.     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\

  956.     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\

  957.     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\

  958.     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\

  959.     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\

  960.     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\

  961.     "paddw             %%mm3, %%mm4     \n\t"\

  962.     "movq              %%mm2, %%mm0     \n\t"\

  963.     "movq              %%mm5, %%mm6     \n\t"\

  964.     "movq              %%mm4, %%mm3     \n\t"\

  965.     "punpcklwd         %%mm2, %%mm2     \n\t"\

  966.     "punpcklwd         %%mm5, %%mm5     \n\t"\

  967.     "punpcklwd         %%mm4, %%mm4     \n\t"\

  968.     "paddw             %%mm1, %%mm2     \n\t"\

  969.     "paddw             %%mm1, %%mm5     \n\t"\

  970.     "paddw             %%mm1, %%mm4     \n\t"\

  971.     "punpckhwd         %%mm0, %%mm0     \n\t"\

  972.     "punpckhwd         %%mm6, %%mm6     \n\t"\

  973.     "punpckhwd         %%mm3, %%mm3     \n\t"\

  974.     "paddw             %%mm7, %%mm0     \n\t"\

  975.     "paddw             %%mm7, %%mm6     \n\t"\

  976.     "paddw             %%mm7, %%mm3     \n\t"\

  977.     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\

  978.     "packuswb          %%mm0, %%mm2     \n\t"\

  979.     "packuswb          %%mm6, %%mm5     \n\t"\

  980.     "packuswb          %%mm3, %%mm4     \n\t"\

  981. 

  982. #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)

  983. 

  984. #define YSCALEYUV2RGB(index, c) \

  985.     REAL_YSCALEYUV2RGB_UV(index, c) \

  986.     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \

  987.     REAL_YSCALEYUV2RGB_COEFF(c)

  988. 

  989. /**

  990.  * vertical bilinear scale YV12 to RGB

  991.  */

  992. static inline void RENAME(yuv2rgb32_2)(SwsContext *c, const uint16_t *buf0,

  993.                                        const uint16_t *buf1, const uint16_t *ubuf0,

  994.                                        const uint16_t *ubuf1, const uint16_t *vbuf0,

  995.                                        const uint16_t *vbuf1, const uint16_t *abuf0,

  996.                                        const uint16_t *abuf1, uint8_t *dest,

  997.                                        int dstW, int yalpha, int uvalpha, int y)

  998. {

  999.     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {

  1000. #if ARCH_X86_64

  1001.         __asm__ volatile(

  1002.             YSCALEYUV2RGB(%%r8, %5)

  1003.             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)

  1004.             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/

  1005.             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/

  1006.             "packuswb            %%mm7, %%mm1       \n\t"

  1007.             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)

  1008.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),

  1009.                "a" (&c->redDither),

  1010.                "r" (abuf0), "r" (abuf1)

  1011.             : "%r8"

  1012.         );

  1013. #else

  1014.         c->u_temp=(intptr_t)abuf0;

  1015.         c->v_temp=(intptr_t)abuf1;

  1016.         __asm__ volatile(

  1017.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1018.             "mov        %4, %%"REG_b"               \n\t"

  1019.             "push %%"REG_BP"                        \n\t"

  1020.             YSCALEYUV2RGB(%%REGBP, %5)

  1021.             "push                   %0              \n\t"

  1022.             "push                   %1              \n\t"

  1023.             "mov          "U_TEMP"(%5), %0          \n\t"

  1024.             "mov          "V_TEMP"(%5), %1          \n\t"

  1025.             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)

  1026.             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/

  1027.             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/

  1028.             "packuswb            %%mm7, %%mm1       \n\t"

  1029.             "pop                    %1              \n\t"

  1030.             "pop                    %0              \n\t"

  1031.             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)

  1032.             "pop %%"REG_BP"                         \n\t"

  1033.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1034.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1035.                "a" (&c->redDither)

  1036.         );

  1037. #endif

  1038.     } else {

  1039.         __asm__ volatile(

  1040.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1041.             "mov        %4, %%"REG_b"               \n\t"

  1042.             "push %%"REG_BP"                        \n\t"

  1043.             YSCALEYUV2RGB(%%REGBP, %5)

  1044.             "pcmpeqd %%mm7, %%mm7                   \n\t"

  1045.             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  1046.             "pop %%"REG_BP"                         \n\t"

  1047.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1048.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1049.                "a" (&c->redDither)

  1050.         );

  1051.     }

  1052. }

  1053. 

  1054. static inline void RENAME(yuv2bgr24_2)(SwsContext *c, const uint16_t *buf0,

  1055.                                        const uint16_t *buf1, const uint16_t *ubuf0,

  1056.                                        const uint16_t *ubuf1, const uint16_t *vbuf0,

  1057.                                        const uint16_t *vbuf1, const uint16_t *abuf0,

  1058.                                        const uint16_t *abuf1, uint8_t *dest,

  1059.                                        int dstW, int yalpha, int uvalpha, int y)

  1060. {

  1061.     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(

  1062.     __asm__ volatile(

  1063.         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1064.         "mov        %4, %%"REG_b"               \n\t"

  1065.         "push %%"REG_BP"                        \n\t"

  1066.         YSCALEYUV2RGB(%%REGBP, %5)

  1067.         "pxor    %%mm7, %%mm7                   \n\t"

  1068.         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)

  1069.         "pop %%"REG_BP"                         \n\t"

  1070.         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1071.         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1072.            "a" (&c->redDither)

  1073.     );

  1074. }

  1075. 

  1076. static inline void RENAME(yuv2rgb555_2)(SwsContext *c, const uint16_t *buf0,

  1077.                                         const uint16_t *buf1, const uint16_t *ubuf0,

  1078.                                         const uint16_t *ubuf1, const uint16_t *vbuf0,

  1079.                                         const uint16_t *vbuf1, const uint16_t *abuf0,

  1080.                                         const uint16_t *abuf1, uint8_t *dest,

  1081.                                         int dstW, int yalpha, int uvalpha, int y)

  1082. {

  1083.     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(

  1084.     __asm__ volatile(

  1085.         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1086.         "mov        %4, %%"REG_b"               \n\t"

  1087.         "push %%"REG_BP"                        \n\t"

  1088.         YSCALEYUV2RGB(%%REGBP, %5)

  1089.         "pxor    %%mm7, %%mm7                   \n\t"

  1090.         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1091. #ifdef DITHER1XBPP

  1092.         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1093.         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1094.         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1095. #endif

  1096.         WRITERGB15(%%REGb, 8280(%5), %%REGBP)

  1097.         "pop %%"REG_BP"                         \n\t"

  1098.         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1099.         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1100.            "a" (&c->redDither)

  1101.     );

  1102. }

  1103. 

  1104. static inline void RENAME(yuv2rgb565_2)(SwsContext *c, const uint16_t *buf0,

  1105.                                         const uint16_t *buf1, const uint16_t *ubuf0,

  1106.                                         const uint16_t *ubuf1, const uint16_t *vbuf0,

  1107.                                         const uint16_t *vbuf1, const uint16_t *abuf0,

  1108.                                         const uint16_t *abuf1, uint8_t *dest,

  1109.                                         int dstW, int yalpha, int uvalpha, int y)

  1110. {

  1111.     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(

  1112.     __asm__ volatile(

  1113.         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1114.         "mov        %4, %%"REG_b"               \n\t"

  1115.         "push %%"REG_BP"                        \n\t"

  1116.         YSCALEYUV2RGB(%%REGBP, %5)

  1117.         "pxor    %%mm7, %%mm7                   \n\t"

  1118.         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1119. #ifdef DITHER1XBPP

  1120.         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1121.         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1122.         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1123. #endif

  1124.         WRITERGB16(%%REGb, 8280(%5), %%REGBP)

  1125.         "pop %%"REG_BP"                         \n\t"

  1126.         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1127.         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1128.            "a" (&c->redDither)

  1129.     );

  1130. }

  1131. 

  1132. #define REAL_YSCALEYUV2PACKED(index, c) \

  1133.     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\

  1134.     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\

  1135.     "psraw                $3, %%mm0                           \n\t"\

  1136.     "psraw                $3, %%mm1                           \n\t"\

  1137.     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\

  1138.     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\

  1139.     "xor            "#index", "#index"                        \n\t"\

  1140.     ".p2align              4            \n\t"\

  1141.     "1:                                 \n\t"\

  1142.     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\

  1143.     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\

  1144.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1145.     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\

  1146.     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\

  1147.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1148.     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\

  1149.     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\

  1150.     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\

  1151.     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\

  1152.     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\

  1153.     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\

  1154.     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\

  1155.     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\

  1156.     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\

  1157.     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\

  1158.     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\

  1159.     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\

  1160.     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\

  1161.     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\

  1162.     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\

  1163.     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\

  1164.     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\

  1165.     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1166.     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1167.     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\

  1168.     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\

  1169. 

  1170. #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)

  1171. 

  1172. static inline void RENAME(yuv2yuyv422_2)(SwsContext *c, const uint16_t *buf0,

  1173.                                          const uint16_t *buf1, const uint16_t *ubuf0,

  1174.                                          const uint16_t *ubuf1, const uint16_t *vbuf0,

  1175.                                          const uint16_t *vbuf1, const uint16_t *abuf0,

  1176.                                          const uint16_t *abuf1, uint8_t *dest,

  1177.                                          int dstW, int yalpha, int uvalpha, int y)

  1178. {

  1179.     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(

  1180.     __asm__ volatile(

  1181.         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1182.         "mov %4, %%"REG_b"                        \n\t"

  1183.         "push %%"REG_BP"                        \n\t"

  1184.         YSCALEYUV2PACKED(%%REGBP, %5)

  1185.         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)

  1186.         "pop %%"REG_BP"                         \n\t"

  1187.         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1188.         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1189.            "a" (&c->redDither)

  1190.     );

  1191. }

  1192. 

  1193. #define REAL_YSCALEYUV2RGB1(index, c) \

  1194.     "xor            "#index", "#index"  \n\t"\

  1195.     ".p2align              4            \n\t"\

  1196.     "1:                                 \n\t"\

  1197.     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\

  1198.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1199.     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\

  1200.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1201.     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\

  1202.     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\

  1203.     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\

  1204.     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\

  1205.     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\

  1206.     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\

  1207.     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\

  1208.     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\

  1209.     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\

  1210.     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\

  1211.     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\

  1212.     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1213.     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1214.     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\

  1215.     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\

  1216.     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\

  1217.     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\

  1218.     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\

  1219.     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\

  1220.     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\

  1221.     "paddw             %%mm3, %%mm4     \n\t"\

  1222.     "movq              %%mm2, %%mm0     \n\t"\

  1223.     "movq              %%mm5, %%mm6     \n\t"\

  1224.     "movq              %%mm4, %%mm3     \n\t"\

  1225.     "punpcklwd         %%mm2, %%mm2     \n\t"\

  1226.     "punpcklwd         %%mm5, %%mm5     \n\t"\

  1227.     "punpcklwd         %%mm4, %%mm4     \n\t"\

  1228.     "paddw             %%mm1, %%mm2     \n\t"\

  1229.     "paddw             %%mm1, %%mm5     \n\t"\

  1230.     "paddw             %%mm1, %%mm4     \n\t"\

  1231.     "punpckhwd         %%mm0, %%mm0     \n\t"\

  1232.     "punpckhwd         %%mm6, %%mm6     \n\t"\

  1233.     "punpckhwd         %%mm3, %%mm3     \n\t"\

  1234.     "paddw             %%mm7, %%mm0     \n\t"\

  1235.     "paddw             %%mm7, %%mm6     \n\t"\

  1236.     "paddw             %%mm7, %%mm3     \n\t"\

  1237.     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\

  1238.     "packuswb          %%mm0, %%mm2     \n\t"\

  1239.     "packuswb          %%mm6, %%mm5     \n\t"\

  1240.     "packuswb          %%mm3, %%mm4     \n\t"\

  1241. 

  1242. #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)

  1243. 

  1244. // do vertical chrominance interpolation

  1245. #define REAL_YSCALEYUV2RGB1b(index, c) \

  1246.     "xor            "#index", "#index"  \n\t"\

  1247.     ".p2align              4            \n\t"\

  1248.     "1:                                 \n\t"\

  1249.     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\

  1250.     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\

  1251.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1252.     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\

  1253.     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\

  1254.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1255.     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\

  1256.     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\

  1257.     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\

  1258.     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\

  1259.     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\

  1260.     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\

  1261.     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\

  1262.     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\

  1263.     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\

  1264.     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\

  1265.     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\

  1266.     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\

  1267.     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\

  1268.     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1269.     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\

  1270.     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\

  1271.     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\

  1272.     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\

  1273.     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\

  1274.     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\

  1275.     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\

  1276.     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\

  1277.     "paddw             %%mm3, %%mm4     \n\t"\

  1278.     "movq              %%mm2, %%mm0     \n\t"\

  1279.     "movq              %%mm5, %%mm6     \n\t"\

  1280.     "movq              %%mm4, %%mm3     \n\t"\

  1281.     "punpcklwd         %%mm2, %%mm2     \n\t"\

  1282.     "punpcklwd         %%mm5, %%mm5     \n\t"\

  1283.     "punpcklwd         %%mm4, %%mm4     \n\t"\

  1284.     "paddw             %%mm1, %%mm2     \n\t"\

  1285.     "paddw             %%mm1, %%mm5     \n\t"\

  1286.     "paddw             %%mm1, %%mm4     \n\t"\

  1287.     "punpckhwd         %%mm0, %%mm0     \n\t"\

  1288.     "punpckhwd         %%mm6, %%mm6     \n\t"\

  1289.     "punpckhwd         %%mm3, %%mm3     \n\t"\

  1290.     "paddw             %%mm7, %%mm0     \n\t"\

  1291.     "paddw             %%mm7, %%mm6     \n\t"\

  1292.     "paddw             %%mm7, %%mm3     \n\t"\

  1293.     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\

  1294.     "packuswb          %%mm0, %%mm2     \n\t"\

  1295.     "packuswb          %%mm6, %%mm5     \n\t"\

  1296.     "packuswb          %%mm3, %%mm4     \n\t"\

  1297. 

  1298. #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)

  1299. 

  1300. #define REAL_YSCALEYUV2RGB1_ALPHA(index) \

  1301.     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\

  1302.     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\

  1303.     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\

  1304.     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\

  1305.     "packuswb          %%mm1, %%mm7     \n\t"

  1306. #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)

  1307. 

  1308. /**

  1309.  * YV12 to RGB without scaling or interpolating

  1310.  */

  1311. static inline void RENAME(yuv2rgb32_1)(SwsContext *c, const uint16_t *buf0,

  1312.                                        const uint16_t *ubuf0, const uint16_t *ubuf1,

  1313.                                        const uint16_t *vbuf0, const uint16_t *vbuf1,

  1314.                                        const uint16_t *abuf0, uint8_t *dest,

  1315.                                        int dstW, int uvalpha, enum PixelFormat dstFormat,

  1316.                                        int flags, int y)

  1317. {

  1318.     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1

  1319. 

  1320.     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster

  1321.         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {

  1322.             __asm__ volatile(

  1323.                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1324.                 "mov        %4, %%"REG_b"               \n\t"

  1325.                 "push %%"REG_BP"                        \n\t"

  1326.                 YSCALEYUV2RGB1(%%REGBP, %5)

  1327.                 YSCALEYUV2RGB1_ALPHA(%%REGBP)

  1328.                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  1329.                 "pop %%"REG_BP"                         \n\t"

  1330.                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1331.                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1332.                    "a" (&c->redDither)

  1333.             );

  1334.         } else {

  1335.             __asm__ volatile(

  1336.                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1337.                 "mov        %4, %%"REG_b"               \n\t"

  1338.                 "push %%"REG_BP"                        \n\t"

  1339.                 YSCALEYUV2RGB1(%%REGBP, %5)

  1340.                 "pcmpeqd %%mm7, %%mm7                   \n\t"

  1341.                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  1342.                 "pop %%"REG_BP"                         \n\t"

  1343.                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1344.                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1345.                    "a" (&c->redDither)

  1346.             );

  1347.         }

  1348.     } else {

  1349.         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {

  1350.             __asm__ volatile(

  1351.                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1352.                 "mov        %4, %%"REG_b"               \n\t"

  1353.                 "push %%"REG_BP"                        \n\t"

  1354.                 YSCALEYUV2RGB1b(%%REGBP, %5)

  1355.                 YSCALEYUV2RGB1_ALPHA(%%REGBP)

  1356.                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  1357.                 "pop %%"REG_BP"                         \n\t"

  1358.                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1359.                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1360.                    "a" (&c->redDither)

  1361.             );

  1362.         } else {

  1363.             __asm__ volatile(

  1364.                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1365.                 "mov        %4, %%"REG_b"               \n\t"

  1366.                 "push %%"REG_BP"                        \n\t"

  1367.                 YSCALEYUV2RGB1b(%%REGBP, %5)

  1368.                 "pcmpeqd %%mm7, %%mm7                   \n\t"

  1369.                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)

  1370.                 "pop %%"REG_BP"                         \n\t"

  1371.                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1372.                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1373.                    "a" (&c->redDither)

  1374.             );

  1375.         }

  1376.     }

  1377. }

  1378. 

  1379. static inline void RENAME(yuv2bgr24_1)(SwsContext *c, const uint16_t *buf0,

  1380.                                        const uint16_t *ubuf0, const uint16_t *ubuf1,

  1381.                                        const uint16_t *vbuf0, const uint16_t *vbuf1,

  1382.                                        const uint16_t *abuf0, uint8_t *dest,

  1383.                                        int dstW, int uvalpha, enum PixelFormat dstFormat,

  1384.                                        int flags, int y)

  1385. {

  1386.     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1

  1387. 

  1388.     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster

  1389.         __asm__ volatile(

  1390.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1391.             "mov        %4, %%"REG_b"               \n\t"

  1392.             "push %%"REG_BP"                        \n\t"

  1393.             YSCALEYUV2RGB1(%%REGBP, %5)

  1394.             "pxor    %%mm7, %%mm7                   \n\t"

  1395.             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)

  1396.             "pop %%"REG_BP"                         \n\t"

  1397.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1398.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1399.                "a" (&c->redDither)

  1400.         );

  1401.     } else {

  1402.         __asm__ volatile(

  1403.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1404.             "mov        %4, %%"REG_b"               \n\t"

  1405.             "push %%"REG_BP"                        \n\t"

  1406.             YSCALEYUV2RGB1b(%%REGBP, %5)

  1407.             "pxor    %%mm7, %%mm7                   \n\t"

  1408.             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)

  1409.             "pop %%"REG_BP"                         \n\t"

  1410.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1411.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1412.                "a" (&c->redDither)

  1413.         );

  1414.     }

  1415. }

  1416. 

  1417. static inline void RENAME(yuv2rgb555_1)(SwsContext *c, const uint16_t *buf0,

  1418.                                         const uint16_t *ubuf0, const uint16_t *ubuf1,

  1419.                                         const uint16_t *vbuf0, const uint16_t *vbuf1,

  1420.                                         const uint16_t *abuf0, uint8_t *dest,

  1421.                                         int dstW, int uvalpha, enum PixelFormat dstFormat,

  1422.                                         int flags, int y)

  1423. {

  1424.     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1

  1425. 

  1426.     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster

  1427.         __asm__ volatile(

  1428.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1429.             "mov        %4, %%"REG_b"               \n\t"

  1430.             "push %%"REG_BP"                        \n\t"

  1431.             YSCALEYUV2RGB1(%%REGBP, %5)

  1432.             "pxor    %%mm7, %%mm7                   \n\t"

  1433.             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1434. #ifdef DITHER1XBPP

  1435.             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1436.             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1437.             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1438. #endif

  1439.             WRITERGB15(%%REGb, 8280(%5), %%REGBP)

  1440.             "pop %%"REG_BP"                         \n\t"

  1441.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1442.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1443.                "a" (&c->redDither)

  1444.         );

  1445.     } else {

  1446.         __asm__ volatile(

  1447.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1448.             "mov        %4, %%"REG_b"               \n\t"

  1449.             "push %%"REG_BP"                        \n\t"

  1450.             YSCALEYUV2RGB1b(%%REGBP, %5)

  1451.             "pxor    %%mm7, %%mm7                   \n\t"

  1452.             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1453. #ifdef DITHER1XBPP

  1454.             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1455.             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1456.             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1457. #endif

  1458.             WRITERGB15(%%REGb, 8280(%5), %%REGBP)

  1459.             "pop %%"REG_BP"                         \n\t"

  1460.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1461.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1462.                "a" (&c->redDither)

  1463.         );

  1464.     }

  1465. }

  1466. 

  1467. static inline void RENAME(yuv2rgb565_1)(SwsContext *c, const uint16_t *buf0,

  1468.                                         const uint16_t *ubuf0, const uint16_t *ubuf1,

  1469.                                         const uint16_t *vbuf0, const uint16_t *vbuf1,

  1470.                                         const uint16_t *abuf0, uint8_t *dest,

  1471.                                         int dstW, int uvalpha, enum PixelFormat dstFormat,

  1472.                                         int flags, int y)

  1473. {

  1474.     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1

  1475. 

  1476.     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster

  1477.         __asm__ volatile(

  1478.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1479.             "mov        %4, %%"REG_b"               \n\t"

  1480.             "push %%"REG_BP"                        \n\t"

  1481.             YSCALEYUV2RGB1(%%REGBP, %5)

  1482.             "pxor    %%mm7, %%mm7                   \n\t"

  1483.             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1484. #ifdef DITHER1XBPP

  1485.             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1486.             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1487.             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1488. #endif

  1489.             WRITERGB16(%%REGb, 8280(%5), %%REGBP)

  1490.             "pop %%"REG_BP"                         \n\t"

  1491.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1492.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1493.                "a" (&c->redDither)

  1494.         );

  1495.     } else {

  1496.         __asm__ volatile(

  1497.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1498.             "mov        %4, %%"REG_b"               \n\t"

  1499.             "push %%"REG_BP"                        \n\t"

  1500.             YSCALEYUV2RGB1b(%%REGBP, %5)

  1501.             "pxor    %%mm7, %%mm7                   \n\t"

  1502.             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */

  1503. #ifdef DITHER1XBPP

  1504.             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"

  1505.             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"

  1506.             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"

  1507. #endif

  1508.             WRITERGB16(%%REGb, 8280(%5), %%REGBP)

  1509.             "pop %%"REG_BP"                         \n\t"

  1510.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1511.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1512.                "a" (&c->redDither)

  1513.         );

  1514.     }

  1515. }

  1516. 

  1517. #define REAL_YSCALEYUV2PACKED1(index, c) \

  1518.     "xor            "#index", "#index"  \n\t"\

  1519.     ".p2align              4            \n\t"\

  1520.     "1:                                 \n\t"\

  1521.     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\

  1522.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1523.     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\

  1524.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1525.     "psraw                $7, %%mm3     \n\t" \

  1526.     "psraw                $7, %%mm4     \n\t" \

  1527.     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\

  1528.     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\

  1529.     "psraw                $7, %%mm1     \n\t" \

  1530.     "psraw                $7, %%mm7     \n\t" \

  1531. 

  1532. #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)

  1533. 

  1534. #define REAL_YSCALEYUV2PACKED1b(index, c) \

  1535.     "xor "#index", "#index"             \n\t"\

  1536.     ".p2align              4            \n\t"\

  1537.     "1:                                 \n\t"\

  1538.     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\

  1539.     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\

  1540.     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1541.     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\

  1542.     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\

  1543.     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \

  1544.     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\

  1545.     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\

  1546.     "psrlw                $8, %%mm3     \n\t" \

  1547.     "psrlw                $8, %%mm4     \n\t" \

  1548.     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\

  1549.     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\

  1550.     "psraw                $7, %%mm1     \n\t" \

  1551.     "psraw                $7, %%mm7     \n\t"

  1552. #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)

  1553. 

  1554. static inline void RENAME(yuv2yuyv422_1)(SwsContext *c, const uint16_t *buf0,

  1555.                                          const uint16_t *ubuf0, const uint16_t *ubuf1,

  1556.                                          const uint16_t *vbuf0, const uint16_t *vbuf1,

  1557.                                          const uint16_t *abuf0, uint8_t *dest,

  1558.                                          int dstW, int uvalpha, enum PixelFormat dstFormat,

  1559.                                          int flags, int y)

  1560. {

  1561.     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1

  1562. 

  1563.     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster

  1564.         __asm__ volatile(

  1565.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1566.             "mov        %4, %%"REG_b"               \n\t"

  1567.             "push %%"REG_BP"                        \n\t"

  1568.             YSCALEYUV2PACKED1(%%REGBP, %5)

  1569.             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)

  1570.             "pop %%"REG_BP"                         \n\t"

  1571.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1572.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1573.                "a" (&c->redDither)

  1574.         );

  1575.     } else {

  1576.         __asm__ volatile(

  1577.             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"

  1578.             "mov        %4, %%"REG_b"               \n\t"

  1579.             "push %%"REG_BP"                        \n\t"

  1580.             YSCALEYUV2PACKED1b(%%REGBP, %5)

  1581.             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)

  1582.             "pop %%"REG_BP"                         \n\t"

  1583.             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"

  1584.             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),

  1585.                "a" (&c->redDither)

  1586.         );

  1587.     }

  1588. }

  1589. 

  1590. #if !COMPILE_TEMPLATE_MMX2

  1591. //FIXME yuy2* can read up to 7 samples too much

  1592. 

  1593. static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)

  1594. {

  1595.     __asm__ volatile(

  1596.         "movq "MANGLE(bm01010101)", %%mm2           \n\t"

  1597.         "mov                    %0, %%"REG_a"       \n\t"

  1598.         "1:                                         \n\t"

  1599.         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"

  1600.         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"

  1601.         "pand                %%mm2, %%mm0           \n\t"

  1602.         "pand                %%mm2, %%mm1           \n\t"

  1603.         "packuswb            %%mm1, %%mm0           \n\t"

  1604.         "movq                %%mm0, (%2, %%"REG_a") \n\t"

  1605.         "add                    $8, %%"REG_a"       \n\t"

  1606.         " js                    1b                  \n\t"

  1607.         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)

  1608.         : "%"REG_a

  1609.     );

  1610. }

  1611. 

  1612. static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1613. {

  1614.     __asm__ volatile(

  1615.         "movq "MANGLE(bm01010101)", %%mm4           \n\t"

  1616.         "mov                    %0, %%"REG_a"       \n\t"

  1617.         "1:                                         \n\t"

  1618.         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"

  1619.         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"

  1620.         "psrlw                  $8, %%mm0           \n\t"

  1621.         "psrlw                  $8, %%mm1           \n\t"

  1622.         "packuswb            %%mm1, %%mm0           \n\t"

  1623.         "movq                %%mm0, %%mm1           \n\t"

  1624.         "psrlw                  $8, %%mm0           \n\t"

  1625.         "pand                %%mm4, %%mm1           \n\t"

  1626.         "packuswb            %%mm0, %%mm0           \n\t"

  1627.         "packuswb            %%mm1, %%mm1           \n\t"

  1628.         "movd                %%mm0, (%3, %%"REG_a") \n\t"

  1629.         "movd                %%mm1, (%2, %%"REG_a") \n\t"

  1630.         "add                    $4, %%"REG_a"       \n\t"

  1631.         " js                    1b                  \n\t"

  1632.         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)

  1633.         : "%"REG_a

  1634.     );

  1635.     assert(src1 == src2);

  1636. }

  1637. 

  1638. static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1639. {

  1640.     __asm__ volatile(

  1641.         "mov                    %0, %%"REG_a"       \n\t"

  1642.         "1:                                         \n\t"

  1643.         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"

  1644.         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"

  1645.         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"

  1646.         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"

  1647.         "psrlw                  $8, %%mm0           \n\t"

  1648.         "psrlw                  $8, %%mm1           \n\t"

  1649.         "psrlw                  $8, %%mm2           \n\t"

  1650.         "psrlw                  $8, %%mm3           \n\t"

  1651.         "packuswb            %%mm1, %%mm0           \n\t"

  1652.         "packuswb            %%mm3, %%mm2           \n\t"

  1653.         "movq                %%mm0, (%3, %%"REG_a") \n\t"

  1654.         "movq                %%mm2, (%4, %%"REG_a") \n\t"

  1655.         "add                    $8, %%"REG_a"       \n\t"

  1656.         " js                    1b                  \n\t"

  1657.         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)

  1658.         : "%"REG_a

  1659.     );

  1660. }

  1661. 

  1662. /* This is almost identical to the previous, end exists only because

  1663.  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */

  1664. static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)

  1665. {

  1666.     __asm__ volatile(

  1667.         "mov                  %0, %%"REG_a"         \n\t"

  1668.         "1:                                         \n\t"

  1669.         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"

  1670.         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"

  1671.         "psrlw                $8, %%mm0             \n\t"

  1672.         "psrlw                $8, %%mm1             \n\t"

  1673.         "packuswb          %%mm1, %%mm0             \n\t"

  1674.         "movq              %%mm0, (%2, %%"REG_a")   \n\t"

  1675.         "add                  $8, %%"REG_a"         \n\t"

  1676.         " js                  1b                    \n\t"

  1677.         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)

  1678.         : "%"REG_a

  1679.     );

  1680. }

  1681. 

  1682. static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1683. {

  1684.     __asm__ volatile(

  1685.         "movq "MANGLE(bm01010101)", %%mm4           \n\t"

  1686.         "mov                    %0, %%"REG_a"       \n\t"

  1687.         "1:                                         \n\t"

  1688.         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"

  1689.         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"

  1690.         "pand                %%mm4, %%mm0           \n\t"

  1691.         "pand                %%mm4, %%mm1           \n\t"

  1692.         "packuswb            %%mm1, %%mm0           \n\t"

  1693.         "movq                %%mm0, %%mm1           \n\t"

  1694.         "psrlw                  $8, %%mm0           \n\t"

  1695.         "pand                %%mm4, %%mm1           \n\t"

  1696.         "packuswb            %%mm0, %%mm0           \n\t"

  1697.         "packuswb            %%mm1, %%mm1           \n\t"

  1698.         "movd                %%mm0, (%3, %%"REG_a") \n\t"

  1699.         "movd                %%mm1, (%2, %%"REG_a") \n\t"

  1700.         "add                    $4, %%"REG_a"       \n\t"

  1701.         " js                    1b                  \n\t"

  1702.         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)

  1703.         : "%"REG_a

  1704.     );

  1705.     assert(src1 == src2);

  1706. }

  1707. 

  1708. static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1709. {

  1710.     __asm__ volatile(

  1711.         "movq "MANGLE(bm01010101)", %%mm4           \n\t"

  1712.         "mov                    %0, %%"REG_a"       \n\t"

  1713.         "1:                                         \n\t"

  1714.         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"

  1715.         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"

  1716.         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"

  1717.         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"

  1718.         "pand                %%mm4, %%mm0           \n\t"

  1719.         "pand                %%mm4, %%mm1           \n\t"

  1720.         "pand                %%mm4, %%mm2           \n\t"

  1721.         "pand                %%mm4, %%mm3           \n\t"

  1722.         "packuswb            %%mm1, %%mm0           \n\t"

  1723.         "packuswb            %%mm3, %%mm2           \n\t"

  1724.         "movq                %%mm0, (%3, %%"REG_a") \n\t"

  1725.         "movq                %%mm2, (%4, %%"REG_a") \n\t"

  1726.         "add                    $8, %%"REG_a"       \n\t"

  1727.         " js                    1b                  \n\t"

  1728.         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)

  1729.         : "%"REG_a

  1730.     );

  1731. }

  1732. 

  1733. static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,

  1734.                                     const uint8_t *src, int width)

  1735. {

  1736.     __asm__ volatile(

  1737.         "movq "MANGLE(bm01010101)", %%mm4           \n\t"

  1738.         "mov                    %0, %%"REG_a"       \n\t"

  1739.         "1:                                         \n\t"

  1740.         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"

  1741.         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"

  1742.         "movq                %%mm0, %%mm2           \n\t"

  1743.         "movq                %%mm1, %%mm3           \n\t"

  1744.         "pand                %%mm4, %%mm0           \n\t"

  1745.         "pand                %%mm4, %%mm1           \n\t"

  1746.         "psrlw                  $8, %%mm2           \n\t"

  1747.         "psrlw                  $8, %%mm3           \n\t"

  1748.         "packuswb            %%mm1, %%mm0           \n\t"

  1749.         "packuswb            %%mm3, %%mm2           \n\t"

  1750.         "movq                %%mm0, (%2, %%"REG_a") \n\t"

  1751.         "movq                %%mm2, (%3, %%"REG_a") \n\t"

  1752.         "add                    $8, %%"REG_a"       \n\t"

  1753.         " js                    1b                  \n\t"

  1754.         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)

  1755.         : "%"REG_a

  1756.     );

  1757. }

  1758. 

  1759. static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,

  1760.                                     const uint8_t *src1, const uint8_t *src2,

  1761.                                     int width, uint32_t *unused)

  1762. {

  1763.     RENAME(nvXXtoUV)(dstU, dstV, src1, width);

  1764. }

  1765. 

  1766. static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,

  1767.                                     const uint8_t *src1, const uint8_t *src2,

  1768.                                     int width, uint32_t *unused)

  1769. {

  1770.     RENAME(nvXXtoUV)(dstV, dstU, src1, width);

  1771. }

  1772. #endif /* !COMPILE_TEMPLATE_MMX2 */

  1773. 

  1774. static inline void RENAME(bgr24ToY_mmx)(int16_t *dst, const uint8_t *src, int width, enum PixelFormat srcFormat)

  1775. {

  1776. 

  1777.     if(srcFormat == PIX_FMT_BGR24) {

  1778.         __asm__ volatile(

  1779.             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"

  1780.             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"

  1781.             :

  1782.         );

  1783.     } else {

  1784.         __asm__ volatile(

  1785.             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"

  1786.             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"

  1787.             :

  1788.         );

  1789.     }

  1790. 

  1791.     __asm__ volatile(

  1792.         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"

  1793.         "mov                        %2, %%"REG_a"   \n\t"

  1794.         "pxor                    %%mm7, %%mm7       \n\t"

  1795.         "1:                                         \n\t"

  1796.         PREFETCH"               64(%0)              \n\t"

  1797.         "movd                     (%0), %%mm0       \n\t"

  1798.         "movd                    2(%0), %%mm1       \n\t"

  1799.         "movd                    6(%0), %%mm2       \n\t"

  1800.         "movd                    8(%0), %%mm3       \n\t"

  1801.         "add                       $12, %0          \n\t"

  1802.         "punpcklbw               %%mm7, %%mm0       \n\t"

  1803.         "punpcklbw               %%mm7, %%mm1       \n\t"

  1804.         "punpcklbw               %%mm7, %%mm2       \n\t"

  1805.         "punpcklbw               %%mm7, %%mm3       \n\t"

  1806.         "pmaddwd                 %%mm5, %%mm0       \n\t"

  1807.         "pmaddwd                 %%mm6, %%mm1       \n\t"

  1808.         "pmaddwd                 %%mm5, %%mm2       \n\t"

  1809.         "pmaddwd                 %%mm6, %%mm3       \n\t"

  1810.         "paddd                   %%mm1, %%mm0       \n\t"

  1811.         "paddd                   %%mm3, %%mm2       \n\t"

  1812.         "paddd                   %%mm4, %%mm0       \n\t"

  1813.         "paddd                   %%mm4, %%mm2       \n\t"

  1814.         "psrad                     $9, %%mm0       \n\t"

  1815.         "psrad                     $9, %%mm2       \n\t"

  1816.         "packssdw                %%mm2, %%mm0       \n\t"

  1817.         "movq                %%mm0, (%1, %%"REG_a") \n\t"

  1818.         "add                        $8, %%"REG_a"   \n\t"

  1819.         " js                        1b              \n\t"

  1820.     : "+r" (src)

  1821.     : "r" (dst+width), "g" ((x86_reg)-2*width)

  1822.     : "%"REG_a

  1823.     );

  1824. }

  1825. 

  1826. static inline void RENAME(bgr24ToUV_mmx)(int16_t *dstU, int16_t *dstV, const uint8_t *src, int width, enum PixelFormat srcFormat)

  1827. {

  1828.     __asm__ volatile(

  1829.         "movq                    24(%4), %%mm6       \n\t"

  1830.         "mov                        %3, %%"REG_a"   \n\t"

  1831.         "pxor                    %%mm7, %%mm7       \n\t"

  1832.         "1:                                         \n\t"

  1833.         PREFETCH"               64(%0)              \n\t"

  1834.         "movd                     (%0), %%mm0       \n\t"

  1835.         "movd                    2(%0), %%mm1       \n\t"

  1836.         "punpcklbw               %%mm7, %%mm0       \n\t"

  1837.         "punpcklbw               %%mm7, %%mm1       \n\t"

  1838.         "movq                    %%mm0, %%mm2       \n\t"

  1839.         "movq                    %%mm1, %%mm3       \n\t"

  1840.         "pmaddwd                  (%4), %%mm0       \n\t"

  1841.         "pmaddwd                 8(%4), %%mm1       \n\t"

  1842.         "pmaddwd                16(%4), %%mm2       \n\t"

  1843.         "pmaddwd                 %%mm6, %%mm3       \n\t"

  1844.         "paddd                   %%mm1, %%mm0       \n\t"

  1845.         "paddd                   %%mm3, %%mm2       \n\t"

  1846. 

  1847.         "movd                    6(%0), %%mm1       \n\t"

  1848.         "movd                    8(%0), %%mm3       \n\t"

  1849.         "add                       $12, %0          \n\t"

  1850.         "punpcklbw               %%mm7, %%mm1       \n\t"

  1851.         "punpcklbw               %%mm7, %%mm3       \n\t"

  1852.         "movq                    %%mm1, %%mm4       \n\t"

  1853.         "movq                    %%mm3, %%mm5       \n\t"

  1854.         "pmaddwd                  (%4), %%mm1       \n\t"

  1855.         "pmaddwd                 8(%4), %%mm3       \n\t"

  1856.         "pmaddwd                16(%4), %%mm4       \n\t"

  1857.         "pmaddwd                 %%mm6, %%mm5       \n\t"

  1858.         "paddd                   %%mm3, %%mm1       \n\t"

  1859.         "paddd                   %%mm5, %%mm4       \n\t"

  1860. 

  1861.         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"

  1862.         "paddd                   %%mm3, %%mm0       \n\t"

  1863.         "paddd                   %%mm3, %%mm2       \n\t"

  1864.         "paddd                   %%mm3, %%mm1       \n\t"

  1865.         "paddd                   %%mm3, %%mm4       \n\t"

  1866.         "psrad                     $9, %%mm0       \n\t"

  1867.         "psrad                     $9, %%mm2       \n\t"

  1868.         "psrad                     $9, %%mm1       \n\t"

  1869.         "psrad                     $9, %%mm4       \n\t"

  1870.         "packssdw                %%mm1, %%mm0       \n\t"

  1871.         "packssdw                %%mm4, %%mm2       \n\t"

  1872.         "movq                %%mm0, (%1, %%"REG_a") \n\t"

  1873.         "movq                %%mm2, (%2, %%"REG_a") \n\t"

  1874.         "add                        $8, %%"REG_a"   \n\t"

  1875.         " js                        1b              \n\t"

  1876.     : "+r" (src)

  1877.     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-2*width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])

  1878.     : "%"REG_a

  1879.     );

  1880. }

  1881. 

  1882. static inline void RENAME(bgr24ToY)(int16_t *dst, const uint8_t *src, int width, uint32_t *unused)

  1883. {

  1884.     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);

  1885. }

  1886. 

  1887. static inline void RENAME(bgr24ToUV)(int16_t *dstU, int16_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1888. {

  1889.     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);

  1890.     assert(src1 == src2);

  1891. }

  1892. 

  1893. static inline void RENAME(rgb24ToY)(int16_t *dst, const uint8_t *src, int width, uint32_t *unused)

  1894. {

  1895.     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);

  1896. }

  1897. 

  1898. static inline void RENAME(rgb24ToUV)(int16_t *dstU, int16_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)

  1899. {

  1900.     assert(src1==src2);

  1901.     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);

  1902. }

  1903. 

  1904. #if !COMPILE_TEMPLATE_MMX2

  1905. // bilinear / bicubic scaling

  1906. static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,

  1907.                                   const int16_t *filter, const int16_t *filterPos, int filterSize)

  1908. {

  1909.     assert(filterSize % 4 == 0 && filterSize>0);

  1910.     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.

  1911.         x86_reg counter= -2*dstW;

  1912.         filter-= counter*2;

  1913.         filterPos-= counter/2;

  1914.         dst-= counter/2;

  1915.         __asm__ volatile(

  1916. #if defined(PIC)

  1917.             "push            %%"REG_b"              \n\t"

  1918. #endif

  1919.             "pxor                %%mm7, %%mm7       \n\t"

  1920.             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...

  1921.             "mov             %%"REG_a", %%"REG_BP"  \n\t"

  1922.             ".p2align                4              \n\t"

  1923.             "1:                                     \n\t"

  1924.             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"

  1925.             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"

  1926.             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"

  1927.             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"

  1928.             "movd      (%3, %%"REG_a"), %%mm0       \n\t"

  1929.             "movd      (%3, %%"REG_b"), %%mm2       \n\t"

  1930.             "punpcklbw           %%mm7, %%mm0       \n\t"

  1931.             "punpcklbw           %%mm7, %%mm2       \n\t"

  1932.             "pmaddwd             %%mm1, %%mm0       \n\t"

  1933.             "pmaddwd             %%mm2, %%mm3       \n\t"

  1934.             "movq                %%mm0, %%mm4       \n\t"

  1935.             "punpckldq           %%mm3, %%mm0       \n\t"

  1936.             "punpckhdq           %%mm3, %%mm4       \n\t"

  1937.             "paddd               %%mm4, %%mm0       \n\t"

  1938.             "psrad                  $7, %%mm0       \n\t"

  1939.             "packssdw            %%mm0, %%mm0       \n\t"

  1940.             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"

  1941.             "add                    $4, %%"REG_BP"  \n\t"

  1942.             " jnc                   1b              \n\t"

  1943. 

  1944.             "pop            %%"REG_BP"              \n\t"

  1945. #if defined(PIC)

  1946.             "pop             %%"REG_b"              \n\t"

  1947. #endif

  1948.             : "+a" (counter)

  1949.             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)

  1950. #if !defined(PIC)

  1951.             : "%"REG_b

  1952. #endif

  1953.         );

  1954.     } else if (filterSize==8) {

  1955.         x86_reg counter= -2*dstW;

  1956.         filter-= counter*4;

  1957.         filterPos-= counter/2;

  1958.         dst-= counter/2;

  1959.         __asm__ volatile(

  1960. #if defined(PIC)

  1961.             "push             %%"REG_b"             \n\t"

  1962. #endif

  1963.             "pxor                 %%mm7, %%mm7      \n\t"

  1964.             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...

  1965.             "mov              %%"REG_a", %%"REG_BP" \n\t"

  1966.             ".p2align                 4             \n\t"

  1967.             "1:                                     \n\t"

  1968.             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"

  1969.             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"

  1970.             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"

  1971.             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"

  1972.             "movd       (%3, %%"REG_a"), %%mm0      \n\t"

  1973.             "movd       (%3, %%"REG_b"), %%mm2      \n\t"

  1974.             "punpcklbw            %%mm7, %%mm0      \n\t"

  1975.             "punpcklbw            %%mm7, %%mm2      \n\t"

  1976.             "pmaddwd              %%mm1, %%mm0      \n\t"

  1977.             "pmaddwd              %%mm2, %%mm3      \n\t"

  1978. 

  1979.             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"

  1980.             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"

  1981.             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"

  1982.             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"

  1983.             "punpcklbw            %%mm7, %%mm4      \n\t"

  1984.             "punpcklbw            %%mm7, %%mm2      \n\t"

  1985.             "pmaddwd              %%mm1, %%mm4      \n\t"

  1986.             "pmaddwd              %%mm2, %%mm5      \n\t"

  1987.             "paddd                %%mm4, %%mm0      \n\t"

  1988.             "paddd                %%mm5, %%mm3      \n\t"

  1989.             "movq                 %%mm0, %%mm4      \n\t"

  1990.             "punpckldq            %%mm3, %%mm0      \n\t"

  1991.             "punpckhdq            %%mm3, %%mm4      \n\t"

  1992.             "paddd                %%mm4, %%mm0      \n\t"

  1993.             "psrad                   $7, %%mm0      \n\t"

  1994.             "packssdw             %%mm0, %%mm0      \n\t"

  1995.             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"

  1996.             "add                     $4, %%"REG_BP" \n\t"

  1997.             " jnc                    1b             \n\t"

  1998. 

  1999.             "pop             %%"REG_BP"             \n\t"

  2000. #if defined(PIC)

  2001.             "pop              %%"REG_b"             \n\t"

  2002. #endif

  2003.             : "+a" (counter)

  2004.             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)

  2005. #if !defined(PIC)

  2006.             : "%"REG_b

  2007. #endif

  2008.         );

  2009.     } else {

  2010.         const uint8_t *offset = src+filterSize;

  2011.         x86_reg counter= -2*dstW;

  2012.         //filter-= counter*filterSize/2;

  2013.         filterPos-= counter/2;

  2014.         dst-= counter/2;

  2015.         __asm__ volatile(

  2016.             "pxor                  %%mm7, %%mm7     \n\t"

  2017.             ".p2align                  4            \n\t"

  2018.             "1:                                     \n\t"

  2019.             "mov                      %2, %%"REG_c" \n\t"

  2020.             "movzwl      (%%"REG_c", %0), %%eax     \n\t"

  2021.             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"

  2022.             "mov                      %5, %%"REG_c" \n\t"

  2023.             "pxor                  %%mm4, %%mm4     \n\t"

  2024.             "pxor                  %%mm5, %%mm5     \n\t"

  2025.             "2:                                     \n\t"

  2026.             "movq                   (%1), %%mm1     \n\t"

  2027.             "movq               (%1, %6), %%mm3     \n\t"

  2028.             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"

  2029.             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"

  2030.             "punpcklbw             %%mm7, %%mm0     \n\t"

  2031.             "punpcklbw             %%mm7, %%mm2     \n\t"

  2032.             "pmaddwd               %%mm1, %%mm0     \n\t"

  2033.             "pmaddwd               %%mm2, %%mm3     \n\t"

  2034.             "paddd                 %%mm3, %%mm5     \n\t"

  2035.             "paddd                 %%mm0, %%mm4     \n\t"

  2036.             "add                      $8, %1        \n\t"

  2037.             "add                      $4, %%"REG_c" \n\t"

  2038.             "cmp                      %4, %%"REG_c" \n\t"

  2039.             " jb                      2b            \n\t"

  2040.             "add                      %6, %1        \n\t"

  2041.             "movq                  %%mm4, %%mm0     \n\t"

  2042.             "punpckldq             %%mm5, %%mm4     \n\t"

  2043.             "punpckhdq             %%mm5, %%mm0     \n\t"

  2044.             "paddd                 %%mm0, %%mm4     \n\t"

  2045.             "psrad                    $7, %%mm4     \n\t"

  2046.             "packssdw              %%mm4, %%mm4     \n\t"

  2047.             "mov                      %3, %%"REG_a" \n\t"

  2048.             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"

  2049.             "add                      $4, %0        \n\t"

  2050.             " jnc                     1b            \n\t"

  2051. 

  2052.             : "+r" (counter), "+r" (filter)

  2053.             : "m" (filterPos), "m" (dst), "m"(offset),

  2054.             "m" (src), "r" ((x86_reg)filterSize*2)

  2055.             : "%"REG_a, "%"REG_c, "%"REG_d

  2056.         );

  2057.     }

  2058. }

  2059. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2060. 

  2061. static inline void RENAME(hScale16)(int16_t *dst, int dstW, const uint16_t *src, int srcW, int xInc,

  2062.                                     const int16_t *filter, const int16_t *filterPos, long filterSize, int shift)

  2063. {

  2064.     int i, j;

  2065. 

  2066.     assert(filterSize % 4 == 0 && filterSize>0);

  2067.     if (filterSize==4 && shift<15) { // Always true for upscaling, sometimes for down, too.

  2068.         x86_reg counter= -2*dstW;

  2069.         filter-= counter*2;

  2070.         filterPos-= counter/2;

  2071.         dst-= counter/2;

  2072.         __asm__ volatile(

  2073.             "movd                   %5, %%mm7       \n\t"

  2074. #if defined(PIC)

  2075.             "push            %%"REG_b"              \n\t"

  2076. #endif

  2077.             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...

  2078.             "mov             %%"REG_a", %%"REG_BP"  \n\t"

  2079.             ".p2align                4              \n\t"

  2080.             "1:                                     \n\t"

  2081.             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"

  2082.             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"

  2083.             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"

  2084.             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"

  2085.             "movq      (%3, %%"REG_a", 2), %%mm0    \n\t"

  2086.             "movq      (%3, %%"REG_b", 2), %%mm2    \n\t"

  2087.             "pmaddwd             %%mm1, %%mm0       \n\t"

  2088.             "pmaddwd             %%mm2, %%mm3       \n\t"

  2089.             "movq                %%mm0, %%mm4       \n\t"

  2090.             "punpckldq           %%mm3, %%mm0       \n\t"

  2091.             "punpckhdq           %%mm3, %%mm4       \n\t"

  2092.             "paddd               %%mm4, %%mm0       \n\t"

  2093.             "psrad               %%mm7, %%mm0       \n\t"

  2094.             "packssdw            %%mm0, %%mm0       \n\t"

  2095.             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"

  2096.             "add                    $4, %%"REG_BP"  \n\t"

  2097.             " jnc                   1b              \n\t"

  2098. 

  2099.             "pop            %%"REG_BP"              \n\t"

  2100. #if defined(PIC)

  2101.             "pop             %%"REG_b"              \n\t"

  2102. #endif

  2103.             : "+a" (counter)

  2104.             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst), "m"(shift)

  2105. #if !defined(PIC)

  2106.             : "%"REG_b

  2107. #endif

  2108.         );

  2109.     } else if (filterSize==8 && shift<15) {

  2110.         x86_reg counter= -2*dstW;

  2111.         filter-= counter*4;

  2112.         filterPos-= counter/2;

  2113.         dst-= counter/2;

  2114.         __asm__ volatile(

  2115.             "movd                   %5, %%mm7       \n\t"

  2116. #if defined(PIC)

  2117.             "push            %%"REG_b"              \n\t"

  2118. #endif

  2119.             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...

  2120.             "mov              %%"REG_a", %%"REG_BP" \n\t"

  2121.             ".p2align                 4             \n\t"

  2122.             "1:                                     \n\t"

  2123.             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"

  2124.             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"

  2125.             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"

  2126.             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"

  2127.             "movq       (%3, %%"REG_a", 2), %%mm0   \n\t"

  2128.             "movq       (%3, %%"REG_b", 2), %%mm2   \n\t"

  2129.             "pmaddwd              %%mm1, %%mm0      \n\t"

  2130.             "pmaddwd              %%mm2, %%mm3      \n\t"

  2131. 

  2132.             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"

  2133.             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"

  2134.             "movq      8(%3, %%"REG_a", 2), %%mm4   \n\t"

  2135.             "movq      8(%3, %%"REG_b", 2), %%mm2   \n\t"

  2136.             "pmaddwd              %%mm1, %%mm4      \n\t"

  2137.             "pmaddwd              %%mm2, %%mm5      \n\t"

  2138.             "paddd                %%mm4, %%mm0      \n\t"

  2139.             "paddd                %%mm5, %%mm3      \n\t"

  2140.             "movq                 %%mm0, %%mm4      \n\t"

  2141.             "punpckldq            %%mm3, %%mm0      \n\t"

  2142.             "punpckhdq            %%mm3, %%mm4      \n\t"

  2143.             "paddd                %%mm4, %%mm0      \n\t"

  2144.             "psrad                %%mm7, %%mm0      \n\t"

  2145.             "packssdw             %%mm0, %%mm0      \n\t"

  2146.             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"

  2147.             "add                     $4, %%"REG_BP" \n\t"

  2148.             " jnc                    1b             \n\t"

  2149. 

  2150.             "pop             %%"REG_BP"             \n\t"

  2151. #if defined(PIC)

  2152.             "pop             %%"REG_b"              \n\t"

  2153. #endif

  2154.             : "+a" (counter)

  2155.             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst), "m"(shift)

  2156. #if !defined(PIC)

  2157.             : "%"REG_b

  2158. #endif

  2159.         );

  2160.     } else if (shift<15){

  2161.         const uint16_t *offset = src+filterSize;

  2162.         x86_reg counter= -2*dstW;

  2163.         //filter-= counter*filterSize/2;

  2164.         filterPos-= counter/2;

  2165.         dst-= counter/2;

  2166.         __asm__ volatile(

  2167.             "movd                   %7, %%mm7       \n\t"

  2168.             ".p2align                  4            \n\t"

  2169.             "1:                                     \n\t"

  2170.             "mov                      %2, %%"REG_c" \n\t"

  2171.             "movzwl      (%%"REG_c", %0), %%eax     \n\t"

  2172.             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"

  2173.             "mov                      %5, %%"REG_c" \n\t"

  2174.             "pxor                  %%mm4, %%mm4     \n\t"

  2175.             "pxor                  %%mm5, %%mm5     \n\t"

  2176.             "2:                                     \n\t"

  2177.             "movq                   (%1), %%mm1     \n\t"

  2178.             "movq               (%1, %6), %%mm3     \n\t"

  2179.             "movq (%%"REG_c", %%"REG_a", 2), %%mm0     \n\t"

  2180.             "movq (%%"REG_c", %%"REG_d", 2), %%mm2     \n\t"

  2181.             "pmaddwd               %%mm1, %%mm0     \n\t"

  2182.             "pmaddwd               %%mm2, %%mm3     \n\t"

  2183.             "paddd                 %%mm3, %%mm5     \n\t"

  2184.             "paddd                 %%mm0, %%mm4     \n\t"

  2185.             "add                      $8, %1        \n\t"

  2186.             "add                      $8, %%"REG_c" \n\t"

  2187.             "cmp                      %4, %%"REG_c" \n\t"

  2188.             " jb                      2b            \n\t"

  2189.             "add                      %6, %1        \n\t"

  2190.             "movq                  %%mm4, %%mm0     \n\t"

  2191.             "punpckldq             %%mm5, %%mm4     \n\t"

  2192.             "punpckhdq             %%mm5, %%mm0     \n\t"

  2193.             "paddd                 %%mm0, %%mm4     \n\t"

  2194.             "psrad                 %%mm7, %%mm4     \n\t"

  2195.             "packssdw              %%mm4, %%mm4     \n\t"

  2196.             "mov                      %3, %%"REG_a" \n\t"

  2197.             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"

  2198.             "add                      $4, %0        \n\t"

  2199.             " jnc                     1b            \n\t"

  2200. 

  2201.             : "+r" (counter), "+r" (filter)

  2202.             : "m" (filterPos), "m" (dst), "m"(offset),

  2203.             "m" (src), "r" ((x86_reg)filterSize*2), "m"(shift)

  2204.             : "%"REG_a, "%"REG_c, "%"REG_d

  2205.         );

  2206.     } else

  2207.     for (i=0; i<dstW; i++) {

  2208.         int srcPos= filterPos[i];

  2209.         int val=0;

  2210.         for (j=0; j<filterSize; j++) {

  2211.             val += ((int)src[srcPos + j])*filter[filterSize*i + j];

  2212.         }

  2213.         dst[i] = FFMIN(val>>shift, (1<<15)-1); // the cubic equation does overflow ...

  2214.     }

  2215. }

  2216. 

  2217. 

  2218. #if COMPILE_TEMPLATE_MMX2

  2219. static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,

  2220.                                         int dstWidth, const uint8_t *src, int srcW,

  2221.                                         int xInc)

  2222. {

  2223.     int32_t *filterPos = c->hLumFilterPos;

  2224.     int16_t *filter    = c->hLumFilter;

  2225.     void    *mmx2FilterCode= c->lumMmx2FilterCode;

  2226.     int i;

  2227. #if defined(PIC)

  2228.     DECLARE_ALIGNED(8, uint64_t, ebxsave);

  2229. #endif

  2230. 

  2231.     __asm__ volatile(

  2232. #if defined(PIC)

  2233.         "mov               %%"REG_b", %5        \n\t"

  2234. #endif

  2235.         "pxor                  %%mm7, %%mm7     \n\t"

  2236.         "mov                      %0, %%"REG_c" \n\t"

  2237.         "mov                      %1, %%"REG_D" \n\t"

  2238.         "mov                      %2, %%"REG_d" \n\t"

  2239.         "mov                      %3, %%"REG_b" \n\t"

  2240.         "xor               %%"REG_a", %%"REG_a" \n\t" // i

  2241.         PREFETCH"        (%%"REG_c")            \n\t"

  2242.         PREFETCH"      32(%%"REG_c")            \n\t"

  2243.         PREFETCH"      64(%%"REG_c")            \n\t"

  2244. 

  2245. #if ARCH_X86_64

  2246. #define CALL_MMX2_FILTER_CODE \

  2247.         "movl            (%%"REG_b"), %%esi     \n\t"\

  2248.         "call                    *%4            \n\t"\

  2249.         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\

  2250.         "add               %%"REG_S", %%"REG_c" \n\t"\

  2251.         "add               %%"REG_a", %%"REG_D" \n\t"\

  2252.         "xor               %%"REG_a", %%"REG_a" \n\t"\

  2253. 

  2254. #else

  2255. #define CALL_MMX2_FILTER_CODE \

  2256.         "movl (%%"REG_b"), %%esi        \n\t"\

  2257.         "call         *%4                       \n\t"\

  2258.         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\

  2259.         "add               %%"REG_a", %%"REG_D" \n\t"\

  2260.         "xor               %%"REG_a", %%"REG_a" \n\t"\

  2261. 

  2262. #endif /* ARCH_X86_64 */

  2263. 

  2264.         CALL_MMX2_FILTER_CODE

  2265.         CALL_MMX2_FILTER_CODE

  2266.         CALL_MMX2_FILTER_CODE

  2267.         CALL_MMX2_FILTER_CODE

  2268.         CALL_MMX2_FILTER_CODE

  2269.         CALL_MMX2_FILTER_CODE

  2270.         CALL_MMX2_FILTER_CODE

  2271.         CALL_MMX2_FILTER_CODE

  2272. 

  2273. #if defined(PIC)

  2274.         "mov                      %5, %%"REG_b" \n\t"

  2275. #endif

  2276.         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),

  2277.            "m" (mmx2FilterCode)

  2278. #if defined(PIC)

  2279.           ,"m" (ebxsave)

  2280. #endif

  2281.         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D

  2282. #if !defined(PIC)

  2283.          ,"%"REG_b

  2284. #endif

  2285.     );

  2286. 

  2287.     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)

  2288.         dst[i] = src[srcW-1]*128;

  2289. }

  2290. 

  2291. static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,

  2292.                                         int dstWidth, const uint8_t *src1,

  2293.                                         const uint8_t *src2, int srcW, int xInc)

  2294. {

  2295.     int32_t *filterPos = c->hChrFilterPos;

  2296.     int16_t *filter    = c->hChrFilter;

  2297.     void    *mmx2FilterCode= c->chrMmx2FilterCode;

  2298.     int i;

  2299. #if defined(PIC)

  2300.     DECLARE_ALIGNED(8, uint64_t, ebxsave);

  2301. #endif

  2302. 

  2303.     __asm__ volatile(

  2304. #if defined(PIC)

  2305.         "mov          %%"REG_b", %7         \n\t"

  2306. #endif

  2307.         "pxor             %%mm7, %%mm7      \n\t"

  2308.         "mov                 %0, %%"REG_c"  \n\t"

  2309.         "mov                 %1, %%"REG_D"  \n\t"

  2310.         "mov                 %2, %%"REG_d"  \n\t"

  2311.         "mov                 %3, %%"REG_b"  \n\t"

  2312.         "xor          %%"REG_a", %%"REG_a"  \n\t" // i

  2313.         PREFETCH"   (%%"REG_c")             \n\t"

  2314.         PREFETCH" 32(%%"REG_c")             \n\t"

  2315.         PREFETCH" 64(%%"REG_c")             \n\t"

  2316. 

  2317.         CALL_MMX2_FILTER_CODE

  2318.         CALL_MMX2_FILTER_CODE

  2319.         CALL_MMX2_FILTER_CODE

  2320.         CALL_MMX2_FILTER_CODE

  2321.         "xor          %%"REG_a", %%"REG_a"  \n\t" // i

  2322.         "mov                 %5, %%"REG_c"  \n\t" // src

  2323.         "mov                 %6, %%"REG_D"  \n\t" // buf2

  2324.         PREFETCH"   (%%"REG_c")             \n\t"

  2325.         PREFETCH" 32(%%"REG_c")             \n\t"

  2326.         PREFETCH" 64(%%"REG_c")             \n\t"

  2327. 

  2328.         CALL_MMX2_FILTER_CODE

  2329.         CALL_MMX2_FILTER_CODE

  2330.         CALL_MMX2_FILTER_CODE

  2331.         CALL_MMX2_FILTER_CODE

  2332. 

  2333. #if defined(PIC)

  2334.         "mov %7, %%"REG_b"    \n\t"

  2335. #endif

  2336.         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),

  2337.            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)

  2338. #if defined(PIC)

  2339.           ,"m" (ebxsave)

  2340. #endif

  2341.         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D

  2342. #if !defined(PIC)

  2343.          ,"%"REG_b

  2344. #endif

  2345.     );

  2346. 

  2347.     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {

  2348.         dst1[i] = src1[srcW-1]*128;

  2349.         dst2[i] = src2[srcW-1]*128;

  2350.     }

  2351. }

  2352. #endif /* COMPILE_TEMPLATE_MMX2 */

  2353. 

  2354. #if !COMPILE_TEMPLATE_MMX2

  2355. static void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufIndex,

  2356.                                   int lastInLumBuf, int lastInChrBuf)

  2357. {

  2358.     const int dstH= c->dstH;

  2359.     const int flags= c->flags;

  2360.     int16_t **lumPixBuf= c->lumPixBuf;

  2361.     int16_t **chrUPixBuf= c->chrUPixBuf;

  2362.     int16_t **alpPixBuf= c->alpPixBuf;

  2363.     const int vLumBufSize= c->vLumBufSize;

  2364.     const int vChrBufSize= c->vChrBufSize;

  2365.     int16_t *vLumFilterPos= c->vLumFilterPos;

  2366.     int16_t *vChrFilterPos= c->vChrFilterPos;

  2367.     int16_t *vLumFilter= c->vLumFilter;

  2368.     int16_t *vChrFilter= c->vChrFilter;

  2369.     int32_t *lumMmxFilter= c->lumMmxFilter;

  2370.     int32_t *chrMmxFilter= c->chrMmxFilter;

  2371.     int32_t av_unused *alpMmxFilter= c->alpMmxFilter;

  2372.     const int vLumFilterSize= c->vLumFilterSize;

  2373.     const int vChrFilterSize= c->vChrFilterSize;

  2374.     const int chrDstY= dstY>>c->chrDstVSubSample;

  2375.     const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input

  2376.     const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input

  2377. 

  2378.     c->blueDither= ff_dither8[dstY&1];

  2379.     if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)

  2380.         c->greenDither= ff_dither8[dstY&1];

  2381.     else

  2382.         c->greenDither= ff_dither4[dstY&1];

  2383.     c->redDither= ff_dither8[(dstY+1)&1];

  2384.     if (dstY < dstH - 2) {

  2385.         const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;

  2386.         const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;

  2387.         const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;

  2388.         int i;

  2389.         if (flags & SWS_ACCURATE_RND) {

  2390.             int s= APCK_SIZE / 8;

  2391.             for (i=0; i<vLumFilterSize; i+=2) {

  2392.                 *(const void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];

  2393.                 *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];

  2394.                 lumMmxFilter[s*i+APCK_COEF/4  ]=

  2395.                 lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]

  2396.                            + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);

  2397.                 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {

  2398.                     *(const void**)&alpMmxFilter[s*i              ]= alpSrcPtr[i  ];

  2399.                     *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4  ]= alpSrcPtr[i+(vLumFilterSize>1)];

  2400.                     alpMmxFilter[s*i+APCK_COEF/4  ]=

  2401.                     alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4  ];

  2402.                 }

  2403.             }

  2404.             for (i=0; i<vChrFilterSize; i+=2) {

  2405.                 *(const void**)&chrMmxFilter[s*i              ]= chrUSrcPtr[i  ];

  2406.                 *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrUSrcPtr[i+(vChrFilterSize>1)];

  2407.                 chrMmxFilter[s*i+APCK_COEF/4  ]=

  2408.                 chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]

  2409.                            + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);

  2410.             }

  2411.         } else {

  2412.             for (i=0; i<vLumFilterSize; i++) {

  2413.                 *(const void**)&lumMmxFilter[4*i+0]= lumSrcPtr[i];

  2414.                 lumMmxFilter[4*i+2]=

  2415.                 lumMmxFilter[4*i+3]=

  2416.                     ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;

  2417.                 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {

  2418.                     *(const void**)&alpMmxFilter[4*i+0]= alpSrcPtr[i];

  2419.                     alpMmxFilter[4*i+2]=

  2420.                     alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];

  2421.                 }

  2422.             }

  2423.             for (i=0; i<vChrFilterSize; i++) {

  2424.                 *(const void**)&chrMmxFilter[4*i+0]= chrUSrcPtr[i];

  2425.                 chrMmxFilter[4*i+2]=

  2426.                 chrMmxFilter[4*i+3]=

  2427.                     ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;

  2428.             }

  2429.         }

  2430.     }

  2431. }

  2432. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2433. 

  2434. static void RENAME(sws_init_swScale)(SwsContext *c)

  2435. {

  2436.     enum PixelFormat srcFormat = c->srcFormat;

  2437. 

  2438.     if (!(c->flags & SWS_BITEXACT)) {

  2439.         if (c->flags & SWS_ACCURATE_RND) {

  2440.             c->yuv2yuv1     = RENAME(yuv2yuv1_ar    );

  2441.             c->yuv2yuvX     = RENAME(yuv2yuvX_ar    );

  2442.             switch (c->dstFormat) {

  2443.             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;

  2444.             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;

  2445.             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;

  2446.             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;

  2447.             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;

  2448.             default: break;

  2449.             }

  2450.         } else {

  2451.             c->yuv2yuv1     = RENAME(yuv2yuv1    );

  2452.             c->yuv2yuvX     = RENAME(yuv2yuvX    );

  2453.             switch (c->dstFormat) {

  2454.             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;

  2455.             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;

  2456.             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;

  2457.             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;

  2458.             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;

  2459.             default: break;

  2460.             }

  2461.         }

  2462.         switch (c->dstFormat) {

  2463.         case PIX_FMT_RGB32:

  2464.                 c->yuv2packed1 = RENAME(yuv2rgb32_1);

  2465.                 c->yuv2packed2 = RENAME(yuv2rgb32_2);

  2466.                 break;

  2467.         case PIX_FMT_BGR24:

  2468.                 c->yuv2packed1 = RENAME(yuv2bgr24_1);

  2469.                 c->yuv2packed2 = RENAME(yuv2bgr24_2);

  2470.                 break;

  2471.         case PIX_FMT_RGB555:

  2472.                 c->yuv2packed1 = RENAME(yuv2rgb555_1);

  2473.                 c->yuv2packed2 = RENAME(yuv2rgb555_2);

  2474.                 break;

  2475.         case PIX_FMT_RGB565:

  2476.                 c->yuv2packed1 = RENAME(yuv2rgb565_1);

  2477.                 c->yuv2packed2 = RENAME(yuv2rgb565_2);

  2478.                 break;

  2479.         case PIX_FMT_YUYV422:

  2480.                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);

  2481.                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);

  2482.                 break;

  2483.         default:

  2484.                 break;

  2485.         }

  2486.     }

  2487. 

  2488. #if !COMPILE_TEMPLATE_MMX2

  2489.     c->hScale       = RENAME(hScale      );

  2490. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2491. 

  2492.     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).

  2493. #if COMPILE_TEMPLATE_MMX2

  2494.     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)

  2495.     {

  2496.         c->hyscale_fast = RENAME(hyscale_fast);

  2497.         c->hcscale_fast = RENAME(hcscale_fast);

  2498.     } else {

  2499. #endif /* COMPILE_TEMPLATE_MMX2 */

  2500.         c->hyscale_fast = NULL;

  2501.         c->hcscale_fast = NULL;

  2502. #if COMPILE_TEMPLATE_MMX2

  2503.     }

  2504. #endif /* COMPILE_TEMPLATE_MMX2 */

  2505. 

  2506. #if !COMPILE_TEMPLATE_MMX2

  2507.     switch(srcFormat) {

  2508.         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;

  2509.         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;

  2510.         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;

  2511.         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;

  2512.         case PIX_FMT_GRAY16LE :

  2513.         case PIX_FMT_YUV420P9LE:

  2514.         case PIX_FMT_YUV422P10LE:

  2515.         case PIX_FMT_YUV420P10LE:

  2516.         case PIX_FMT_YUV420P16LE:

  2517.         case PIX_FMT_YUV422P16LE:

  2518.         case PIX_FMT_YUV444P16LE: c->hScale16= RENAME(hScale16); break;

  2519.     }

  2520. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2521.     if (!c->chrSrcHSubSample) {

  2522.         switch(srcFormat) {

  2523.         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;

  2524.         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;

  2525.         default: break;

  2526.         }

  2527.     }

  2528. 

  2529.     switch (srcFormat) {

  2530. #if !COMPILE_TEMPLATE_MMX2

  2531.     case PIX_FMT_YUYV422  :

  2532.     case PIX_FMT_Y400A    :

  2533.                             c->lumToYV12 = RENAME(yuy2ToY); break;

  2534.     case PIX_FMT_UYVY422  :

  2535.                             c->lumToYV12 = RENAME(uyvyToY); break;

  2536. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2537.     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;

  2538.     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;

  2539.     default: break;

  2540.     }

  2541. #if !COMPILE_TEMPLATE_MMX2

  2542.     if (c->alpPixBuf) {

  2543.         switch (srcFormat) {

  2544.         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;

  2545.         default: break;

  2546.         }

  2547.     }

  2548. #endif /* !COMPILE_TEMPLATE_MMX2 */

  2549.     if(isAnyRGB(c->srcFormat))

  2550.         c->hScale16= RENAME(hScale16);

  2551. }