falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
91155 Commits
32 Branches
293MB
Tree: 7b51b8020c
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '7b51b8020c'
${ noResults }
FFmpeg/libswscale/x86/rgb2rgb_template.c

2609 lines
107KB
Raw Blame History 

  1. /*

  2.  * software RGB to RGB converter

  3.  * pluralize by software PAL8 to RGB converter

  4.  *              software YUV to YUV converter

  5.  *              software YUV to RGB converter

  6.  * Written by Nick Kurshev.

  7.  * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)

  8.  * lot of big-endian byte order fixes by Alex Beregszaszi

  9.  *

  10.  * This file is part of FFmpeg.

  11.  *

  12.  * FFmpeg is free software; you can redistribute it and/or

  13.  * modify it under the terms of the GNU Lesser General Public

  14.  * License as published by the Free Software Foundation; either

  15.  * version 2.1 of the License, or (at your option) any later version.

  16.  *

  17.  * FFmpeg is distributed in the hope that it will be useful,

  18.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  19.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  20.  * Lesser General Public License for more details.

  21.  *

  22.  * You should have received a copy of the GNU Lesser General Public

  23.  * License along with FFmpeg; if not, write to the Free Software

  24.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  25.  */

  26. 

  27. #include <stddef.h>

  28. #include <stdint.h>

  29. 

  30. #include "libavutil/attributes.h"

  31. #include "libavutil/x86/asm.h"

  32. 

  33. #undef PREFETCH

  34. #undef MOVNTQ

  35. #undef EMMS

  36. #undef SFENCE

  37. #undef PAVGB

  38. 

  39. #if COMPILE_TEMPLATE_AMD3DNOW

  40. #define PREFETCH  "prefetch"

  41. #define PAVGB     "pavgusb"

  42. #elif COMPILE_TEMPLATE_MMXEXT

  43. #define PREFETCH "prefetchnta"

  44. #define PAVGB     "pavgb"

  45. #else

  46. #define PREFETCH  " # nop"

  47. #endif

  48. 

  49. #if COMPILE_TEMPLATE_AMD3DNOW

  50. /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */

  51. #define EMMS     "femms"

  52. #else

  53. #define EMMS     "emms"

  54. #endif

  55. 

  56. #if COMPILE_TEMPLATE_MMXEXT

  57. #define MOVNTQ "movntq"

  58. #define SFENCE "sfence"

  59. #else

  60. #define MOVNTQ "movq"

  61. #define SFENCE " # nop"

  62. #endif

  63. 

  64. #if !COMPILE_TEMPLATE_SSE2

  65. 

  66. #if !COMPILE_TEMPLATE_AMD3DNOW

  67. 

  68. static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, int src_size)

  69. {

  70.     uint8_t *dest = dst;

  71.     const uint8_t *s = src;

  72.     const uint8_t *end;

  73.     const uint8_t *mm_end;

  74.     end = s + src_size;

  75.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  76.     mm_end = end - 23;

  77.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask32a):"memory");

  78.     while (s < mm_end) {

  79.         __asm__ volatile(

  80.             PREFETCH"  32(%1)           \n\t"

  81.             "movd        (%1), %%mm0    \n\t"

  82.             "punpckldq  3(%1), %%mm0    \n\t"

  83.             "movd       6(%1), %%mm1    \n\t"

  84.             "punpckldq  9(%1), %%mm1    \n\t"

  85.             "movd      12(%1), %%mm2    \n\t"

  86.             "punpckldq 15(%1), %%mm2    \n\t"

  87.             "movd      18(%1), %%mm3    \n\t"

  88.             "punpckldq 21(%1), %%mm3    \n\t"

  89.             "por        %%mm7, %%mm0    \n\t"

  90.             "por        %%mm7, %%mm1    \n\t"

  91.             "por        %%mm7, %%mm2    \n\t"

  92.             "por        %%mm7, %%mm3    \n\t"

  93.             MOVNTQ"     %%mm0,   (%0)   \n\t"

  94.             MOVNTQ"     %%mm1,  8(%0)   \n\t"

  95.             MOVNTQ"     %%mm2, 16(%0)   \n\t"

  96.             MOVNTQ"     %%mm3, 24(%0)"

  97.             :: "r"(dest), "r"(s)

  98.             :"memory");

  99.         dest += 32;

  100.         s += 24;

  101.     }

  102.     __asm__ volatile(SFENCE:::"memory");

  103.     __asm__ volatile(EMMS:::"memory");

  104.     while (s < end) {

  105.         *dest++ = *s++;

  106.         *dest++ = *s++;

  107.         *dest++ = *s++;

  108.         *dest++ = 255;

  109.     }

  110. }

  111. 

  112. #define STORE_BGR24_MMX \

  113.             "psrlq         $8, %%mm2    \n\t" \

  114.             "psrlq         $8, %%mm3    \n\t" \

  115.             "psrlq         $8, %%mm6    \n\t" \

  116.             "psrlq         $8, %%mm7    \n\t" \

  117.             "pand "MANGLE(mask24l)", %%mm0\n\t" \

  118.             "pand "MANGLE(mask24l)", %%mm1\n\t" \

  119.             "pand "MANGLE(mask24l)", %%mm4\n\t" \

  120.             "pand "MANGLE(mask24l)", %%mm5\n\t" \

  121.             "pand "MANGLE(mask24h)", %%mm2\n\t" \

  122.             "pand "MANGLE(mask24h)", %%mm3\n\t" \

  123.             "pand "MANGLE(mask24h)", %%mm6\n\t" \

  124.             "pand "MANGLE(mask24h)", %%mm7\n\t" \

  125.             "por        %%mm2, %%mm0    \n\t" \

  126.             "por        %%mm3, %%mm1    \n\t" \

  127.             "por        %%mm6, %%mm4    \n\t" \

  128.             "por        %%mm7, %%mm5    \n\t" \

  129.  \

  130.             "movq       %%mm1, %%mm2    \n\t" \

  131.             "movq       %%mm4, %%mm3    \n\t" \

  132.             "psllq        $48, %%mm2    \n\t" \

  133.             "psllq        $32, %%mm3    \n\t" \

  134.             "por        %%mm2, %%mm0    \n\t" \

  135.             "psrlq        $16, %%mm1    \n\t" \

  136.             "psrlq        $32, %%mm4    \n\t" \

  137.             "psllq        $16, %%mm5    \n\t" \

  138.             "por        %%mm3, %%mm1    \n\t" \

  139.             "por        %%mm5, %%mm4    \n\t" \

  140.  \

  141.             MOVNTQ"     %%mm0,   (%0)    \n\t" \

  142.             MOVNTQ"     %%mm1,  8(%0)    \n\t" \

  143.             MOVNTQ"     %%mm4, 16(%0)"

  144. 

  145. 

  146. static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)

  147. {

  148.     uint8_t *dest = dst;

  149.     const uint8_t *s = src;

  150.     const uint8_t *end;

  151.     const uint8_t *mm_end;

  152.     end = s + src_size;

  153.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  154.     mm_end = end - 31;

  155.     while (s < mm_end) {

  156.         __asm__ volatile(

  157.             PREFETCH"  32(%1)           \n\t"

  158.             "movq        (%1), %%mm0    \n\t"

  159.             "movq       8(%1), %%mm1    \n\t"

  160.             "movq      16(%1), %%mm4    \n\t"

  161.             "movq      24(%1), %%mm5    \n\t"

  162.             "movq       %%mm0, %%mm2    \n\t"

  163.             "movq       %%mm1, %%mm3    \n\t"

  164.             "movq       %%mm4, %%mm6    \n\t"

  165.             "movq       %%mm5, %%mm7    \n\t"

  166.             STORE_BGR24_MMX

  167.             :: "r"(dest), "r"(s)

  168.               NAMED_CONSTRAINTS_ADD(mask24l,mask24h)

  169.             :"memory");

  170.         dest += 24;

  171.         s += 32;

  172.     }

  173.     __asm__ volatile(SFENCE:::"memory");

  174.     __asm__ volatile(EMMS:::"memory");

  175.     while (s < end) {

  176.         *dest++ = *s++;

  177.         *dest++ = *s++;

  178.         *dest++ = *s++;

  179.         s++;

  180.     }

  181. }

  182. 

  183. /*

  184.  original by Strepto/Astral

  185.  ported to gcc & bugfixed: A'rpi

  186.  MMXEXT, 3DNOW optimization by Nick Kurshev

  187.  32-bit C version, and and&add trick by Michael Niedermayer

  188. */

  189. static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, int src_size)

  190. {

  191.     register const uint8_t* s=src;

  192.     register uint8_t* d=dst;

  193.     register const uint8_t *end;

  194.     const uint8_t *mm_end;

  195.     end = s + src_size;

  196.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));

  197.     __asm__ volatile("movq        %0, %%mm4"::"m"(mask15s));

  198.     mm_end = end - 15;

  199.     while (s<mm_end) {

  200.         __asm__ volatile(

  201.             PREFETCH" 32(%1)        \n\t"

  202.             "movq      (%1), %%mm0  \n\t"

  203.             "movq     8(%1), %%mm2  \n\t"

  204.             "movq     %%mm0, %%mm1  \n\t"

  205.             "movq     %%mm2, %%mm3  \n\t"

  206.             "pand     %%mm4, %%mm0  \n\t"

  207.             "pand     %%mm4, %%mm2  \n\t"

  208.             "paddw    %%mm1, %%mm0  \n\t"

  209.             "paddw    %%mm3, %%mm2  \n\t"

  210.             MOVNTQ"   %%mm0,  (%0)  \n\t"

  211.             MOVNTQ"   %%mm2, 8(%0)"

  212.             :: "r"(d), "r"(s)

  213.         );

  214.         d+=16;

  215.         s+=16;

  216.     }

  217.     __asm__ volatile(SFENCE:::"memory");

  218.     __asm__ volatile(EMMS:::"memory");

  219.     mm_end = end - 3;

  220.     while (s < mm_end) {

  221.         register unsigned x= *((const uint32_t *)s);

  222.         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);

  223.         d+=4;

  224.         s+=4;

  225.     }

  226.     if (s < end) {

  227.         register unsigned short x= *((const uint16_t *)s);

  228.         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);

  229.     }

  230. }

  231. 

  232. static inline void RENAME(rgb16to15)(const uint8_t *src, uint8_t *dst, int src_size)

  233. {

  234.     register const uint8_t* s=src;

  235.     register uint8_t* d=dst;

  236.     register const uint8_t *end;

  237.     const uint8_t *mm_end;

  238.     end = s + src_size;

  239.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));

  240.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask15rg));

  241.     __asm__ volatile("movq        %0, %%mm6"::"m"(mask15b));

  242.     mm_end = end - 15;

  243.     while (s<mm_end) {

  244.         __asm__ volatile(

  245.             PREFETCH" 32(%1)        \n\t"

  246.             "movq      (%1), %%mm0  \n\t"

  247.             "movq     8(%1), %%mm2  \n\t"

  248.             "movq     %%mm0, %%mm1  \n\t"

  249.             "movq     %%mm2, %%mm3  \n\t"

  250.             "psrlq       $1, %%mm0  \n\t"

  251.             "psrlq       $1, %%mm2  \n\t"

  252.             "pand     %%mm7, %%mm0  \n\t"

  253.             "pand     %%mm7, %%mm2  \n\t"

  254.             "pand     %%mm6, %%mm1  \n\t"

  255.             "pand     %%mm6, %%mm3  \n\t"

  256.             "por      %%mm1, %%mm0  \n\t"

  257.             "por      %%mm3, %%mm2  \n\t"

  258.             MOVNTQ"   %%mm0,  (%0)  \n\t"

  259.             MOVNTQ"   %%mm2, 8(%0)"

  260.             :: "r"(d), "r"(s)

  261.         );

  262.         d+=16;

  263.         s+=16;

  264.     }

  265.     __asm__ volatile(SFENCE:::"memory");

  266.     __asm__ volatile(EMMS:::"memory");

  267.     mm_end = end - 3;

  268.     while (s < mm_end) {

  269.         register uint32_t x= *((const uint32_t*)s);

  270.         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);

  271.         s+=4;

  272.         d+=4;

  273.     }

  274.     if (s < end) {

  275.         register uint16_t x= *((const uint16_t*)s);

  276.         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);

  277.     }

  278. }

  279. 

  280. static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, int src_size)

  281. {

  282.     const uint8_t *s = src;

  283.     const uint8_t *end;

  284.     const uint8_t *mm_end;

  285.     uint16_t *d = (uint16_t *)dst;

  286.     end = s + src_size;

  287.     mm_end = end - 15;

  288.     __asm__ volatile(

  289.         "movq           %3, %%mm5   \n\t"

  290.         "movq           %4, %%mm6   \n\t"

  291.         "movq           %5, %%mm7   \n\t"

  292.         "jmp 2f                     \n\t"

  293.         ".p2align        4          \n\t"

  294.         "1:                         \n\t"

  295.         PREFETCH"   32(%1)          \n\t"

  296.         "movd         (%1), %%mm0   \n\t"

  297.         "movd        4(%1), %%mm3   \n\t"

  298.         "punpckldq   8(%1), %%mm0   \n\t"

  299.         "punpckldq  12(%1), %%mm3   \n\t"

  300.         "movq        %%mm0, %%mm1   \n\t"

  301.         "movq        %%mm3, %%mm4   \n\t"

  302.         "pand        %%mm6, %%mm0   \n\t"

  303.         "pand        %%mm6, %%mm3   \n\t"

  304.         "pmaddwd     %%mm7, %%mm0   \n\t"

  305.         "pmaddwd     %%mm7, %%mm3   \n\t"

  306.         "pand        %%mm5, %%mm1   \n\t"

  307.         "pand        %%mm5, %%mm4   \n\t"

  308.         "por         %%mm1, %%mm0   \n\t"

  309.         "por         %%mm4, %%mm3   \n\t"

  310.         "psrld          $5, %%mm0   \n\t"

  311.         "pslld         $11, %%mm3   \n\t"

  312.         "por         %%mm3, %%mm0   \n\t"

  313.         MOVNTQ"      %%mm0, (%0)    \n\t"

  314.         "add           $16,  %1     \n\t"

  315.         "add            $8,  %0     \n\t"

  316.         "2:                         \n\t"

  317.         "cmp            %2,  %1     \n\t"

  318.         " jb            1b          \n\t"

  319.         : "+r" (d), "+r"(s)

  320.         : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)

  321.     );

  322.     __asm__ volatile(SFENCE:::"memory");

  323.     __asm__ volatile(EMMS:::"memory");

  324.     while (s < end) {

  325.         register int rgb = *(const uint32_t*)s; s += 4;

  326.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);

  327.     }

  328. }

  329. 

  330. static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)

  331. {

  332.     const uint8_t *s = src;

  333.     const uint8_t *end;

  334.     const uint8_t *mm_end;

  335.     uint16_t *d = (uint16_t *)dst;

  336.     end = s + src_size;

  337.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  338.     __asm__ volatile(

  339.         "movq          %0, %%mm7    \n\t"

  340.         "movq          %1, %%mm6    \n\t"

  341.         ::"m"(red_16mask),"m"(green_16mask));

  342.     mm_end = end - 15;

  343.     while (s < mm_end) {

  344.         __asm__ volatile(

  345.             PREFETCH"  32(%1)           \n\t"

  346.             "movd        (%1), %%mm0    \n\t"

  347.             "movd       4(%1), %%mm3    \n\t"

  348.             "punpckldq  8(%1), %%mm0    \n\t"

  349.             "punpckldq 12(%1), %%mm3    \n\t"

  350.             "movq       %%mm0, %%mm1    \n\t"

  351.             "movq       %%mm0, %%mm2    \n\t"

  352.             "movq       %%mm3, %%mm4    \n\t"

  353.             "movq       %%mm3, %%mm5    \n\t"

  354.             "psllq         $8, %%mm0    \n\t"

  355.             "psllq         $8, %%mm3    \n\t"

  356.             "pand       %%mm7, %%mm0    \n\t"

  357.             "pand       %%mm7, %%mm3    \n\t"

  358.             "psrlq         $5, %%mm1    \n\t"

  359.             "psrlq         $5, %%mm4    \n\t"

  360.             "pand       %%mm6, %%mm1    \n\t"

  361.             "pand       %%mm6, %%mm4    \n\t"

  362.             "psrlq        $19, %%mm2    \n\t"

  363.             "psrlq        $19, %%mm5    \n\t"

  364.             "pand          %2, %%mm2    \n\t"

  365.             "pand          %2, %%mm5    \n\t"

  366.             "por        %%mm1, %%mm0    \n\t"

  367.             "por        %%mm4, %%mm3    \n\t"

  368.             "por        %%mm2, %%mm0    \n\t"

  369.             "por        %%mm5, %%mm3    \n\t"

  370.             "psllq        $16, %%mm3    \n\t"

  371.             "por        %%mm3, %%mm0    \n\t"

  372.             MOVNTQ"     %%mm0, (%0)     \n\t"

  373.             :: "r"(d),"r"(s),"m"(blue_16mask):"memory");

  374.         d += 4;

  375.         s += 16;

  376.     }

  377.     __asm__ volatile(SFENCE:::"memory");

  378.     __asm__ volatile(EMMS:::"memory");

  379.     while (s < end) {

  380.         register int rgb = *(const uint32_t*)s; s += 4;

  381.         *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);

  382.     }

  383. }

  384. 

  385. static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, int src_size)

  386. {

  387.     const uint8_t *s = src;

  388.     const uint8_t *end;

  389.     const uint8_t *mm_end;

  390.     uint16_t *d = (uint16_t *)dst;

  391.     end = s + src_size;

  392.     mm_end = end - 15;

  393.     __asm__ volatile(

  394.         "movq           %3, %%mm5   \n\t"

  395.         "movq           %4, %%mm6   \n\t"

  396.         "movq           %5, %%mm7   \n\t"

  397.         "jmp            2f          \n\t"

  398.         ".p2align        4          \n\t"

  399.         "1:                         \n\t"

  400.         PREFETCH"   32(%1)          \n\t"

  401.         "movd         (%1), %%mm0   \n\t"

  402.         "movd        4(%1), %%mm3   \n\t"

  403.         "punpckldq   8(%1), %%mm0   \n\t"

  404.         "punpckldq  12(%1), %%mm3   \n\t"

  405.         "movq        %%mm0, %%mm1   \n\t"

  406.         "movq        %%mm3, %%mm4   \n\t"

  407.         "pand        %%mm6, %%mm0   \n\t"

  408.         "pand        %%mm6, %%mm3   \n\t"

  409.         "pmaddwd     %%mm7, %%mm0   \n\t"

  410.         "pmaddwd     %%mm7, %%mm3   \n\t"

  411.         "pand        %%mm5, %%mm1   \n\t"

  412.         "pand        %%mm5, %%mm4   \n\t"

  413.         "por         %%mm1, %%mm0   \n\t"

  414.         "por         %%mm4, %%mm3   \n\t"

  415.         "psrld          $6, %%mm0   \n\t"

  416.         "pslld         $10, %%mm3   \n\t"

  417.         "por         %%mm3, %%mm0   \n\t"

  418.         MOVNTQ"      %%mm0, (%0)    \n\t"

  419.         "add           $16,  %1     \n\t"

  420.         "add            $8,  %0     \n\t"

  421.         "2:                         \n\t"

  422.         "cmp            %2,  %1     \n\t"

  423.         " jb            1b          \n\t"

  424.         : "+r" (d), "+r"(s)

  425.         : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)

  426.     );

  427.     __asm__ volatile(SFENCE:::"memory");

  428.     __asm__ volatile(EMMS:::"memory");

  429.     while (s < end) {

  430.         register int rgb = *(const uint32_t*)s; s += 4;

  431.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);

  432.     }

  433. }

  434. 

  435. static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)

  436. {

  437.     const uint8_t *s = src;

  438.     const uint8_t *end;

  439.     const uint8_t *mm_end;

  440.     uint16_t *d = (uint16_t *)dst;

  441.     end = s + src_size;

  442.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  443.     __asm__ volatile(

  444.         "movq          %0, %%mm7    \n\t"

  445.         "movq          %1, %%mm6    \n\t"

  446.         ::"m"(red_15mask),"m"(green_15mask));

  447.     mm_end = end - 15;

  448.     while (s < mm_end) {

  449.         __asm__ volatile(

  450.             PREFETCH"  32(%1)           \n\t"

  451.             "movd        (%1), %%mm0    \n\t"

  452.             "movd       4(%1), %%mm3    \n\t"

  453.             "punpckldq  8(%1), %%mm0    \n\t"

  454.             "punpckldq 12(%1), %%mm3    \n\t"

  455.             "movq       %%mm0, %%mm1    \n\t"

  456.             "movq       %%mm0, %%mm2    \n\t"

  457.             "movq       %%mm3, %%mm4    \n\t"

  458.             "movq       %%mm3, %%mm5    \n\t"

  459.             "psllq         $7, %%mm0    \n\t"

  460.             "psllq         $7, %%mm3    \n\t"

  461.             "pand       %%mm7, %%mm0    \n\t"

  462.             "pand       %%mm7, %%mm3    \n\t"

  463.             "psrlq         $6, %%mm1    \n\t"

  464.             "psrlq         $6, %%mm4    \n\t"

  465.             "pand       %%mm6, %%mm1    \n\t"

  466.             "pand       %%mm6, %%mm4    \n\t"

  467.             "psrlq        $19, %%mm2    \n\t"

  468.             "psrlq        $19, %%mm5    \n\t"

  469.             "pand          %2, %%mm2    \n\t"

  470.             "pand          %2, %%mm5    \n\t"

  471.             "por        %%mm1, %%mm0    \n\t"

  472.             "por        %%mm4, %%mm3    \n\t"

  473.             "por        %%mm2, %%mm0    \n\t"

  474.             "por        %%mm5, %%mm3    \n\t"

  475.             "psllq        $16, %%mm3    \n\t"

  476.             "por        %%mm3, %%mm0    \n\t"

  477.             MOVNTQ"     %%mm0, (%0)     \n\t"

  478.             ::"r"(d),"r"(s),"m"(blue_15mask):"memory");

  479.         d += 4;

  480.         s += 16;

  481.     }

  482.     __asm__ volatile(SFENCE:::"memory");

  483.     __asm__ volatile(EMMS:::"memory");

  484.     while (s < end) {

  485.         register int rgb = *(const uint32_t*)s; s += 4;

  486.         *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);

  487.     }

  488. }

  489. 

  490. static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)

  491. {

  492.     const uint8_t *s = src;

  493.     const uint8_t *end;

  494.     const uint8_t *mm_end;

  495.     uint16_t *d = (uint16_t *)dst;

  496.     end = s + src_size;

  497.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  498.     __asm__ volatile(

  499.         "movq         %0, %%mm7     \n\t"

  500.         "movq         %1, %%mm6     \n\t"

  501.         ::"m"(red_16mask),"m"(green_16mask));

  502.     mm_end = end - 11;

  503.     while (s < mm_end) {

  504.         __asm__ volatile(

  505.             PREFETCH"  32(%1)           \n\t"

  506.             "movd        (%1), %%mm0    \n\t"

  507.             "movd       3(%1), %%mm3    \n\t"

  508.             "punpckldq  6(%1), %%mm0    \n\t"

  509.             "punpckldq  9(%1), %%mm3    \n\t"

  510.             "movq       %%mm0, %%mm1    \n\t"

  511.             "movq       %%mm0, %%mm2    \n\t"

  512.             "movq       %%mm3, %%mm4    \n\t"

  513.             "movq       %%mm3, %%mm5    \n\t"

  514.             "psrlq         $3, %%mm0    \n\t"

  515.             "psrlq         $3, %%mm3    \n\t"

  516.             "pand          %2, %%mm0    \n\t"

  517.             "pand          %2, %%mm3    \n\t"

  518.             "psrlq         $5, %%mm1    \n\t"

  519.             "psrlq         $5, %%mm4    \n\t"

  520.             "pand       %%mm6, %%mm1    \n\t"

  521.             "pand       %%mm6, %%mm4    \n\t"

  522.             "psrlq         $8, %%mm2    \n\t"

  523.             "psrlq         $8, %%mm5    \n\t"

  524.             "pand       %%mm7, %%mm2    \n\t"

  525.             "pand       %%mm7, %%mm5    \n\t"

  526.             "por        %%mm1, %%mm0    \n\t"

  527.             "por        %%mm4, %%mm3    \n\t"

  528.             "por        %%mm2, %%mm0    \n\t"

  529.             "por        %%mm5, %%mm3    \n\t"

  530.             "psllq        $16, %%mm3    \n\t"

  531.             "por        %%mm3, %%mm0    \n\t"

  532.             MOVNTQ"     %%mm0, (%0)     \n\t"

  533.             ::"r"(d),"r"(s),"m"(blue_16mask):"memory");

  534.         d += 4;

  535.         s += 12;

  536.     }

  537.     __asm__ volatile(SFENCE:::"memory");

  538.     __asm__ volatile(EMMS:::"memory");

  539.     while (s < end) {

  540.         const int b = *s++;

  541.         const int g = *s++;

  542.         const int r = *s++;

  543.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);

  544.     }

  545. }

  546. 

  547. static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, int src_size)

  548. {

  549.     const uint8_t *s = src;

  550.     const uint8_t *end;

  551.     const uint8_t *mm_end;

  552.     uint16_t *d = (uint16_t *)dst;

  553.     end = s + src_size;

  554.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  555.     __asm__ volatile(

  556.         "movq         %0, %%mm7     \n\t"

  557.         "movq         %1, %%mm6     \n\t"

  558.         ::"m"(red_16mask),"m"(green_16mask));

  559.     mm_end = end - 15;

  560.     while (s < mm_end) {

  561.         __asm__ volatile(

  562.             PREFETCH"  32(%1)           \n\t"

  563.             "movd        (%1), %%mm0    \n\t"

  564.             "movd       3(%1), %%mm3    \n\t"

  565.             "punpckldq  6(%1), %%mm0    \n\t"

  566.             "punpckldq  9(%1), %%mm3    \n\t"

  567.             "movq       %%mm0, %%mm1    \n\t"

  568.             "movq       %%mm0, %%mm2    \n\t"

  569.             "movq       %%mm3, %%mm4    \n\t"

  570.             "movq       %%mm3, %%mm5    \n\t"

  571.             "psllq         $8, %%mm0    \n\t"

  572.             "psllq         $8, %%mm3    \n\t"

  573.             "pand       %%mm7, %%mm0    \n\t"

  574.             "pand       %%mm7, %%mm3    \n\t"

  575.             "psrlq         $5, %%mm1    \n\t"

  576.             "psrlq         $5, %%mm4    \n\t"

  577.             "pand       %%mm6, %%mm1    \n\t"

  578.             "pand       %%mm6, %%mm4    \n\t"

  579.             "psrlq        $19, %%mm2    \n\t"

  580.             "psrlq        $19, %%mm5    \n\t"

  581.             "pand          %2, %%mm2    \n\t"

  582.             "pand          %2, %%mm5    \n\t"

  583.             "por        %%mm1, %%mm0    \n\t"

  584.             "por        %%mm4, %%mm3    \n\t"

  585.             "por        %%mm2, %%mm0    \n\t"

  586.             "por        %%mm5, %%mm3    \n\t"

  587.             "psllq        $16, %%mm3    \n\t"

  588.             "por        %%mm3, %%mm0    \n\t"

  589.             MOVNTQ"     %%mm0, (%0)     \n\t"

  590.             ::"r"(d),"r"(s),"m"(blue_16mask):"memory");

  591.         d += 4;

  592.         s += 12;

  593.     }

  594.     __asm__ volatile(SFENCE:::"memory");

  595.     __asm__ volatile(EMMS:::"memory");

  596.     while (s < end) {

  597.         const int r = *s++;

  598.         const int g = *s++;

  599.         const int b = *s++;

  600.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);

  601.     }

  602. }

  603. 

  604. static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)

  605. {

  606.     const uint8_t *s = src;

  607.     const uint8_t *end;

  608.     const uint8_t *mm_end;

  609.     uint16_t *d = (uint16_t *)dst;

  610.     end = s + src_size;

  611.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  612.     __asm__ volatile(

  613.         "movq          %0, %%mm7    \n\t"

  614.         "movq          %1, %%mm6    \n\t"

  615.         ::"m"(red_15mask),"m"(green_15mask));

  616.     mm_end = end - 11;

  617.     while (s < mm_end) {

  618.         __asm__ volatile(

  619.             PREFETCH"  32(%1)           \n\t"

  620.             "movd        (%1), %%mm0    \n\t"

  621.             "movd       3(%1), %%mm3    \n\t"

  622.             "punpckldq  6(%1), %%mm0    \n\t"

  623.             "punpckldq  9(%1), %%mm3    \n\t"

  624.             "movq       %%mm0, %%mm1    \n\t"

  625.             "movq       %%mm0, %%mm2    \n\t"

  626.             "movq       %%mm3, %%mm4    \n\t"

  627.             "movq       %%mm3, %%mm5    \n\t"

  628.             "psrlq         $3, %%mm0    \n\t"

  629.             "psrlq         $3, %%mm3    \n\t"

  630.             "pand          %2, %%mm0    \n\t"

  631.             "pand          %2, %%mm3    \n\t"

  632.             "psrlq         $6, %%mm1    \n\t"

  633.             "psrlq         $6, %%mm4    \n\t"

  634.             "pand       %%mm6, %%mm1    \n\t"

  635.             "pand       %%mm6, %%mm4    \n\t"

  636.             "psrlq         $9, %%mm2    \n\t"

  637.             "psrlq         $9, %%mm5    \n\t"

  638.             "pand       %%mm7, %%mm2    \n\t"

  639.             "pand       %%mm7, %%mm5    \n\t"

  640.             "por        %%mm1, %%mm0    \n\t"

  641.             "por        %%mm4, %%mm3    \n\t"

  642.             "por        %%mm2, %%mm0    \n\t"

  643.             "por        %%mm5, %%mm3    \n\t"

  644.             "psllq        $16, %%mm3    \n\t"

  645.             "por        %%mm3, %%mm0    \n\t"

  646.             MOVNTQ"     %%mm0, (%0)     \n\t"

  647.             ::"r"(d),"r"(s),"m"(blue_15mask):"memory");

  648.         d += 4;

  649.         s += 12;

  650.     }

  651.     __asm__ volatile(SFENCE:::"memory");

  652.     __asm__ volatile(EMMS:::"memory");

  653.     while (s < end) {

  654.         const int b = *s++;

  655.         const int g = *s++;

  656.         const int r = *s++;

  657.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);

  658.     }

  659. }

  660. 

  661. static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, int src_size)

  662. {

  663.     const uint8_t *s = src;

  664.     const uint8_t *end;

  665.     const uint8_t *mm_end;

  666.     uint16_t *d = (uint16_t *)dst;

  667.     end = s + src_size;

  668.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");

  669.     __asm__ volatile(

  670.         "movq         %0, %%mm7     \n\t"

  671.         "movq         %1, %%mm6     \n\t"

  672.         ::"m"(red_15mask),"m"(green_15mask));

  673.     mm_end = end - 15;

  674.     while (s < mm_end) {

  675.         __asm__ volatile(

  676.             PREFETCH" 32(%1)            \n\t"

  677.             "movd       (%1), %%mm0     \n\t"

  678.             "movd      3(%1), %%mm3     \n\t"

  679.             "punpckldq 6(%1), %%mm0     \n\t"

  680.             "punpckldq 9(%1), %%mm3     \n\t"

  681.             "movq      %%mm0, %%mm1     \n\t"

  682.             "movq      %%mm0, %%mm2     \n\t"

  683.             "movq      %%mm3, %%mm4     \n\t"

  684.             "movq      %%mm3, %%mm5     \n\t"

  685.             "psllq        $7, %%mm0     \n\t"

  686.             "psllq        $7, %%mm3     \n\t"

  687.             "pand      %%mm7, %%mm0     \n\t"

  688.             "pand      %%mm7, %%mm3     \n\t"

  689.             "psrlq        $6, %%mm1     \n\t"

  690.             "psrlq        $6, %%mm4     \n\t"

  691.             "pand      %%mm6, %%mm1     \n\t"

  692.             "pand      %%mm6, %%mm4     \n\t"

  693.             "psrlq       $19, %%mm2     \n\t"

  694.             "psrlq       $19, %%mm5     \n\t"

  695.             "pand         %2, %%mm2     \n\t"

  696.             "pand         %2, %%mm5     \n\t"

  697.             "por       %%mm1, %%mm0     \n\t"

  698.             "por       %%mm4, %%mm3     \n\t"

  699.             "por       %%mm2, %%mm0     \n\t"

  700.             "por       %%mm5, %%mm3     \n\t"

  701.             "psllq       $16, %%mm3     \n\t"

  702.             "por       %%mm3, %%mm0     \n\t"

  703.             MOVNTQ"    %%mm0, (%0)      \n\t"

  704.             ::"r"(d),"r"(s),"m"(blue_15mask):"memory");

  705.         d += 4;

  706.         s += 12;

  707.     }

  708.     __asm__ volatile(SFENCE:::"memory");

  709.     __asm__ volatile(EMMS:::"memory");

  710.     while (s < end) {

  711.         const int r = *s++;

  712.         const int g = *s++;

  713.         const int b = *s++;

  714.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);

  715.     }

  716. }

  717. 

  718. static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)

  719. {

  720.     const uint16_t *end;

  721.     const uint16_t *mm_end;

  722.     uint8_t *d = dst;

  723.     const uint16_t *s = (const uint16_t*)src;

  724.     end = s + src_size/2;

  725.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  726.     mm_end = end - 7;

  727.     while (s < mm_end) {

  728.         __asm__ volatile(

  729.             PREFETCH"  32(%1)           \n\t"

  730.             "movq        (%1), %%mm0    \n\t"

  731.             "movq        (%1), %%mm1    \n\t"

  732.             "movq        (%1), %%mm2    \n\t"

  733.             "pand          %2, %%mm0    \n\t"

  734.             "pand          %3, %%mm1    \n\t"

  735.             "pand          %4, %%mm2    \n\t"

  736.             "psllq         $5, %%mm0    \n\t"

  737.             "pmulhw        "MANGLE(mul15_mid)", %%mm0    \n\t"

  738.             "pmulhw        "MANGLE(mul15_mid)", %%mm1    \n\t"

  739.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  740.             "movq       %%mm0, %%mm3    \n\t"

  741.             "movq       %%mm1, %%mm4    \n\t"

  742.             "movq       %%mm2, %%mm5    \n\t"

  743.             "punpcklwd     %5, %%mm0    \n\t"

  744.             "punpcklwd     %5, %%mm1    \n\t"

  745.             "punpcklwd     %5, %%mm2    \n\t"

  746.             "punpckhwd     %5, %%mm3    \n\t"

  747.             "punpckhwd     %5, %%mm4    \n\t"

  748.             "punpckhwd     %5, %%mm5    \n\t"

  749.             "psllq         $8, %%mm1    \n\t"

  750.             "psllq        $16, %%mm2    \n\t"

  751.             "por        %%mm1, %%mm0    \n\t"

  752.             "por        %%mm2, %%mm0    \n\t"

  753.             "psllq         $8, %%mm4    \n\t"

  754.             "psllq        $16, %%mm5    \n\t"

  755.             "por        %%mm4, %%mm3    \n\t"

  756.             "por        %%mm5, %%mm3    \n\t"

  757. 

  758.             "movq       %%mm0, %%mm6    \n\t"

  759.             "movq       %%mm3, %%mm7    \n\t"

  760. 

  761.             "movq       8(%1), %%mm0    \n\t"

  762.             "movq       8(%1), %%mm1    \n\t"

  763.             "movq       8(%1), %%mm2    \n\t"

  764.             "pand          %2, %%mm0    \n\t"

  765.             "pand          %3, %%mm1    \n\t"

  766.             "pand          %4, %%mm2    \n\t"

  767.             "psllq         $5, %%mm0    \n\t"

  768.             "pmulhw        "MANGLE(mul15_mid)", %%mm0    \n\t"

  769.             "pmulhw        "MANGLE(mul15_mid)", %%mm1    \n\t"

  770.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  771.             "movq       %%mm0, %%mm3    \n\t"

  772.             "movq       %%mm1, %%mm4    \n\t"

  773.             "movq       %%mm2, %%mm5    \n\t"

  774.             "punpcklwd     %5, %%mm0    \n\t"

  775.             "punpcklwd     %5, %%mm1    \n\t"

  776.             "punpcklwd     %5, %%mm2    \n\t"

  777.             "punpckhwd     %5, %%mm3    \n\t"

  778.             "punpckhwd     %5, %%mm4    \n\t"

  779.             "punpckhwd     %5, %%mm5    \n\t"

  780.             "psllq         $8, %%mm1    \n\t"

  781.             "psllq        $16, %%mm2    \n\t"

  782.             "por        %%mm1, %%mm0    \n\t"

  783.             "por        %%mm2, %%mm0    \n\t"

  784.             "psllq         $8, %%mm4    \n\t"

  785.             "psllq        $16, %%mm5    \n\t"

  786.             "por        %%mm4, %%mm3    \n\t"

  787.             "por        %%mm5, %%mm3    \n\t"

  788. 

  789.             :"=m"(*d)

  790.             :"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)

  791.              NAMED_CONSTRAINTS_ADD(mul15_mid,mul15_hi)

  792.             :"memory");

  793.         /* borrowed 32 to 24 */

  794.         __asm__ volatile(

  795.             "movq       %%mm0, %%mm4    \n\t"

  796.             "movq       %%mm3, %%mm5    \n\t"

  797.             "movq       %%mm6, %%mm0    \n\t"

  798.             "movq       %%mm7, %%mm1    \n\t"

  799. 

  800.             "movq       %%mm4, %%mm6    \n\t"

  801.             "movq       %%mm5, %%mm7    \n\t"

  802.             "movq       %%mm0, %%mm2    \n\t"

  803.             "movq       %%mm1, %%mm3    \n\t"

  804. 

  805.             STORE_BGR24_MMX

  806. 

  807.             :: "r"(d), "m"(*s)

  808.               NAMED_CONSTRAINTS_ADD(mask24l,mask24h)

  809.             :"memory");

  810.         d += 24;

  811.         s += 8;

  812.     }

  813.     __asm__ volatile(SFENCE:::"memory");

  814.     __asm__ volatile(EMMS:::"memory");

  815.     while (s < end) {

  816.         register uint16_t bgr;

  817.         bgr = *s++;

  818.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);

  819.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);

  820.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);

  821.     }

  822. }

  823. 

  824. static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)

  825. {

  826.     const uint16_t *end;

  827.     const uint16_t *mm_end;

  828.     uint8_t *d = (uint8_t *)dst;

  829.     const uint16_t *s = (const uint16_t *)src;

  830.     end = s + src_size/2;

  831.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  832.     mm_end = end - 7;

  833.     while (s < mm_end) {

  834.         __asm__ volatile(

  835.             PREFETCH"  32(%1)           \n\t"

  836.             "movq        (%1), %%mm0    \n\t"

  837.             "movq        (%1), %%mm1    \n\t"

  838.             "movq        (%1), %%mm2    \n\t"

  839.             "pand          %2, %%mm0    \n\t"

  840.             "pand          %3, %%mm1    \n\t"

  841.             "pand          %4, %%mm2    \n\t"

  842.             "psllq         $5, %%mm0    \n\t"

  843.             "psrlq         $1, %%mm2    \n\t"

  844.             "pmulhw        "MANGLE(mul15_mid)", %%mm0    \n\t"

  845.             "pmulhw        "MANGLE(mul16_mid)", %%mm1    \n\t"

  846.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  847.             "movq       %%mm0, %%mm3    \n\t"

  848.             "movq       %%mm1, %%mm4    \n\t"

  849.             "movq       %%mm2, %%mm5    \n\t"

  850.             "punpcklwd     %5, %%mm0    \n\t"

  851.             "punpcklwd     %5, %%mm1    \n\t"

  852.             "punpcklwd     %5, %%mm2    \n\t"

  853.             "punpckhwd     %5, %%mm3    \n\t"

  854.             "punpckhwd     %5, %%mm4    \n\t"

  855.             "punpckhwd     %5, %%mm5    \n\t"

  856.             "psllq         $8, %%mm1    \n\t"

  857.             "psllq        $16, %%mm2    \n\t"

  858.             "por        %%mm1, %%mm0    \n\t"

  859.             "por        %%mm2, %%mm0    \n\t"

  860.             "psllq         $8, %%mm4    \n\t"

  861.             "psllq        $16, %%mm5    \n\t"

  862.             "por        %%mm4, %%mm3    \n\t"

  863.             "por        %%mm5, %%mm3    \n\t"

  864. 

  865.             "movq       %%mm0, %%mm6    \n\t"

  866.             "movq       %%mm3, %%mm7    \n\t"

  867. 

  868.             "movq       8(%1), %%mm0    \n\t"

  869.             "movq       8(%1), %%mm1    \n\t"

  870.             "movq       8(%1), %%mm2    \n\t"

  871.             "pand          %2, %%mm0    \n\t"

  872.             "pand          %3, %%mm1    \n\t"

  873.             "pand          %4, %%mm2    \n\t"

  874.             "psllq         $5, %%mm0    \n\t"

  875.             "psrlq         $1, %%mm2    \n\t"

  876.             "pmulhw        "MANGLE(mul15_mid)", %%mm0    \n\t"

  877.             "pmulhw        "MANGLE(mul16_mid)", %%mm1    \n\t"

  878.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  879.             "movq       %%mm0, %%mm3    \n\t"

  880.             "movq       %%mm1, %%mm4    \n\t"

  881.             "movq       %%mm2, %%mm5    \n\t"

  882.             "punpcklwd     %5, %%mm0    \n\t"

  883.             "punpcklwd     %5, %%mm1    \n\t"

  884.             "punpcklwd     %5, %%mm2    \n\t"

  885.             "punpckhwd     %5, %%mm3    \n\t"

  886.             "punpckhwd     %5, %%mm4    \n\t"

  887.             "punpckhwd     %5, %%mm5    \n\t"

  888.             "psllq         $8, %%mm1    \n\t"

  889.             "psllq        $16, %%mm2    \n\t"

  890.             "por        %%mm1, %%mm0    \n\t"

  891.             "por        %%mm2, %%mm0    \n\t"

  892.             "psllq         $8, %%mm4    \n\t"

  893.             "psllq        $16, %%mm5    \n\t"

  894.             "por        %%mm4, %%mm3    \n\t"

  895.             "por        %%mm5, %%mm3    \n\t"

  896.             :"=m"(*d)

  897.             :"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)

  898.              NAMED_CONSTRAINTS_ADD(mul15_mid,mul16_mid,mul15_hi)

  899.             :"memory");

  900.         /* borrowed 32 to 24 */

  901.         __asm__ volatile(

  902.             "movq       %%mm0, %%mm4    \n\t"

  903.             "movq       %%mm3, %%mm5    \n\t"

  904.             "movq       %%mm6, %%mm0    \n\t"

  905.             "movq       %%mm7, %%mm1    \n\t"

  906. 

  907.             "movq       %%mm4, %%mm6    \n\t"

  908.             "movq       %%mm5, %%mm7    \n\t"

  909.             "movq       %%mm0, %%mm2    \n\t"

  910.             "movq       %%mm1, %%mm3    \n\t"

  911. 

  912.             STORE_BGR24_MMX

  913. 

  914.             :: "r"(d), "m"(*s)

  915.               NAMED_CONSTRAINTS_ADD(mask24l,mask24h)

  916.             :"memory");

  917.         d += 24;

  918.         s += 8;

  919.     }

  920.     __asm__ volatile(SFENCE:::"memory");

  921.     __asm__ volatile(EMMS:::"memory");

  922.     while (s < end) {

  923.         register uint16_t bgr;

  924.         bgr = *s++;

  925.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);

  926.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);

  927.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);

  928.     }

  929. }

  930. 

  931. /*

  932.  * mm0 = 00 B3 00 B2 00 B1 00 B0

  933.  * mm1 = 00 G3 00 G2 00 G1 00 G0

  934.  * mm2 = 00 R3 00 R2 00 R1 00 R0

  935.  * mm6 = FF FF FF FF FF FF FF FF

  936.  * mm7 = 00 00 00 00 00 00 00 00

  937.  */

  938. #define PACK_RGB32 \

  939.     "packuswb   %%mm7, %%mm0    \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \

  940.     "packuswb   %%mm7, %%mm1    \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \

  941.     "packuswb   %%mm7, %%mm2    \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \

  942.     "punpcklbw  %%mm1, %%mm0    \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \

  943.     "punpcklbw  %%mm6, %%mm2    \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \

  944.     "movq       %%mm0, %%mm3    \n\t"                               \

  945.     "punpcklwd  %%mm2, %%mm0    \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \

  946.     "punpckhwd  %%mm2, %%mm3    \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \

  947.     MOVNTQ"     %%mm0,  (%0)    \n\t"                               \

  948.     MOVNTQ"     %%mm3, 8(%0)    \n\t"                               \

  949. 

  950. static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, int src_size)

  951. {

  952.     const uint16_t *end;

  953.     const uint16_t *mm_end;

  954.     uint8_t *d = dst;

  955.     const uint16_t *s = (const uint16_t *)src;

  956.     end = s + src_size/2;

  957.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  958.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");

  959.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");

  960.     mm_end = end - 3;

  961.     while (s < mm_end) {

  962.         __asm__ volatile(

  963.             PREFETCH"  32(%1)           \n\t"

  964.             "movq        (%1), %%mm0    \n\t"

  965.             "movq        (%1), %%mm1    \n\t"

  966.             "movq        (%1), %%mm2    \n\t"

  967.             "pand          %2, %%mm0    \n\t"

  968.             "pand          %3, %%mm1    \n\t"

  969.             "pand          %4, %%mm2    \n\t"

  970.             "psllq         $5, %%mm0    \n\t"

  971.             "pmulhw        %5, %%mm0    \n\t"

  972.             "pmulhw        %5, %%mm1    \n\t"

  973.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  974.             PACK_RGB32

  975.             ::"r"(d),"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r) ,"m"(mul15_mid)

  976.               NAMED_CONSTRAINTS_ADD(mul15_hi)

  977.             :"memory");

  978.         d += 16;

  979.         s += 4;

  980.     }

  981.     __asm__ volatile(SFENCE:::"memory");

  982.     __asm__ volatile(EMMS:::"memory");

  983.     while (s < end) {

  984.         register uint16_t bgr;

  985.         bgr = *s++;

  986.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);

  987.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);

  988.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);

  989.         *d++ = 255;

  990.     }

  991. }

  992. 

  993. static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, int src_size)

  994. {

  995.     const uint16_t *end;

  996.     const uint16_t *mm_end;

  997.     uint8_t *d = dst;

  998.     const uint16_t *s = (const uint16_t*)src;

  999.     end = s + src_size/2;

  1000.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");

  1001.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");

  1002.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");

  1003.     mm_end = end - 3;

  1004.     while (s < mm_end) {

  1005.         __asm__ volatile(

  1006.             PREFETCH"  32(%1)           \n\t"

  1007.             "movq        (%1), %%mm0    \n\t"

  1008.             "movq        (%1), %%mm1    \n\t"

  1009.             "movq        (%1), %%mm2    \n\t"

  1010.             "pand          %2, %%mm0    \n\t"

  1011.             "pand          %3, %%mm1    \n\t"

  1012.             "pand          %4, %%mm2    \n\t"

  1013.             "psllq         $5, %%mm0    \n\t"

  1014.             "psrlq         $1, %%mm2    \n\t"

  1015.             "pmulhw        %5, %%mm0    \n\t"

  1016.             "pmulhw        "MANGLE(mul16_mid)", %%mm1    \n\t"

  1017.             "pmulhw        "MANGLE(mul15_hi)", %%mm2    \n\t"

  1018.             PACK_RGB32

  1019.             ::"r"(d),"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mul15_mid)

  1020.               NAMED_CONSTRAINTS_ADD(mul16_mid,mul15_hi)

  1021.             :"memory");

  1022.         d += 16;

  1023.         s += 4;

  1024.     }

  1025.     __asm__ volatile(SFENCE:::"memory");

  1026.     __asm__ volatile(EMMS:::"memory");

  1027.     while (s < end) {

  1028.         register uint16_t bgr;

  1029.         bgr = *s++;

  1030.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);

  1031.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);

  1032.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);

  1033.         *d++ = 255;

  1034.     }

  1035. }

  1036. 

  1037. static inline void RENAME(shuffle_bytes_2103)(const uint8_t *src, uint8_t *dst, int src_size)

  1038. {

  1039.     x86_reg idx = 15 - src_size;

  1040.     const uint8_t *s = src-idx;

  1041.     uint8_t *d = dst-idx;

  1042.     __asm__ volatile(

  1043.         "test          %0, %0           \n\t"

  1044.         "jns           2f               \n\t"

  1045.         PREFETCH"       (%1, %0)        \n\t"

  1046.         "movq          %3, %%mm7        \n\t"

  1047.         "pxor          %4, %%mm7        \n\t"

  1048.         "movq       %%mm7, %%mm6        \n\t"

  1049.         "pxor          %5, %%mm7        \n\t"

  1050.         ".p2align       4               \n\t"

  1051.         "1:                             \n\t"

  1052.         PREFETCH"     32(%1, %0)        \n\t"

  1053.         "movq           (%1, %0), %%mm0 \n\t"

  1054.         "movq          8(%1, %0), %%mm1 \n\t"

  1055. # if COMPILE_TEMPLATE_MMXEXT

  1056.         "pshufw      $177, %%mm0, %%mm3 \n\t"

  1057.         "pshufw      $177, %%mm1, %%mm5 \n\t"

  1058.         "pand       %%mm7, %%mm0        \n\t"

  1059.         "pand       %%mm6, %%mm3        \n\t"

  1060.         "pand       %%mm7, %%mm1        \n\t"

  1061.         "pand       %%mm6, %%mm5        \n\t"

  1062.         "por        %%mm3, %%mm0        \n\t"

  1063.         "por        %%mm5, %%mm1        \n\t"

  1064. # else

  1065.         "movq       %%mm0, %%mm2        \n\t"

  1066.         "movq       %%mm1, %%mm4        \n\t"

  1067.         "pand       %%mm7, %%mm0        \n\t"

  1068.         "pand       %%mm6, %%mm2        \n\t"

  1069.         "pand       %%mm7, %%mm1        \n\t"

  1070.         "pand       %%mm6, %%mm4        \n\t"

  1071.         "movq       %%mm2, %%mm3        \n\t"

  1072.         "movq       %%mm4, %%mm5        \n\t"

  1073.         "pslld        $16, %%mm2        \n\t"

  1074.         "psrld        $16, %%mm3        \n\t"

  1075.         "pslld        $16, %%mm4        \n\t"

  1076.         "psrld        $16, %%mm5        \n\t"

  1077.         "por        %%mm2, %%mm0        \n\t"

  1078.         "por        %%mm4, %%mm1        \n\t"

  1079.         "por        %%mm3, %%mm0        \n\t"

  1080.         "por        %%mm5, %%mm1        \n\t"

  1081. # endif

  1082.         MOVNTQ"     %%mm0,  (%2, %0)    \n\t"

  1083.         MOVNTQ"     %%mm1, 8(%2, %0)    \n\t"

  1084.         "add          $16, %0           \n\t"

  1085.         "js            1b               \n\t"

  1086.         SFENCE"                         \n\t"

  1087.         EMMS"                           \n\t"

  1088.         "2:                             \n\t"

  1089.         : "+&r"(idx)

  1090.         : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)

  1091.         : "memory");

  1092.     for (; idx<15; idx+=4) {

  1093.         register unsigned v  = *(const uint32_t *)&s[idx], g = v & 0xff00ff00;

  1094.         v &= 0xff00ff;

  1095.         *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);

  1096.     }

  1097. }

  1098. 

  1099. static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)

  1100. {

  1101.     unsigned i;

  1102.     x86_reg mmx_size= 23 - src_size;

  1103.     __asm__ volatile (

  1104.         "test             %%"FF_REG_a", %%"FF_REG_a"    \n\t"

  1105.         "jns                     2f                     \n\t"

  1106.         "movq     "MANGLE(mask24r)", %%mm5              \n\t"

  1107.         "movq     "MANGLE(mask24g)", %%mm6              \n\t"

  1108.         "movq     "MANGLE(mask24b)", %%mm7              \n\t"

  1109.         ".p2align                 4                     \n\t"

  1110.         "1:                                             \n\t"

  1111.         PREFETCH" 32(%1, %%"FF_REG_a")                  \n\t"

  1112.         "movq    (%1, %%"FF_REG_a"), %%mm0              \n\t" // BGR BGR BG

  1113.         "movq    (%1, %%"FF_REG_a"), %%mm1              \n\t" // BGR BGR BG

  1114.         "movq   2(%1, %%"FF_REG_a"), %%mm2              \n\t" // R BGR BGR B

  1115.         "psllq                  $16, %%mm0              \n\t" // 00 BGR BGR

  1116.         "pand                 %%mm5, %%mm0              \n\t"

  1117.         "pand                 %%mm6, %%mm1              \n\t"

  1118.         "pand                 %%mm7, %%mm2              \n\t"

  1119.         "por                  %%mm0, %%mm1              \n\t"

  1120.         "por                  %%mm2, %%mm1              \n\t"

  1121.         "movq   6(%1, %%"FF_REG_a"), %%mm0              \n\t" // BGR BGR BG

  1122.         MOVNTQ"               %%mm1,(%2, %%"FF_REG_a")  \n\t" // RGB RGB RG

  1123.         "movq   8(%1, %%"FF_REG_a"), %%mm1              \n\t" // R BGR BGR B

  1124.         "movq  10(%1, %%"FF_REG_a"), %%mm2              \n\t" // GR BGR BGR

  1125.         "pand                 %%mm7, %%mm0              \n\t"

  1126.         "pand                 %%mm5, %%mm1              \n\t"

  1127.         "pand                 %%mm6, %%mm2              \n\t"

  1128.         "por                  %%mm0, %%mm1              \n\t"

  1129.         "por                  %%mm2, %%mm1              \n\t"

  1130.         "movq  14(%1, %%"FF_REG_a"), %%mm0              \n\t" // R BGR BGR B

  1131.         MOVNTQ"               %%mm1, 8(%2, %%"FF_REG_a")\n\t" // B RGB RGB R

  1132.         "movq  16(%1, %%"FF_REG_a"), %%mm1              \n\t" // GR BGR BGR

  1133.         "movq  18(%1, %%"FF_REG_a"), %%mm2              \n\t" // BGR BGR BG

  1134.         "pand                 %%mm6, %%mm0              \n\t"

  1135.         "pand                 %%mm7, %%mm1              \n\t"

  1136.         "pand                 %%mm5, %%mm2              \n\t"

  1137.         "por                  %%mm0, %%mm1              \n\t"

  1138.         "por                  %%mm2, %%mm1              \n\t"

  1139.         MOVNTQ"               %%mm1, 16(%2, %%"FF_REG_a") \n\t"

  1140.         "add                    $24, %%"FF_REG_a"       \n\t"

  1141.         " js                     1b                     \n\t"

  1142.         "2:                                             \n\t"

  1143.         : "+a" (mmx_size)

  1144.         : "r" (src-mmx_size), "r"(dst-mmx_size)

  1145.           NAMED_CONSTRAINTS_ADD(mask24r,mask24g,mask24b)

  1146.     );

  1147. 

  1148.     __asm__ volatile(SFENCE:::"memory");

  1149.     __asm__ volatile(EMMS:::"memory");

  1150. 

  1151.     if (mmx_size==23) return; //finished, was multiple of 8

  1152. 

  1153.     src+= src_size;

  1154.     dst+= src_size;

  1155.     src_size= 23-mmx_size;

  1156.     src-= src_size;

  1157.     dst-= src_size;

  1158.     for (i=0; i<src_size; i+=3) {

  1159.         register uint8_t x;

  1160.         x          = src[i + 2];

  1161.         dst[i + 1] = src[i + 1];

  1162.         dst[i + 2] = src[i + 0];

  1163.         dst[i + 0] = x;

  1164.     }

  1165. }

  1166. 

  1167. static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1168.                                            int width, int height,

  1169.                                            int lumStride, int chromStride, int dstStride, int vertLumPerChroma)

  1170. {

  1171.     int y;

  1172.     const x86_reg chromWidth= width>>1;

  1173.     for (y=0; y<height; y++) {

  1174.         //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)

  1175.         __asm__ volatile(

  1176.             "xor                 %%"FF_REG_a", %%"FF_REG_a" \n\t"

  1177.             ".p2align                    4              \n\t"

  1178.             "1:                                         \n\t"

  1179.             PREFETCH" 32(%1, %%"FF_REG_a", 2)           \n\t"

  1180.             PREFETCH" 32(%2, %%"FF_REG_a")              \n\t"

  1181.             PREFETCH" 32(%3, %%"FF_REG_a")              \n\t"

  1182.             "movq       (%2, %%"FF_REG_a"), %%mm0       \n\t" // U(0)

  1183.             "movq                    %%mm0, %%mm2       \n\t" // U(0)

  1184.             "movq       (%3, %%"FF_REG_a"), %%mm1       \n\t" // V(0)

  1185.             "punpcklbw               %%mm1, %%mm0       \n\t" // UVUV UVUV(0)

  1186.             "punpckhbw               %%mm1, %%mm2       \n\t" // UVUV UVUV(8)

  1187. 

  1188.             "movq     (%1, %%"FF_REG_a",2), %%mm3       \n\t" // Y(0)

  1189.             "movq    8(%1, %%"FF_REG_a",2), %%mm5       \n\t" // Y(8)

  1190.             "movq                    %%mm3, %%mm4       \n\t" // Y(0)

  1191.             "movq                    %%mm5, %%mm6       \n\t" // Y(8)

  1192.             "punpcklbw               %%mm0, %%mm3       \n\t" // YUYV YUYV(0)

  1193.             "punpckhbw               %%mm0, %%mm4       \n\t" // YUYV YUYV(4)

  1194.             "punpcklbw               %%mm2, %%mm5       \n\t" // YUYV YUYV(8)

  1195.             "punpckhbw               %%mm2, %%mm6       \n\t" // YUYV YUYV(12)

  1196. 

  1197.             MOVNTQ"                  %%mm3,   (%0, %%"FF_REG_a", 4)    \n\t"

  1198.             MOVNTQ"                  %%mm4,  8(%0, %%"FF_REG_a", 4)    \n\t"

  1199.             MOVNTQ"                  %%mm5, 16(%0, %%"FF_REG_a", 4)    \n\t"

  1200.             MOVNTQ"                  %%mm6, 24(%0, %%"FF_REG_a", 4)    \n\t"

  1201. 

  1202.             "add                        $8, %%"FF_REG_a" \n\t"

  1203.             "cmp                        %4, %%"FF_REG_a" \n\t"

  1204.             " jb                        1b               \n\t"

  1205.             ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)

  1206.             : "%"FF_REG_a

  1207.         );

  1208.         if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {

  1209.             usrc += chromStride;

  1210.             vsrc += chromStride;

  1211.         }

  1212.         ysrc += lumStride;

  1213.         dst  += dstStride;

  1214.     }

  1215.     __asm__(EMMS"       \n\t"

  1216.             SFENCE"     \n\t"

  1217.             :::"memory");

  1218. }

  1219. 

  1220. /**

  1221.  * Height should be a multiple of 2 and width should be a multiple of 16.

  1222.  * (If this is a problem for anyone then tell me, and I will fix it.)

  1223.  */

  1224. static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1225.                                       int width, int height,

  1226.                                       int lumStride, int chromStride, int dstStride)

  1227. {

  1228.     //FIXME interpolate chroma

  1229.     RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);

  1230. }

  1231. 

  1232. static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1233.                                            int width, int height,

  1234.                                            int lumStride, int chromStride, int dstStride, int vertLumPerChroma)

  1235. {

  1236.     int y;

  1237.     const x86_reg chromWidth= width>>1;

  1238.     for (y=0; y<height; y++) {

  1239.         //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)

  1240.         __asm__ volatile(

  1241.             "xor             %%"FF_REG_a", %%"FF_REG_a" \n\t"

  1242.             ".p2align                   4               \n\t"

  1243.             "1:                                         \n\t"

  1244.             PREFETCH" 32(%1, %%"FF_REG_a", 2)           \n\t"

  1245.             PREFETCH" 32(%2, %%"FF_REG_a")              \n\t"

  1246.             PREFETCH" 32(%3, %%"FF_REG_a")              \n\t"

  1247.             "movq      (%2, %%"FF_REG_a"), %%mm0        \n\t" // U(0)

  1248.             "movq                   %%mm0, %%mm2        \n\t" // U(0)

  1249.             "movq      (%3, %%"FF_REG_a"), %%mm1        \n\t" // V(0)

  1250.             "punpcklbw              %%mm1, %%mm0        \n\t" // UVUV UVUV(0)

  1251.             "punpckhbw              %%mm1, %%mm2        \n\t" // UVUV UVUV(8)

  1252. 

  1253.             "movq    (%1, %%"FF_REG_a",2), %%mm3        \n\t" // Y(0)

  1254.             "movq   8(%1, %%"FF_REG_a",2), %%mm5        \n\t" // Y(8)

  1255.             "movq                   %%mm0, %%mm4        \n\t" // Y(0)

  1256.             "movq                   %%mm2, %%mm6        \n\t" // Y(8)

  1257.             "punpcklbw              %%mm3, %%mm0        \n\t" // YUYV YUYV(0)

  1258.             "punpckhbw              %%mm3, %%mm4        \n\t" // YUYV YUYV(4)

  1259.             "punpcklbw              %%mm5, %%mm2        \n\t" // YUYV YUYV(8)

  1260.             "punpckhbw              %%mm5, %%mm6        \n\t" // YUYV YUYV(12)

  1261. 

  1262.             MOVNTQ"                 %%mm0,   (%0, %%"FF_REG_a", 4)     \n\t"

  1263.             MOVNTQ"                 %%mm4,  8(%0, %%"FF_REG_a", 4)     \n\t"

  1264.             MOVNTQ"                 %%mm2, 16(%0, %%"FF_REG_a", 4)     \n\t"

  1265.             MOVNTQ"                 %%mm6, 24(%0, %%"FF_REG_a", 4)     \n\t"

  1266. 

  1267.             "add                       $8, %%"FF_REG_a" \n\t"

  1268.             "cmp                       %4, %%"FF_REG_a" \n\t"

  1269.             " jb                       1b               \n\t"

  1270.             ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)

  1271.             : "%"FF_REG_a

  1272.         );

  1273.         if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {

  1274.             usrc += chromStride;

  1275.             vsrc += chromStride;

  1276.         }

  1277.         ysrc += lumStride;

  1278.         dst += dstStride;

  1279.     }

  1280.     __asm__(EMMS"       \n\t"

  1281.             SFENCE"     \n\t"

  1282.             :::"memory");

  1283. }

  1284. 

  1285. /**

  1286.  * Height should be a multiple of 2 and width should be a multiple of 16

  1287.  * (If this is a problem for anyone then tell me, and I will fix it.)

  1288.  */

  1289. static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1290.                                       int width, int height,

  1291.                                       int lumStride, int chromStride, int dstStride)

  1292. {

  1293.     //FIXME interpolate chroma

  1294.     RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);

  1295. }

  1296. 

  1297. /**

  1298.  * Width should be a multiple of 16.

  1299.  */

  1300. static inline void RENAME(yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1301.                                          int width, int height,

  1302.                                          int lumStride, int chromStride, int dstStride)

  1303. {

  1304.     RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);

  1305. }

  1306. 

  1307. /**

  1308.  * Width should be a multiple of 16.

  1309.  */

  1310. static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1311.                                          int width, int height,

  1312.                                          int lumStride, int chromStride, int dstStride)

  1313. {

  1314.     RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);

  1315. }

  1316. 

  1317. /**

  1318.  * Height should be a multiple of 2 and width should be a multiple of 16.

  1319.  * (If this is a problem for anyone then tell me, and I will fix it.)

  1320.  */

  1321. static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1322.                                       int width, int height,

  1323.                                       int lumStride, int chromStride, int srcStride)

  1324. {

  1325.     int y;

  1326.     const x86_reg chromWidth= width>>1;

  1327.     for (y=0; y<height; y+=2) {

  1328.         __asm__ volatile(

  1329.             "xor              %%"FF_REG_a", %%"FF_REG_a"\n\t"

  1330.             "pcmpeqw                 %%mm7, %%mm7       \n\t"

  1331.             "psrlw                      $8, %%mm7       \n\t" // FF,00,FF,00...

  1332.             ".p2align                    4              \n\t"

  1333.             "1:                \n\t"

  1334.             PREFETCH" 64(%0, %%"FF_REG_a", 4)           \n\t"

  1335.             "movq    (%0, %%"FF_REG_a", 4), %%mm0       \n\t" // YUYV YUYV(0)

  1336.             "movq   8(%0, %%"FF_REG_a", 4), %%mm1       \n\t" // YUYV YUYV(4)

  1337.             "movq                    %%mm0, %%mm2       \n\t" // YUYV YUYV(0)

  1338.             "movq                    %%mm1, %%mm3       \n\t" // YUYV YUYV(4)

  1339.             "psrlw                      $8, %%mm0       \n\t" // U0V0 U0V0(0)

  1340.             "psrlw                      $8, %%mm1       \n\t" // U0V0 U0V0(4)

  1341.             "pand                    %%mm7, %%mm2       \n\t" // Y0Y0 Y0Y0(0)

  1342.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(4)

  1343.             "packuswb                %%mm1, %%mm0       \n\t" // UVUV UVUV(0)

  1344.             "packuswb                %%mm3, %%mm2       \n\t" // YYYY YYYY(0)

  1345. 

  1346.             MOVNTQ"                  %%mm2, (%1, %%"FF_REG_a", 2) \n\t"

  1347. 

  1348.             "movq  16(%0, %%"FF_REG_a", 4), %%mm1       \n\t" // YUYV YUYV(8)

  1349.             "movq  24(%0, %%"FF_REG_a", 4), %%mm2       \n\t" // YUYV YUYV(12)

  1350.             "movq                    %%mm1, %%mm3       \n\t" // YUYV YUYV(8)

  1351.             "movq                    %%mm2, %%mm4       \n\t" // YUYV YUYV(12)

  1352.             "psrlw                      $8, %%mm1       \n\t" // U0V0 U0V0(8)

  1353.             "psrlw                      $8, %%mm2       \n\t" // U0V0 U0V0(12)

  1354.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(8)

  1355.             "pand                    %%mm7, %%mm4       \n\t" // Y0Y0 Y0Y0(12)

  1356.             "packuswb                %%mm2, %%mm1       \n\t" // UVUV UVUV(8)

  1357.             "packuswb                %%mm4, %%mm3       \n\t" // YYYY YYYY(8)

  1358. 

  1359.             MOVNTQ"                  %%mm3, 8(%1, %%"FF_REG_a", 2) \n\t"

  1360. 

  1361.             "movq                    %%mm0, %%mm2       \n\t" // UVUV UVUV(0)

  1362.             "movq                    %%mm1, %%mm3       \n\t" // UVUV UVUV(8)

  1363.             "psrlw                      $8, %%mm0       \n\t" // V0V0 V0V0(0)

  1364.             "psrlw                      $8, %%mm1       \n\t" // V0V0 V0V0(8)

  1365.             "pand                    %%mm7, %%mm2       \n\t" // U0U0 U0U0(0)

  1366.             "pand                    %%mm7, %%mm3       \n\t" // U0U0 U0U0(8)

  1367.             "packuswb                %%mm1, %%mm0       \n\t" // VVVV VVVV(0)

  1368.             "packuswb                %%mm3, %%mm2       \n\t" // UUUU UUUU(0)

  1369. 

  1370.             MOVNTQ"                  %%mm0, (%3, %%"FF_REG_a")     \n\t"

  1371.             MOVNTQ"                  %%mm2, (%2, %%"FF_REG_a")     \n\t"

  1372. 

  1373.             "add                        $8, %%"FF_REG_a" \n\t"

  1374.             "cmp                        %4, %%"FF_REG_a" \n\t"

  1375.             " jb                        1b               \n\t"

  1376.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1377.             : "memory", "%"FF_REG_a

  1378.         );

  1379. 

  1380.         ydst += lumStride;

  1381.         src  += srcStride;

  1382. 

  1383.         __asm__ volatile(

  1384.             "xor              %%"FF_REG_a", %%"FF_REG_a"\n\t"

  1385.             ".p2align                    4              \n\t"

  1386.             "1:                                         \n\t"

  1387.             PREFETCH" 64(%0, %%"FF_REG_a", 4)           \n\t"

  1388.             "movq    (%0, %%"FF_REG_a", 4), %%mm0       \n\t" // YUYV YUYV(0)

  1389.             "movq   8(%0, %%"FF_REG_a", 4), %%mm1       \n\t" // YUYV YUYV(4)

  1390.             "movq  16(%0, %%"FF_REG_a", 4), %%mm2       \n\t" // YUYV YUYV(8)

  1391.             "movq  24(%0, %%"FF_REG_a", 4), %%mm3       \n\t" // YUYV YUYV(12)

  1392.             "pand                    %%mm7, %%mm0       \n\t" // Y0Y0 Y0Y0(0)

  1393.             "pand                    %%mm7, %%mm1       \n\t" // Y0Y0 Y0Y0(4)

  1394.             "pand                    %%mm7, %%mm2       \n\t" // Y0Y0 Y0Y0(8)

  1395.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(12)

  1396.             "packuswb                %%mm1, %%mm0       \n\t" // YYYY YYYY(0)

  1397.             "packuswb                %%mm3, %%mm2       \n\t" // YYYY YYYY(8)

  1398. 

  1399.             MOVNTQ"                  %%mm0,  (%1, %%"FF_REG_a", 2) \n\t"

  1400.             MOVNTQ"                  %%mm2, 8(%1, %%"FF_REG_a", 2) \n\t"

  1401. 

  1402.             "add                        $8, %%"FF_REG_a"\n\t"

  1403.             "cmp                        %4, %%"FF_REG_a"\n\t"

  1404.             " jb                        1b              \n\t"

  1405. 

  1406.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1407.             : "memory", "%"FF_REG_a

  1408.         );

  1409.         udst += chromStride;

  1410.         vdst += chromStride;

  1411.         ydst += lumStride;

  1412.         src  += srcStride;

  1413.     }

  1414.     __asm__ volatile(EMMS"       \n\t"

  1415.                      SFENCE"     \n\t"

  1416.                      :::"memory");

  1417. }

  1418. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  1419. 

  1420. #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW

  1421. static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, int srcWidth, int srcHeight, int srcStride, int dstStride)

  1422. {

  1423.     int x,y;

  1424. 

  1425.     dst[0]= src[0];

  1426. 

  1427.     // first line

  1428.     for (x=0; x<srcWidth-1; x++) {

  1429.         dst[2*x+1]= (3*src[x] +   src[x+1])>>2;

  1430.         dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;

  1431.     }

  1432.     dst[2*srcWidth-1]= src[srcWidth-1];

  1433. 

  1434.     dst+= dstStride;

  1435. 

  1436.     for (y=1; y<srcHeight; y++) {

  1437.         x86_reg mmxSize= srcWidth&~15;

  1438. 

  1439.         if (mmxSize) {

  1440.         __asm__ volatile(

  1441.             "mov                       %4, %%"FF_REG_a" \n\t"

  1442.             "movq        "MANGLE(mmx_ff)", %%mm0    \n\t"

  1443.             "movq      (%0, %%"FF_REG_a"), %%mm4    \n\t"

  1444.             "movq                   %%mm4, %%mm2    \n\t"

  1445.             "psllq                     $8, %%mm4    \n\t"

  1446.             "pand                   %%mm0, %%mm2    \n\t"

  1447.             "por                    %%mm2, %%mm4    \n\t"

  1448.             "movq      (%1, %%"FF_REG_a"), %%mm5    \n\t"

  1449.             "movq                   %%mm5, %%mm3    \n\t"

  1450.             "psllq                     $8, %%mm5    \n\t"

  1451.             "pand                   %%mm0, %%mm3    \n\t"

  1452.             "por                    %%mm3, %%mm5    \n\t"

  1453.             "1:                                     \n\t"

  1454.             "movq      (%0, %%"FF_REG_a"), %%mm0    \n\t"

  1455.             "movq      (%1, %%"FF_REG_a"), %%mm1    \n\t"

  1456.             "movq     1(%0, %%"FF_REG_a"), %%mm2    \n\t"

  1457.             "movq     1(%1, %%"FF_REG_a"), %%mm3    \n\t"

  1458.             PAVGB"                  %%mm0, %%mm5    \n\t"

  1459.             PAVGB"                  %%mm0, %%mm3    \n\t"

  1460.             PAVGB"                  %%mm0, %%mm5    \n\t"

  1461.             PAVGB"                  %%mm0, %%mm3    \n\t"

  1462.             PAVGB"                  %%mm1, %%mm4    \n\t"

  1463.             PAVGB"                  %%mm1, %%mm2    \n\t"

  1464.             PAVGB"                  %%mm1, %%mm4    \n\t"

  1465.             PAVGB"                  %%mm1, %%mm2    \n\t"

  1466.             "movq                   %%mm5, %%mm7    \n\t"

  1467.             "movq                   %%mm4, %%mm6    \n\t"

  1468.             "punpcklbw              %%mm3, %%mm5    \n\t"

  1469.             "punpckhbw              %%mm3, %%mm7    \n\t"

  1470.             "punpcklbw              %%mm2, %%mm4    \n\t"

  1471.             "punpckhbw              %%mm2, %%mm6    \n\t"

  1472.             MOVNTQ"                 %%mm5,  (%2, %%"FF_REG_a", 2)  \n\t"

  1473.             MOVNTQ"                 %%mm7, 8(%2, %%"FF_REG_a", 2)  \n\t"

  1474.             MOVNTQ"                 %%mm4,  (%3, %%"FF_REG_a", 2)  \n\t"

  1475.             MOVNTQ"                 %%mm6, 8(%3, %%"FF_REG_a", 2)  \n\t"

  1476.             "add                       $8, %%"FF_REG_a"            \n\t"

  1477.             "movq    -1(%0, %%"FF_REG_a"), %%mm4    \n\t"

  1478.             "movq    -1(%1, %%"FF_REG_a"), %%mm5    \n\t"

  1479.             " js                       1b           \n\t"

  1480.             :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),

  1481.                "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),

  1482.                "g" (-mmxSize)

  1483.                NAMED_CONSTRAINTS_ADD(mmx_ff)

  1484.             : "%"FF_REG_a

  1485.         );

  1486.         } else {

  1487.             mmxSize = 1;

  1488.             dst[0]         = (src[0] * 3 + src[srcStride]) >> 2;

  1489.             dst[dstStride] = (src[0] + 3 * src[srcStride]) >> 2;

  1490.         }

  1491. 

  1492.         for (x=mmxSize-1; x<srcWidth-1; x++) {

  1493.             dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;

  1494.             dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;

  1495.             dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;

  1496.             dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;

  1497.         }

  1498.         dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;

  1499.         dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;

  1500. 

  1501.         dst+=dstStride*2;

  1502.         src+=srcStride;

  1503.     }

  1504. 

  1505.     // last line

  1506.     dst[0]= src[0];

  1507. 

  1508.     for (x=0; x<srcWidth-1; x++) {

  1509.         dst[2*x+1]= (3*src[x] +   src[x+1])>>2;

  1510.         dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;

  1511.     }

  1512.     dst[2*srcWidth-1]= src[srcWidth-1];

  1513. 

  1514.     __asm__ volatile(EMMS"       \n\t"

  1515.                      SFENCE"     \n\t"

  1516.                      :::"memory");

  1517. }

  1518. #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */

  1519. 

  1520. #if !COMPILE_TEMPLATE_AMD3DNOW

  1521. /**

  1522.  * Height should be a multiple of 2 and width should be a multiple of 16.

  1523.  * (If this is a problem for anyone then tell me, and I will fix it.)

  1524.  * Chrominance data is only taken from every second line, others are ignored.

  1525.  * FIXME: Write HQ version.

  1526.  */

  1527. static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1528.                                       int width, int height,

  1529.                                       int lumStride, int chromStride, int srcStride)

  1530. {

  1531.     int y;

  1532.     const x86_reg chromWidth= width>>1;

  1533.     for (y=0; y<height; y+=2) {

  1534.         __asm__ volatile(

  1535.             "xor          %%"FF_REG_a", %%"FF_REG_a" \n\t"

  1536.             "pcmpeqw             %%mm7, %%mm7   \n\t"

  1537.             "psrlw                  $8, %%mm7   \n\t" // FF,00,FF,00...

  1538.             ".p2align                4          \n\t"

  1539.             "1:                                 \n\t"

  1540.             PREFETCH" 64(%0, %%"FF_REG_a", 4)          \n\t"

  1541.             "movq       (%0, %%"FF_REG_a", 4), %%mm0   \n\t" // UYVY UYVY(0)

  1542.             "movq      8(%0, %%"FF_REG_a", 4), %%mm1   \n\t" // UYVY UYVY(4)

  1543.             "movq                %%mm0, %%mm2   \n\t" // UYVY UYVY(0)

  1544.             "movq                %%mm1, %%mm3   \n\t" // UYVY UYVY(4)

  1545.             "pand                %%mm7, %%mm0   \n\t" // U0V0 U0V0(0)

  1546.             "pand                %%mm7, %%mm1   \n\t" // U0V0 U0V0(4)

  1547.             "psrlw                  $8, %%mm2   \n\t" // Y0Y0 Y0Y0(0)

  1548.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(4)

  1549.             "packuswb            %%mm1, %%mm0   \n\t" // UVUV UVUV(0)

  1550.             "packuswb            %%mm3, %%mm2   \n\t" // YYYY YYYY(0)

  1551. 

  1552.             MOVNTQ"              %%mm2,  (%1, %%"FF_REG_a", 2) \n\t"

  1553. 

  1554.             "movq     16(%0, %%"FF_REG_a", 4), %%mm1   \n\t" // UYVY UYVY(8)

  1555.             "movq     24(%0, %%"FF_REG_a", 4), %%mm2   \n\t" // UYVY UYVY(12)

  1556.             "movq                %%mm1, %%mm3   \n\t" // UYVY UYVY(8)

  1557.             "movq                %%mm2, %%mm4   \n\t" // UYVY UYVY(12)

  1558.             "pand                %%mm7, %%mm1   \n\t" // U0V0 U0V0(8)

  1559.             "pand                %%mm7, %%mm2   \n\t" // U0V0 U0V0(12)

  1560.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(8)

  1561.             "psrlw                  $8, %%mm4   \n\t" // Y0Y0 Y0Y0(12)

  1562.             "packuswb            %%mm2, %%mm1   \n\t" // UVUV UVUV(8)

  1563.             "packuswb            %%mm4, %%mm3   \n\t" // YYYY YYYY(8)

  1564. 

  1565.             MOVNTQ"              %%mm3, 8(%1, %%"FF_REG_a", 2) \n\t"

  1566. 

  1567.             "movq                %%mm0, %%mm2   \n\t" // UVUV UVUV(0)

  1568.             "movq                %%mm1, %%mm3   \n\t" // UVUV UVUV(8)

  1569.             "psrlw                  $8, %%mm0   \n\t" // V0V0 V0V0(0)

  1570.             "psrlw                  $8, %%mm1   \n\t" // V0V0 V0V0(8)

  1571.             "pand                %%mm7, %%mm2   \n\t" // U0U0 U0U0(0)

  1572.             "pand                %%mm7, %%mm3   \n\t" // U0U0 U0U0(8)

  1573.             "packuswb            %%mm1, %%mm0   \n\t" // VVVV VVVV(0)

  1574.             "packuswb            %%mm3, %%mm2   \n\t" // UUUU UUUU(0)

  1575. 

  1576.             MOVNTQ"              %%mm0, (%3, %%"FF_REG_a") \n\t"

  1577.             MOVNTQ"              %%mm2, (%2, %%"FF_REG_a") \n\t"

  1578. 

  1579.             "add                    $8, %%"FF_REG_a" \n\t"

  1580.             "cmp                    %4, %%"FF_REG_a" \n\t"

  1581.             " jb                    1b               \n\t"

  1582.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1583.             : "memory", "%"FF_REG_a

  1584.         );

  1585. 

  1586.         ydst += lumStride;

  1587.         src  += srcStride;

  1588. 

  1589.         __asm__ volatile(

  1590.             "xor          %%"FF_REG_a", %%"FF_REG_a"  \n\t"

  1591.             ".p2align                4                \n\t"

  1592.             "1:                                       \n\t"

  1593.             PREFETCH" 64(%0, %%"FF_REG_a", 4)         \n\t"

  1594.             "movq       (%0, %%"FF_REG_a", 4), %%mm0  \n\t" // YUYV YUYV(0)

  1595.             "movq      8(%0, %%"FF_REG_a", 4), %%mm1  \n\t" // YUYV YUYV(4)

  1596.             "movq     16(%0, %%"FF_REG_a", 4), %%mm2  \n\t" // YUYV YUYV(8)

  1597.             "movq     24(%0, %%"FF_REG_a", 4), %%mm3  \n\t" // YUYV YUYV(12)

  1598.             "psrlw                  $8, %%mm0   \n\t" // Y0Y0 Y0Y0(0)

  1599.             "psrlw                  $8, %%mm1   \n\t" // Y0Y0 Y0Y0(4)

  1600.             "psrlw                  $8, %%mm2   \n\t" // Y0Y0 Y0Y0(8)

  1601.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(12)

  1602.             "packuswb            %%mm1, %%mm0   \n\t" // YYYY YYYY(0)

  1603.             "packuswb            %%mm3, %%mm2   \n\t" // YYYY YYYY(8)

  1604. 

  1605.             MOVNTQ"              %%mm0,  (%1, %%"FF_REG_a", 2) \n\t"

  1606.             MOVNTQ"              %%mm2, 8(%1, %%"FF_REG_a", 2) \n\t"

  1607. 

  1608.             "add                    $8, %%"FF_REG_a" \n\t"

  1609.             "cmp                    %4, %%"FF_REG_a" \n\t"

  1610.             " jb                    1b               \n\t"

  1611. 

  1612.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1613.             : "memory", "%"FF_REG_a

  1614.         );

  1615.         udst += chromStride;

  1616.         vdst += chromStride;

  1617.         ydst += lumStride;

  1618.         src  += srcStride;

  1619.     }

  1620.     __asm__ volatile(EMMS"       \n\t"

  1621.                      SFENCE"     \n\t"

  1622.                      :::"memory");

  1623. }

  1624. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  1625. 

  1626. /**

  1627.  * Height should be a multiple of 2 and width should be a multiple of 2.

  1628.  * (If this is a problem for anyone then tell me, and I will fix it.)

  1629.  * Chrominance data is only taken from every second line,

  1630.  * others are ignored in the C version.

  1631.  * FIXME: Write HQ version.

  1632.  */

  1633. #if HAVE_7REGS

  1634. static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1635.                                        int width, int height,

  1636.                                        int lumStride, int chromStride, int srcStride,

  1637.                                        int32_t *rgb2yuv)

  1638. {

  1639. #define BGR2Y_IDX "16*4+16*32"

  1640. #define BGR2U_IDX "16*4+16*33"

  1641. #define BGR2V_IDX "16*4+16*34"

  1642.     int y;

  1643.     const x86_reg chromWidth= width>>1;

  1644. 

  1645.     if (height > 2) {

  1646.         ff_rgb24toyv12_c(src, ydst, udst, vdst, width, 2, lumStride, chromStride, srcStride, rgb2yuv);

  1647.         src  += 2*srcStride;

  1648.         ydst += 2*lumStride;

  1649.         udst += chromStride;

  1650.         vdst += chromStride;

  1651.         height -= 2;

  1652.     }

  1653. 

  1654.     for (y=0; y<height-2; y+=2) {

  1655.         int i;

  1656.         for (i=0; i<2; i++) {

  1657.             __asm__ volatile(

  1658.                 "mov                        %2, %%"FF_REG_a"\n\t"

  1659.                 "movq          "BGR2Y_IDX"(%3), %%mm6       \n\t"

  1660.                 "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"

  1661.                 "pxor                    %%mm7, %%mm7       \n\t"

  1662.                 "lea (%%"FF_REG_a", %%"FF_REG_a", 2), %%"FF_REG_d" \n\t"

  1663.                 ".p2align                    4              \n\t"

  1664.                 "1:                                         \n\t"

  1665.                 PREFETCH" 64(%0, %%"FF_REG_d")              \n\t"

  1666.                 "movd       (%0, %%"FF_REG_d"), %%mm0       \n\t"

  1667.                 "movd      3(%0, %%"FF_REG_d"), %%mm1       \n\t"

  1668.                 "punpcklbw               %%mm7, %%mm0       \n\t"

  1669.                 "punpcklbw               %%mm7, %%mm1       \n\t"

  1670.                 "movd      6(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1671.                 "movd      9(%0, %%"FF_REG_d"), %%mm3       \n\t"

  1672.                 "punpcklbw               %%mm7, %%mm2       \n\t"

  1673.                 "punpcklbw               %%mm7, %%mm3       \n\t"

  1674.                 "pmaddwd                 %%mm6, %%mm0       \n\t"

  1675.                 "pmaddwd                 %%mm6, %%mm1       \n\t"

  1676.                 "pmaddwd                 %%mm6, %%mm2       \n\t"

  1677.                 "pmaddwd                 %%mm6, %%mm3       \n\t"

  1678.                 "psrad                      $8, %%mm0       \n\t"

  1679.                 "psrad                      $8, %%mm1       \n\t"

  1680.                 "psrad                      $8, %%mm2       \n\t"

  1681.                 "psrad                      $8, %%mm3       \n\t"

  1682.                 "packssdw                %%mm1, %%mm0       \n\t"

  1683.                 "packssdw                %%mm3, %%mm2       \n\t"

  1684.                 "pmaddwd                 %%mm5, %%mm0       \n\t"

  1685.                 "pmaddwd                 %%mm5, %%mm2       \n\t"

  1686.                 "packssdw                %%mm2, %%mm0       \n\t"

  1687.                 "psraw                      $7, %%mm0       \n\t"

  1688. 

  1689.                 "movd     12(%0, %%"FF_REG_d"), %%mm4       \n\t"

  1690.                 "movd     15(%0, %%"FF_REG_d"), %%mm1       \n\t"

  1691.                 "punpcklbw               %%mm7, %%mm4       \n\t"

  1692.                 "punpcklbw               %%mm7, %%mm1       \n\t"

  1693.                 "movd     18(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1694.                 "movd     21(%0, %%"FF_REG_d"), %%mm3       \n\t"

  1695.                 "punpcklbw               %%mm7, %%mm2       \n\t"

  1696.                 "punpcklbw               %%mm7, %%mm3       \n\t"

  1697.                 "pmaddwd                 %%mm6, %%mm4       \n\t"

  1698.                 "pmaddwd                 %%mm6, %%mm1       \n\t"

  1699.                 "pmaddwd                 %%mm6, %%mm2       \n\t"

  1700.                 "pmaddwd                 %%mm6, %%mm3       \n\t"

  1701.                 "psrad                      $8, %%mm4       \n\t"

  1702.                 "psrad                      $8, %%mm1       \n\t"

  1703.                 "psrad                      $8, %%mm2       \n\t"

  1704.                 "psrad                      $8, %%mm3       \n\t"

  1705.                 "packssdw                %%mm1, %%mm4       \n\t"

  1706.                 "packssdw                %%mm3, %%mm2       \n\t"

  1707.                 "pmaddwd                 %%mm5, %%mm4       \n\t"

  1708.                 "pmaddwd                 %%mm5, %%mm2       \n\t"

  1709.                 "add                       $24, %%"FF_REG_d"\n\t"

  1710.                 "packssdw                %%mm2, %%mm4       \n\t"

  1711.                 "psraw                      $7, %%mm4       \n\t"

  1712. 

  1713.                 "packuswb                %%mm4, %%mm0       \n\t"

  1714.                 "paddusb "MANGLE(ff_bgr2YOffset)", %%mm0    \n\t"

  1715. 

  1716.                 MOVNTQ"                  %%mm0, (%1, %%"FF_REG_a") \n\t"

  1717.                 "add                        $8,      %%"FF_REG_a"  \n\t"

  1718.                 " js                        1b                     \n\t"

  1719.                 : : "r" (src+width*3), "r" (ydst+width), "g" ((x86_reg)-width), "r"(rgb2yuv)

  1720.                   NAMED_CONSTRAINTS_ADD(ff_w1111,ff_bgr2YOffset)

  1721.                 : "%"FF_REG_a, "%"FF_REG_d

  1722.             );

  1723.             ydst += lumStride;

  1724.             src  += srcStride;

  1725.         }

  1726.         src -= srcStride*2;

  1727.         __asm__ volatile(

  1728.             "mov                        %4, %%"FF_REG_a"\n\t"

  1729.             "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"

  1730.             "movq          "BGR2U_IDX"(%5), %%mm6       \n\t"

  1731.             "pxor                    %%mm7, %%mm7       \n\t"

  1732.             "lea (%%"FF_REG_a", %%"FF_REG_a", 2), %%"FF_REG_d" \n\t"

  1733.             "add              %%"FF_REG_d", %%"FF_REG_d"\n\t"

  1734.             ".p2align                    4              \n\t"

  1735.             "1:                                         \n\t"

  1736.             PREFETCH" 64(%0, %%"FF_REG_d")              \n\t"

  1737.             PREFETCH" 64(%1, %%"FF_REG_d")              \n\t"

  1738. #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW

  1739.             "movq       (%0, %%"FF_REG_d"), %%mm0       \n\t"

  1740.             "movq       (%1, %%"FF_REG_d"), %%mm1       \n\t"

  1741.             "movq      6(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1742.             "movq      6(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1743.             PAVGB"                   %%mm1, %%mm0       \n\t"

  1744.             PAVGB"                   %%mm3, %%mm2       \n\t"

  1745.             "movq                    %%mm0, %%mm1       \n\t"

  1746.             "movq                    %%mm2, %%mm3       \n\t"

  1747.             "psrlq                     $24, %%mm0       \n\t"

  1748.             "psrlq                     $24, %%mm2       \n\t"

  1749.             PAVGB"                   %%mm1, %%mm0       \n\t"

  1750.             PAVGB"                   %%mm3, %%mm2       \n\t"

  1751.             "punpcklbw               %%mm7, %%mm0       \n\t"

  1752.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1753. #else

  1754.             "movd       (%0, %%"FF_REG_d"), %%mm0       \n\t"

  1755.             "movd       (%1, %%"FF_REG_d"), %%mm1       \n\t"

  1756.             "movd      3(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1757.             "movd      3(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1758.             "punpcklbw               %%mm7, %%mm0       \n\t"

  1759.             "punpcklbw               %%mm7, %%mm1       \n\t"

  1760.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1761.             "punpcklbw               %%mm7, %%mm3       \n\t"

  1762.             "paddw                   %%mm1, %%mm0       \n\t"

  1763.             "paddw                   %%mm3, %%mm2       \n\t"

  1764.             "paddw                   %%mm2, %%mm0       \n\t"

  1765.             "movd      6(%0, %%"FF_REG_d"), %%mm4       \n\t"

  1766.             "movd      6(%1, %%"FF_REG_d"), %%mm1       \n\t"

  1767.             "movd      9(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1768.             "movd      9(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1769.             "punpcklbw               %%mm7, %%mm4       \n\t"

  1770.             "punpcklbw               %%mm7, %%mm1       \n\t"

  1771.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1772.             "punpcklbw               %%mm7, %%mm3       \n\t"

  1773.             "paddw                   %%mm1, %%mm4       \n\t"

  1774.             "paddw                   %%mm3, %%mm2       \n\t"

  1775.             "paddw                   %%mm4, %%mm2       \n\t"

  1776.             "psrlw                      $2, %%mm0       \n\t"

  1777.             "psrlw                      $2, %%mm2       \n\t"

  1778. #endif

  1779.             "movq          "BGR2V_IDX"(%5), %%mm1       \n\t"

  1780.             "movq          "BGR2V_IDX"(%5), %%mm3       \n\t"

  1781. 

  1782.             "pmaddwd                 %%mm0, %%mm1       \n\t"

  1783.             "pmaddwd                 %%mm2, %%mm3       \n\t"

  1784.             "pmaddwd                 %%mm6, %%mm0       \n\t"

  1785.             "pmaddwd                 %%mm6, %%mm2       \n\t"

  1786.             "psrad                      $8, %%mm0       \n\t"

  1787.             "psrad                      $8, %%mm1       \n\t"

  1788.             "psrad                      $8, %%mm2       \n\t"

  1789.             "psrad                      $8, %%mm3       \n\t"

  1790.             "packssdw                %%mm2, %%mm0       \n\t"

  1791.             "packssdw                %%mm3, %%mm1       \n\t"

  1792.             "pmaddwd                 %%mm5, %%mm0       \n\t"

  1793.             "pmaddwd                 %%mm5, %%mm1       \n\t"

  1794.             "packssdw                %%mm1, %%mm0       \n\t" // V1 V0 U1 U0

  1795.             "psraw                      $7, %%mm0       \n\t"

  1796. 

  1797. #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW

  1798.             "movq     12(%0, %%"FF_REG_d"), %%mm4       \n\t"

  1799.             "movq     12(%1, %%"FF_REG_d"), %%mm1       \n\t"

  1800.             "movq     18(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1801.             "movq     18(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1802.             PAVGB"                   %%mm1, %%mm4       \n\t"

  1803.             PAVGB"                   %%mm3, %%mm2       \n\t"

  1804.             "movq                    %%mm4, %%mm1       \n\t"

  1805.             "movq                    %%mm2, %%mm3       \n\t"

  1806.             "psrlq                     $24, %%mm4       \n\t"

  1807.             "psrlq                     $24, %%mm2       \n\t"

  1808.             PAVGB"                   %%mm1, %%mm4       \n\t"

  1809.             PAVGB"                   %%mm3, %%mm2       \n\t"

  1810.             "punpcklbw               %%mm7, %%mm4       \n\t"

  1811.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1812. #else

  1813.             "movd     12(%0, %%"FF_REG_d"), %%mm4       \n\t"

  1814.             "movd     12(%1, %%"FF_REG_d"), %%mm1       \n\t"

  1815.             "movd     15(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1816.             "movd     15(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1817.             "punpcklbw               %%mm7, %%mm4       \n\t"

  1818.             "punpcklbw               %%mm7, %%mm1       \n\t"

  1819.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1820.             "punpcklbw               %%mm7, %%mm3       \n\t"

  1821.             "paddw                   %%mm1, %%mm4       \n\t"

  1822.             "paddw                   %%mm3, %%mm2       \n\t"

  1823.             "paddw                   %%mm2, %%mm4       \n\t"

  1824.             "movd     18(%0, %%"FF_REG_d"), %%mm5       \n\t"

  1825.             "movd     18(%1, %%"FF_REG_d"), %%mm1       \n\t"

  1826.             "movd     21(%0, %%"FF_REG_d"), %%mm2       \n\t"

  1827.             "movd     21(%1, %%"FF_REG_d"), %%mm3       \n\t"

  1828.             "punpcklbw               %%mm7, %%mm5       \n\t"

  1829.             "punpcklbw               %%mm7, %%mm1       \n\t"

  1830.             "punpcklbw               %%mm7, %%mm2       \n\t"

  1831.             "punpcklbw               %%mm7, %%mm3       \n\t"

  1832.             "paddw                   %%mm1, %%mm5       \n\t"

  1833.             "paddw                   %%mm3, %%mm2       \n\t"

  1834.             "paddw                   %%mm5, %%mm2       \n\t"

  1835.             "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"

  1836.             "psrlw                      $2, %%mm4       \n\t"

  1837.             "psrlw                      $2, %%mm2       \n\t"

  1838. #endif

  1839.             "movq          "BGR2V_IDX"(%5), %%mm1       \n\t"

  1840.             "movq          "BGR2V_IDX"(%5), %%mm3       \n\t"

  1841. 

  1842.             "pmaddwd                 %%mm4, %%mm1       \n\t"

  1843.             "pmaddwd                 %%mm2, %%mm3       \n\t"

  1844.             "pmaddwd                 %%mm6, %%mm4       \n\t"

  1845.             "pmaddwd                 %%mm6, %%mm2       \n\t"

  1846.             "psrad                      $8, %%mm4       \n\t"

  1847.             "psrad                      $8, %%mm1       \n\t"

  1848.             "psrad                      $8, %%mm2       \n\t"

  1849.             "psrad                      $8, %%mm3       \n\t"

  1850.             "packssdw                %%mm2, %%mm4       \n\t"

  1851.             "packssdw                %%mm3, %%mm1       \n\t"

  1852.             "pmaddwd                 %%mm5, %%mm4       \n\t"

  1853.             "pmaddwd                 %%mm5, %%mm1       \n\t"

  1854.             "add                       $24, %%"FF_REG_d"\n\t"

  1855.             "packssdw                %%mm1, %%mm4       \n\t" // V3 V2 U3 U2

  1856.             "psraw                      $7, %%mm4       \n\t"

  1857. 

  1858.             "movq                    %%mm0, %%mm1           \n\t"

  1859.             "punpckldq               %%mm4, %%mm0           \n\t"

  1860.             "punpckhdq               %%mm4, %%mm1           \n\t"

  1861.             "packsswb                %%mm1, %%mm0           \n\t"

  1862.             "paddb "MANGLE(ff_bgr2UVOffset)", %%mm0         \n\t"

  1863.             "movd                    %%mm0, (%2, %%"FF_REG_a") \n\t"

  1864.             "punpckhdq               %%mm0, %%mm0              \n\t"

  1865.             "movd                    %%mm0, (%3, %%"FF_REG_a") \n\t"

  1866.             "add                        $4, %%"FF_REG_a"       \n\t"

  1867.             " js                        1b              \n\t"

  1868.             : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth), "r"(rgb2yuv)

  1869.               NAMED_CONSTRAINTS_ADD(ff_w1111,ff_bgr2UVOffset)

  1870.             : "%"FF_REG_a, "%"FF_REG_d

  1871.         );

  1872. 

  1873.         udst += chromStride;

  1874.         vdst += chromStride;

  1875.         src  += srcStride*2;

  1876.     }

  1877. 

  1878.     __asm__ volatile(EMMS"       \n\t"

  1879.                      SFENCE"     \n\t"

  1880.                      :::"memory");

  1881. 

  1882.      ff_rgb24toyv12_c(src, ydst, udst, vdst, width, height-y, lumStride, chromStride, srcStride, rgb2yuv);

  1883. }

  1884. #endif /* HAVE_7REGS */

  1885. #endif /* !COMPILE_TEMPLATE_SSE2 */

  1886. 

  1887. #if !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX

  1888. static void RENAME(interleaveBytes)(const uint8_t *src1, const uint8_t *src2, uint8_t *dest,

  1889.                                     int width, int height, int src1Stride,

  1890.                                     int src2Stride, int dstStride)

  1891. {

  1892.     int h;

  1893. 

  1894.     for (h=0; h < height; h++) {

  1895.         int w;

  1896. 

  1897.         if (width >= 16) {

  1898. #if COMPILE_TEMPLATE_SSE2

  1899.             if (!((((intptr_t)src1) | ((intptr_t)src2) | ((intptr_t)dest))&15)) {

  1900.         __asm__(

  1901.             "xor              %%"FF_REG_a", %%"FF_REG_a"  \n\t"

  1902.             "1:                                     \n\t"

  1903.             PREFETCH" 64(%1, %%"FF_REG_a")          \n\t"

  1904.             PREFETCH" 64(%2, %%"FF_REG_a")          \n\t"

  1905.             "movdqa  (%1, %%"FF_REG_a"), %%xmm0     \n\t"

  1906.             "movdqa  (%1, %%"FF_REG_a"), %%xmm1     \n\t"

  1907.             "movdqa  (%2, %%"FF_REG_a"), %%xmm2     \n\t"

  1908.             "punpcklbw           %%xmm2, %%xmm0     \n\t"

  1909.             "punpckhbw           %%xmm2, %%xmm1     \n\t"

  1910.             "movntdq             %%xmm0,   (%0, %%"FF_REG_a", 2) \n\t"

  1911.             "movntdq             %%xmm1, 16(%0, %%"FF_REG_a", 2) \n\t"

  1912.             "add                    $16, %%"FF_REG_a"            \n\t"

  1913.             "cmp                     %3, %%"FF_REG_a"            \n\t"

  1914.             " jb                     1b             \n\t"

  1915.             ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)

  1916.             : "memory", XMM_CLOBBERS("xmm0", "xmm1", "xmm2",) "%"FF_REG_a

  1917.         );

  1918.             } else

  1919. #endif

  1920.         __asm__(

  1921.             "xor %%"FF_REG_a", %%"FF_REG_a"         \n\t"

  1922.             "1:                                     \n\t"

  1923.             PREFETCH" 64(%1, %%"FF_REG_a")          \n\t"

  1924.             PREFETCH" 64(%2, %%"FF_REG_a")          \n\t"

  1925.             "movq    (%1, %%"FF_REG_a"), %%mm0      \n\t"

  1926.             "movq   8(%1, %%"FF_REG_a"), %%mm2      \n\t"

  1927.             "movq                 %%mm0, %%mm1      \n\t"

  1928.             "movq                 %%mm2, %%mm3      \n\t"

  1929.             "movq    (%2, %%"FF_REG_a"), %%mm4      \n\t"

  1930.             "movq   8(%2, %%"FF_REG_a"), %%mm5      \n\t"

  1931.             "punpcklbw            %%mm4, %%mm0      \n\t"

  1932.             "punpckhbw            %%mm4, %%mm1      \n\t"

  1933.             "punpcklbw            %%mm5, %%mm2      \n\t"

  1934.             "punpckhbw            %%mm5, %%mm3      \n\t"

  1935.             MOVNTQ"               %%mm0,   (%0, %%"FF_REG_a", 2) \n\t"

  1936.             MOVNTQ"               %%mm1,  8(%0, %%"FF_REG_a", 2) \n\t"

  1937.             MOVNTQ"               %%mm2, 16(%0, %%"FF_REG_a", 2) \n\t"

  1938.             MOVNTQ"               %%mm3, 24(%0, %%"FF_REG_a", 2) \n\t"

  1939.             "add                    $16, %%"FF_REG_a"            \n\t"

  1940.             "cmp                     %3, %%"FF_REG_a"            \n\t"

  1941.             " jb                     1b                          \n\t"

  1942.             ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)

  1943.             : "memory", "%"FF_REG_a

  1944.         );

  1945. 

  1946.         }

  1947.         for (w= (width&(~15)); w < width; w++) {

  1948.             dest[2*w+0] = src1[w];

  1949.             dest[2*w+1] = src2[w];

  1950.         }

  1951.         dest += dstStride;

  1952.         src1 += src1Stride;

  1953.         src2 += src2Stride;

  1954.     }

  1955.     __asm__(

  1956.             EMMS"       \n\t"

  1957.             SFENCE"     \n\t"

  1958.             ::: "memory"

  1959.             );

  1960. }

  1961. #endif /* !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX */

  1962. 

  1963. #if !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL

  1964. #if !COMPILE_TEMPLATE_AMD3DNOW && (ARCH_X86_32 || COMPILE_TEMPLATE_SSE2) && COMPILE_TEMPLATE_MMXEXT == COMPILE_TEMPLATE_SSE2 && HAVE_X86ASM

  1965. void RENAME(ff_nv12ToUV)(uint8_t *dstU, uint8_t *dstV,

  1966.                          const uint8_t *unused,

  1967.                          const uint8_t *src1,

  1968.                          const uint8_t *src2,

  1969.                          int w,

  1970.                          uint32_t *unused2);

  1971. static void RENAME(deinterleaveBytes)(const uint8_t *src, uint8_t *dst1, uint8_t *dst2,

  1972.                                       int width, int height, int srcStride,

  1973.                                       int dst1Stride, int dst2Stride)

  1974. {

  1975.     int h;

  1976. 

  1977.     for (h = 0; h < height; h++) {

  1978.         RENAME(ff_nv12ToUV)(dst1, dst2, NULL, src, NULL, width, NULL);

  1979.         src  += srcStride;

  1980.         dst1 += dst1Stride;

  1981.         dst2 += dst2Stride;

  1982.     }

  1983.     __asm__(

  1984. #if !COMPILE_TEMPLATE_SSE2

  1985.             EMMS"       \n\t"

  1986. #endif

  1987.             SFENCE"     \n\t"

  1988.             ::: "memory"

  1989.             );

  1990. }

  1991. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  1992. #endif /* !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL */

  1993. 

  1994. #if !COMPILE_TEMPLATE_SSE2

  1995. #if !COMPILE_TEMPLATE_AMD3DNOW

  1996. static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,

  1997.                                        uint8_t *dst1, uint8_t *dst2,

  1998.                                        int width, int height,

  1999.                                        int srcStride1, int srcStride2,

  2000.                                        int dstStride1, int dstStride2)

  2001. {

  2002.     x86_reg x, y;

  2003.     int w,h;

  2004.     w=width/2; h=height/2;

  2005.     __asm__ volatile(

  2006.         PREFETCH" %0    \n\t"

  2007.         PREFETCH" %1    \n\t"

  2008.         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");

  2009.     for (y=0;y<h;y++) {

  2010.         const uint8_t* s1=src1+srcStride1*(y>>1);

  2011.         uint8_t* d=dst1+dstStride1*y;

  2012.         x=0;

  2013.         for (;x<w-31;x+=32) {

  2014.             __asm__ volatile(

  2015.                 PREFETCH"   32(%1,%2)        \n\t"

  2016.                 "movq         (%1,%2), %%mm0 \n\t"

  2017.                 "movq        8(%1,%2), %%mm2 \n\t"

  2018.                 "movq       16(%1,%2), %%mm4 \n\t"

  2019.                 "movq       24(%1,%2), %%mm6 \n\t"

  2020.                 "movq      %%mm0, %%mm1 \n\t"

  2021.                 "movq      %%mm2, %%mm3 \n\t"

  2022.                 "movq      %%mm4, %%mm5 \n\t"

  2023.                 "movq      %%mm6, %%mm7 \n\t"

  2024.                 "punpcklbw %%mm0, %%mm0 \n\t"

  2025.                 "punpckhbw %%mm1, %%mm1 \n\t"

  2026.                 "punpcklbw %%mm2, %%mm2 \n\t"

  2027.                 "punpckhbw %%mm3, %%mm3 \n\t"

  2028.                 "punpcklbw %%mm4, %%mm4 \n\t"

  2029.                 "punpckhbw %%mm5, %%mm5 \n\t"

  2030.                 "punpcklbw %%mm6, %%mm6 \n\t"

  2031.                 "punpckhbw %%mm7, %%mm7 \n\t"

  2032.                 MOVNTQ"    %%mm0,   (%0,%2,2)  \n\t"

  2033.                 MOVNTQ"    %%mm1,  8(%0,%2,2)  \n\t"

  2034.                 MOVNTQ"    %%mm2, 16(%0,%2,2)  \n\t"

  2035.                 MOVNTQ"    %%mm3, 24(%0,%2,2)  \n\t"

  2036.                 MOVNTQ"    %%mm4, 32(%0,%2,2)  \n\t"

  2037.                 MOVNTQ"    %%mm5, 40(%0,%2,2)  \n\t"

  2038.                 MOVNTQ"    %%mm6, 48(%0,%2,2)  \n\t"

  2039.                 MOVNTQ"    %%mm7, 56(%0,%2,2)"

  2040.                 :: "r"(d), "r"(s1), "r"(x)

  2041.                 :"memory");

  2042.         }

  2043.         for (;x<w;x++) d[2*x]=d[2*x+1]=s1[x];

  2044.     }

  2045.     for (y=0;y<h;y++) {

  2046.         const uint8_t* s2=src2+srcStride2*(y>>1);

  2047.         uint8_t* d=dst2+dstStride2*y;

  2048.         x=0;

  2049.         for (;x<w-31;x+=32) {

  2050.             __asm__ volatile(

  2051.                 PREFETCH"   32(%1,%2)        \n\t"

  2052.                 "movq         (%1,%2), %%mm0 \n\t"

  2053.                 "movq        8(%1,%2), %%mm2 \n\t"

  2054.                 "movq       16(%1,%2), %%mm4 \n\t"

  2055.                 "movq       24(%1,%2), %%mm6 \n\t"

  2056.                 "movq      %%mm0, %%mm1 \n\t"

  2057.                 "movq      %%mm2, %%mm3 \n\t"

  2058.                 "movq      %%mm4, %%mm5 \n\t"

  2059.                 "movq      %%mm6, %%mm7 \n\t"

  2060.                 "punpcklbw %%mm0, %%mm0 \n\t"

  2061.                 "punpckhbw %%mm1, %%mm1 \n\t"

  2062.                 "punpcklbw %%mm2, %%mm2 \n\t"

  2063.                 "punpckhbw %%mm3, %%mm3 \n\t"

  2064.                 "punpcklbw %%mm4, %%mm4 \n\t"

  2065.                 "punpckhbw %%mm5, %%mm5 \n\t"

  2066.                 "punpcklbw %%mm6, %%mm6 \n\t"

  2067.                 "punpckhbw %%mm7, %%mm7 \n\t"

  2068.                 MOVNTQ"    %%mm0,   (%0,%2,2)  \n\t"

  2069.                 MOVNTQ"    %%mm1,  8(%0,%2,2)  \n\t"

  2070.                 MOVNTQ"    %%mm2, 16(%0,%2,2)  \n\t"

  2071.                 MOVNTQ"    %%mm3, 24(%0,%2,2)  \n\t"

  2072.                 MOVNTQ"    %%mm4, 32(%0,%2,2)  \n\t"

  2073.                 MOVNTQ"    %%mm5, 40(%0,%2,2)  \n\t"

  2074.                 MOVNTQ"    %%mm6, 48(%0,%2,2)  \n\t"

  2075.                 MOVNTQ"    %%mm7, 56(%0,%2,2)"

  2076.                 :: "r"(d), "r"(s2), "r"(x)

  2077.                 :"memory");

  2078.         }

  2079.         for (;x<w;x++) d[2*x]=d[2*x+1]=s2[x];

  2080.     }

  2081.     __asm__(

  2082.             EMMS"       \n\t"

  2083.             SFENCE"     \n\t"

  2084.             ::: "memory"

  2085.         );

  2086. }

  2087. 

  2088. static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,

  2089.                                         uint8_t *dst,

  2090.                                         int width, int height,

  2091.                                         int srcStride1, int srcStride2,

  2092.                                         int srcStride3, int dstStride)

  2093. {

  2094.     x86_reg x;

  2095.     int y,w,h;

  2096.     w=width/2; h=height;

  2097.     for (y=0;y<h;y++) {

  2098.         const uint8_t* yp=src1+srcStride1*y;

  2099.         const uint8_t* up=src2+srcStride2*(y>>2);

  2100.         const uint8_t* vp=src3+srcStride3*(y>>2);

  2101.         uint8_t* d=dst+dstStride*y;

  2102.         x=0;

  2103.         for (;x<w-7;x+=8) {

  2104.             __asm__ volatile(

  2105.                 PREFETCH"   32(%1, %0)          \n\t"

  2106.                 PREFETCH"   32(%2, %0)          \n\t"

  2107.                 PREFETCH"   32(%3, %0)          \n\t"

  2108.                 "movq      (%1, %0, 4), %%mm0   \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */

  2109.                 "movq         (%2, %0), %%mm1   \n\t" /* U0U1U2U3U4U5U6U7 */

  2110.                 "movq         (%3, %0), %%mm2   \n\t" /* V0V1V2V3V4V5V6V7 */

  2111.                 "movq            %%mm0, %%mm3   \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */

  2112.                 "movq            %%mm1, %%mm4   \n\t" /* U0U1U2U3U4U5U6U7 */

  2113.                 "movq            %%mm2, %%mm5   \n\t" /* V0V1V2V3V4V5V6V7 */

  2114.                 "punpcklbw       %%mm1, %%mm1   \n\t" /* U0U0 U1U1 U2U2 U3U3 */

  2115.                 "punpcklbw       %%mm2, %%mm2   \n\t" /* V0V0 V1V1 V2V2 V3V3 */

  2116.                 "punpckhbw       %%mm4, %%mm4   \n\t" /* U4U4 U5U5 U6U6 U7U7 */

  2117.                 "punpckhbw       %%mm5, %%mm5   \n\t" /* V4V4 V5V5 V6V6 V7V7 */

  2118. 

  2119.                 "movq            %%mm1, %%mm6   \n\t"

  2120.                 "punpcklbw       %%mm2, %%mm1   \n\t" /* U0V0 U0V0 U1V1 U1V1*/

  2121.                 "punpcklbw       %%mm1, %%mm0   \n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/

  2122.                 "punpckhbw       %%mm1, %%mm3   \n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/

  2123.                 MOVNTQ"          %%mm0,  (%4, %0, 8)    \n\t"

  2124.                 MOVNTQ"          %%mm3, 8(%4, %0, 8)    \n\t"

  2125. 

  2126.                 "punpckhbw       %%mm2, %%mm6   \n\t" /* U2V2 U2V2 U3V3 U3V3*/

  2127.                 "movq     8(%1, %0, 4), %%mm0   \n\t"

  2128.                 "movq            %%mm0, %%mm3   \n\t"

  2129.                 "punpcklbw       %%mm6, %%mm0   \n\t" /* Y U2 Y V2 Y U2 Y V2*/

  2130.                 "punpckhbw       %%mm6, %%mm3   \n\t" /* Y U3 Y V3 Y U3 Y V3*/

  2131.                 MOVNTQ"          %%mm0, 16(%4, %0, 8)   \n\t"

  2132.                 MOVNTQ"          %%mm3, 24(%4, %0, 8)   \n\t"

  2133. 

  2134.                 "movq            %%mm4, %%mm6   \n\t"

  2135.                 "movq    16(%1, %0, 4), %%mm0   \n\t"

  2136.                 "movq            %%mm0, %%mm3   \n\t"

  2137.                 "punpcklbw       %%mm5, %%mm4   \n\t"

  2138.                 "punpcklbw       %%mm4, %%mm0   \n\t" /* Y U4 Y V4 Y U4 Y V4*/

  2139.                 "punpckhbw       %%mm4, %%mm3   \n\t" /* Y U5 Y V5 Y U5 Y V5*/

  2140.                 MOVNTQ"          %%mm0, 32(%4, %0, 8)   \n\t"

  2141.                 MOVNTQ"          %%mm3, 40(%4, %0, 8)   \n\t"

  2142. 

  2143.                 "punpckhbw       %%mm5, %%mm6   \n\t"

  2144.                 "movq    24(%1, %0, 4), %%mm0   \n\t"

  2145.                 "movq            %%mm0, %%mm3   \n\t"

  2146.                 "punpcklbw       %%mm6, %%mm0   \n\t" /* Y U6 Y V6 Y U6 Y V6*/

  2147.                 "punpckhbw       %%mm6, %%mm3   \n\t" /* Y U7 Y V7 Y U7 Y V7*/

  2148.                 MOVNTQ"          %%mm0, 48(%4, %0, 8)   \n\t"

  2149.                 MOVNTQ"          %%mm3, 56(%4, %0, 8)   \n\t"

  2150. 

  2151.                 : "+r" (x)

  2152.                 : "r"(yp), "r" (up), "r"(vp), "r"(d)

  2153.                 :"memory");

  2154.         }

  2155.         for (; x<w; x++) {

  2156.             const int x2 = x<<2;

  2157.             d[8*x+0] = yp[x2];

  2158.             d[8*x+1] = up[x];

  2159.             d[8*x+2] = yp[x2+1];

  2160.             d[8*x+3] = vp[x];

  2161.             d[8*x+4] = yp[x2+2];

  2162.             d[8*x+5] = up[x];

  2163.             d[8*x+6] = yp[x2+3];

  2164.             d[8*x+7] = vp[x];

  2165.         }

  2166.     }

  2167.     __asm__(

  2168.             EMMS"       \n\t"

  2169.             SFENCE"     \n\t"

  2170.             ::: "memory"

  2171.         );

  2172. }

  2173. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2174. 

  2175. static void RENAME(extract_even)(const uint8_t *src, uint8_t *dst, x86_reg count)

  2176. {

  2177.     dst +=   count;

  2178.     src += 2*count;

  2179.     count= - count;

  2180. 

  2181.     if(count <= -16) {

  2182.         count += 15;

  2183.         __asm__ volatile(

  2184.             "pcmpeqw       %%mm7, %%mm7        \n\t"

  2185.             "psrlw            $8, %%mm7        \n\t"

  2186.             "1:                                \n\t"

  2187.             "movq -30(%1, %0, 2), %%mm0        \n\t"

  2188.             "movq -22(%1, %0, 2), %%mm1        \n\t"

  2189.             "movq -14(%1, %0, 2), %%mm2        \n\t"

  2190.             "movq  -6(%1, %0, 2), %%mm3        \n\t"

  2191.             "pand          %%mm7, %%mm0        \n\t"

  2192.             "pand          %%mm7, %%mm1        \n\t"

  2193.             "pand          %%mm7, %%mm2        \n\t"

  2194.             "pand          %%mm7, %%mm3        \n\t"

  2195.             "packuswb      %%mm1, %%mm0        \n\t"

  2196.             "packuswb      %%mm3, %%mm2        \n\t"

  2197.             MOVNTQ"        %%mm0,-15(%2, %0)   \n\t"

  2198.             MOVNTQ"        %%mm2,- 7(%2, %0)   \n\t"

  2199.             "add             $16, %0           \n\t"

  2200.             " js 1b                            \n\t"

  2201.             : "+r"(count)

  2202.             : "r"(src), "r"(dst)

  2203.         );

  2204.         count -= 15;

  2205.     }

  2206.     while(count<0) {

  2207.         dst[count]= src[2*count];

  2208.         count++;

  2209.     }

  2210. }

  2211. 

  2212. static void RENAME(extract_odd)(const uint8_t *src, uint8_t *dst, x86_reg count)

  2213. {

  2214.     src ++;

  2215.     dst +=   count;

  2216.     src += 2*count;

  2217.     count= - count;

  2218. 

  2219.     if(count < -16) {

  2220.         count += 16;

  2221.         __asm__ volatile(

  2222.             "pcmpeqw       %%mm7, %%mm7        \n\t"

  2223.             "psrlw            $8, %%mm7        \n\t"

  2224.             "1:                                \n\t"

  2225.             "movq -32(%1, %0, 2), %%mm0        \n\t"

  2226.             "movq -24(%1, %0, 2), %%mm1        \n\t"

  2227.             "movq -16(%1, %0, 2), %%mm2        \n\t"

  2228.             "movq  -8(%1, %0, 2), %%mm3        \n\t"

  2229.             "pand          %%mm7, %%mm0        \n\t"

  2230.             "pand          %%mm7, %%mm1        \n\t"

  2231.             "pand          %%mm7, %%mm2        \n\t"

  2232.             "pand          %%mm7, %%mm3        \n\t"

  2233.             "packuswb      %%mm1, %%mm0        \n\t"

  2234.             "packuswb      %%mm3, %%mm2        \n\t"

  2235.             MOVNTQ"        %%mm0,-16(%2, %0)   \n\t"

  2236.             MOVNTQ"        %%mm2,- 8(%2, %0)   \n\t"

  2237.             "add             $16, %0           \n\t"

  2238.             " js 1b                            \n\t"

  2239.             : "+r"(count)

  2240.             : "r"(src), "r"(dst)

  2241.         );

  2242.         count -= 16;

  2243.     }

  2244.     while(count<0) {

  2245.         dst[count]= src[2*count];

  2246.         count++;

  2247.     }

  2248. }

  2249. 

  2250. #if !COMPILE_TEMPLATE_AMD3DNOW

  2251. static void RENAME(extract_even2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)

  2252. {

  2253.     dst0+=   count;

  2254.     dst1+=   count;

  2255.     src += 4*count;

  2256.     count= - count;

  2257.     if(count <= -8) {

  2258.         count += 7;

  2259.         __asm__ volatile(

  2260.             "pcmpeqw       %%mm7, %%mm7        \n\t"

  2261.             "psrlw            $8, %%mm7        \n\t"

  2262.             "1:                                \n\t"

  2263.             "movq -28(%1, %0, 4), %%mm0        \n\t"

  2264.             "movq -20(%1, %0, 4), %%mm1        \n\t"

  2265.             "movq -12(%1, %0, 4), %%mm2        \n\t"

  2266.             "movq  -4(%1, %0, 4), %%mm3        \n\t"

  2267.             "pand          %%mm7, %%mm0        \n\t"

  2268.             "pand          %%mm7, %%mm1        \n\t"

  2269.             "pand          %%mm7, %%mm2        \n\t"

  2270.             "pand          %%mm7, %%mm3        \n\t"

  2271.             "packuswb      %%mm1, %%mm0        \n\t"

  2272.             "packuswb      %%mm3, %%mm2        \n\t"

  2273.             "movq          %%mm0, %%mm1        \n\t"

  2274.             "movq          %%mm2, %%mm3        \n\t"

  2275.             "psrlw            $8, %%mm0        \n\t"

  2276.             "psrlw            $8, %%mm2        \n\t"

  2277.             "pand          %%mm7, %%mm1        \n\t"

  2278.             "pand          %%mm7, %%mm3        \n\t"

  2279.             "packuswb      %%mm2, %%mm0        \n\t"

  2280.             "packuswb      %%mm3, %%mm1        \n\t"

  2281.             MOVNTQ"        %%mm0,- 7(%3, %0)   \n\t"

  2282.             MOVNTQ"        %%mm1,- 7(%2, %0)   \n\t"

  2283.             "add              $8, %0           \n\t"

  2284.             " js 1b                            \n\t"

  2285.             : "+r"(count)

  2286.             : "r"(src), "r"(dst0), "r"(dst1)

  2287.         );

  2288.         count -= 7;

  2289.     }

  2290.     while(count<0) {

  2291.         dst0[count]= src[4*count+0];

  2292.         dst1[count]= src[4*count+2];

  2293.         count++;

  2294.     }

  2295. }

  2296. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2297. 

  2298. static void RENAME(extract_even2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)

  2299. {

  2300.     dst0 +=   count;

  2301.     dst1 +=   count;

  2302.     src0 += 4*count;

  2303.     src1 += 4*count;

  2304.     count= - count;

  2305. #ifdef PAVGB

  2306.     if(count <= -8) {

  2307.         count += 7;

  2308.         __asm__ volatile(

  2309.             "pcmpeqw        %%mm7, %%mm7        \n\t"

  2310.             "psrlw             $8, %%mm7        \n\t"

  2311.             "1:                                \n\t"

  2312.             "movq  -28(%1, %0, 4), %%mm0        \n\t"

  2313.             "movq  -20(%1, %0, 4), %%mm1        \n\t"

  2314.             "movq  -12(%1, %0, 4), %%mm2        \n\t"

  2315.             "movq   -4(%1, %0, 4), %%mm3        \n\t"

  2316.             PAVGB" -28(%2, %0, 4), %%mm0        \n\t"

  2317.             PAVGB" -20(%2, %0, 4), %%mm1        \n\t"

  2318.             PAVGB" -12(%2, %0, 4), %%mm2        \n\t"

  2319.             PAVGB" - 4(%2, %0, 4), %%mm3        \n\t"

  2320.             "pand           %%mm7, %%mm0        \n\t"

  2321.             "pand           %%mm7, %%mm1        \n\t"

  2322.             "pand           %%mm7, %%mm2        \n\t"

  2323.             "pand           %%mm7, %%mm3        \n\t"

  2324.             "packuswb       %%mm1, %%mm0        \n\t"

  2325.             "packuswb       %%mm3, %%mm2        \n\t"

  2326.             "movq           %%mm0, %%mm1        \n\t"

  2327.             "movq           %%mm2, %%mm3        \n\t"

  2328.             "psrlw             $8, %%mm0        \n\t"

  2329.             "psrlw             $8, %%mm2        \n\t"

  2330.             "pand           %%mm7, %%mm1        \n\t"

  2331.             "pand           %%mm7, %%mm3        \n\t"

  2332.             "packuswb       %%mm2, %%mm0        \n\t"

  2333.             "packuswb       %%mm3, %%mm1        \n\t"

  2334.             MOVNTQ"         %%mm0,- 7(%4, %0)   \n\t"

  2335.             MOVNTQ"         %%mm1,- 7(%3, %0)   \n\t"

  2336.             "add               $8, %0           \n\t"

  2337.             " js 1b                            \n\t"

  2338.             : "+r"(count)

  2339.             : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)

  2340.         );

  2341.         count -= 7;

  2342.     }

  2343. #endif

  2344.     while(count<0) {

  2345.         dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;

  2346.         dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;

  2347.         count++;

  2348.     }

  2349. }

  2350. 

  2351. #if !COMPILE_TEMPLATE_AMD3DNOW

  2352. static void RENAME(extract_odd2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)

  2353. {

  2354.     dst0+=   count;

  2355.     dst1+=   count;

  2356.     src += 4*count;

  2357.     count= - count;

  2358.     if(count <= -8) {

  2359.         count += 7;

  2360.         __asm__ volatile(

  2361.             "pcmpeqw       %%mm7, %%mm7        \n\t"

  2362.             "psrlw            $8, %%mm7        \n\t"

  2363.             "1:                                \n\t"

  2364.             "movq -28(%1, %0, 4), %%mm0        \n\t"

  2365.             "movq -20(%1, %0, 4), %%mm1        \n\t"

  2366.             "movq -12(%1, %0, 4), %%mm2        \n\t"

  2367.             "movq  -4(%1, %0, 4), %%mm3        \n\t"

  2368.             "psrlw            $8, %%mm0        \n\t"

  2369.             "psrlw            $8, %%mm1        \n\t"

  2370.             "psrlw            $8, %%mm2        \n\t"

  2371.             "psrlw            $8, %%mm3        \n\t"

  2372.             "packuswb      %%mm1, %%mm0        \n\t"

  2373.             "packuswb      %%mm3, %%mm2        \n\t"

  2374.             "movq          %%mm0, %%mm1        \n\t"

  2375.             "movq          %%mm2, %%mm3        \n\t"

  2376.             "psrlw            $8, %%mm0        \n\t"

  2377.             "psrlw            $8, %%mm2        \n\t"

  2378.             "pand          %%mm7, %%mm1        \n\t"

  2379.             "pand          %%mm7, %%mm3        \n\t"

  2380.             "packuswb      %%mm2, %%mm0        \n\t"

  2381.             "packuswb      %%mm3, %%mm1        \n\t"

  2382.             MOVNTQ"        %%mm0,- 7(%3, %0)   \n\t"

  2383.             MOVNTQ"        %%mm1,- 7(%2, %0)   \n\t"

  2384.             "add              $8, %0           \n\t"

  2385.             " js 1b                            \n\t"

  2386.             : "+r"(count)

  2387.             : "r"(src), "r"(dst0), "r"(dst1)

  2388.         );

  2389.         count -= 7;

  2390.     }

  2391.     src++;

  2392.     while(count<0) {

  2393.         dst0[count]= src[4*count+0];

  2394.         dst1[count]= src[4*count+2];

  2395.         count++;

  2396.     }

  2397. }

  2398. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2399. 

  2400. static void RENAME(extract_odd2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)

  2401. {

  2402.     dst0 +=   count;

  2403.     dst1 +=   count;

  2404.     src0 += 4*count;

  2405.     src1 += 4*count;

  2406.     count= - count;

  2407. #ifdef PAVGB

  2408.     if(count <= -8) {

  2409.         count += 7;

  2410.         __asm__ volatile(

  2411.             "pcmpeqw        %%mm7, %%mm7        \n\t"

  2412.             "psrlw             $8, %%mm7        \n\t"

  2413.             "1:                                \n\t"

  2414.             "movq  -28(%1, %0, 4), %%mm0        \n\t"

  2415.             "movq  -20(%1, %0, 4), %%mm1        \n\t"

  2416.             "movq  -12(%1, %0, 4), %%mm2        \n\t"

  2417.             "movq   -4(%1, %0, 4), %%mm3        \n\t"

  2418.             PAVGB" -28(%2, %0, 4), %%mm0        \n\t"

  2419.             PAVGB" -20(%2, %0, 4), %%mm1        \n\t"

  2420.             PAVGB" -12(%2, %0, 4), %%mm2        \n\t"

  2421.             PAVGB" - 4(%2, %0, 4), %%mm3        \n\t"

  2422.             "psrlw             $8, %%mm0        \n\t"

  2423.             "psrlw             $8, %%mm1        \n\t"

  2424.             "psrlw             $8, %%mm2        \n\t"

  2425.             "psrlw             $8, %%mm3        \n\t"

  2426.             "packuswb       %%mm1, %%mm0        \n\t"

  2427.             "packuswb       %%mm3, %%mm2        \n\t"

  2428.             "movq           %%mm0, %%mm1        \n\t"

  2429.             "movq           %%mm2, %%mm3        \n\t"

  2430.             "psrlw             $8, %%mm0        \n\t"

  2431.             "psrlw             $8, %%mm2        \n\t"

  2432.             "pand           %%mm7, %%mm1        \n\t"

  2433.             "pand           %%mm7, %%mm3        \n\t"

  2434.             "packuswb       %%mm2, %%mm0        \n\t"

  2435.             "packuswb       %%mm3, %%mm1        \n\t"

  2436.             MOVNTQ"         %%mm0,- 7(%4, %0)   \n\t"

  2437.             MOVNTQ"         %%mm1,- 7(%3, %0)   \n\t"

  2438.             "add               $8, %0           \n\t"

  2439.             " js 1b                            \n\t"

  2440.             : "+r"(count)

  2441.             : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)

  2442.         );

  2443.         count -= 7;

  2444.     }

  2445. #endif

  2446.     src0++;

  2447.     src1++;

  2448.     while(count<0) {

  2449.         dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;

  2450.         dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;

  2451.         count++;

  2452.     }

  2453. }

  2454. 

  2455. static void RENAME(yuyvtoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,

  2456.                                  int width, int height,

  2457.                                  int lumStride, int chromStride, int srcStride)

  2458. {

  2459.     int y;

  2460.     const int chromWidth = AV_CEIL_RSHIFT(width, 1);

  2461. 

  2462.     for (y=0; y<height; y++) {

  2463.         RENAME(extract_even)(src, ydst, width);

  2464.         if(y&1) {

  2465.             RENAME(extract_odd2avg)(src-srcStride, src, udst, vdst, chromWidth);

  2466.             udst+= chromStride;

  2467.             vdst+= chromStride;

  2468.         }

  2469. 

  2470.         src += srcStride;

  2471.         ydst+= lumStride;

  2472.     }

  2473.     __asm__(

  2474.             EMMS"       \n\t"

  2475.             SFENCE"     \n\t"

  2476.             ::: "memory"

  2477.         );

  2478. }

  2479. 

  2480. #if !COMPILE_TEMPLATE_AMD3DNOW

  2481. static void RENAME(yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,

  2482.                                  int width, int height,

  2483.                                  int lumStride, int chromStride, int srcStride)

  2484. {

  2485.     int y;

  2486.     const int chromWidth = AV_CEIL_RSHIFT(width, 1);

  2487. 

  2488.     for (y=0; y<height; y++) {

  2489.         RENAME(extract_even)(src, ydst, width);

  2490.         RENAME(extract_odd2)(src, udst, vdst, chromWidth);

  2491. 

  2492.         src += srcStride;

  2493.         ydst+= lumStride;

  2494.         udst+= chromStride;

  2495.         vdst+= chromStride;

  2496.     }

  2497.     __asm__(

  2498.             EMMS"       \n\t"

  2499.             SFENCE"     \n\t"

  2500.             ::: "memory"

  2501.         );

  2502. }

  2503. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2504. 

  2505. static void RENAME(uyvytoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,

  2506.                                  int width, int height,

  2507.                                  int lumStride, int chromStride, int srcStride)

  2508. {

  2509.     int y;

  2510.     const int chromWidth = AV_CEIL_RSHIFT(width, 1);

  2511. 

  2512.     for (y=0; y<height; y++) {

  2513.         RENAME(extract_odd)(src, ydst, width);

  2514.         if(y&1) {

  2515.             RENAME(extract_even2avg)(src-srcStride, src, udst, vdst, chromWidth);

  2516.             udst+= chromStride;

  2517.             vdst+= chromStride;

  2518.         }

  2519. 

  2520.         src += srcStride;

  2521.         ydst+= lumStride;

  2522.     }

  2523.     __asm__(

  2524.             EMMS"       \n\t"

  2525.             SFENCE"     \n\t"

  2526.             ::: "memory"

  2527.         );

  2528. }

  2529. 

  2530. #if !COMPILE_TEMPLATE_AMD3DNOW

  2531. static void RENAME(uyvytoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,

  2532.                                  int width, int height,

  2533.                                  int lumStride, int chromStride, int srcStride)

  2534. {

  2535.     int y;

  2536.     const int chromWidth = AV_CEIL_RSHIFT(width, 1);

  2537. 

  2538.     for (y=0; y<height; y++) {

  2539.         RENAME(extract_odd)(src, ydst, width);

  2540.         RENAME(extract_even2)(src, udst, vdst, chromWidth);

  2541. 

  2542.         src += srcStride;

  2543.         ydst+= lumStride;

  2544.         udst+= chromStride;

  2545.         vdst+= chromStride;

  2546.     }

  2547.     __asm__(

  2548.             EMMS"       \n\t"

  2549.             SFENCE"     \n\t"

  2550.             ::: "memory"

  2551.         );

  2552. }

  2553. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2554. #endif /* !COMPILE_TEMPLATE_SSE2 */

  2555. 

  2556. static av_cold void RENAME(rgb2rgb_init)(void)

  2557. {

  2558. #if !COMPILE_TEMPLATE_SSE2

  2559. #if !COMPILE_TEMPLATE_AMD3DNOW

  2560.     rgb15to16          = RENAME(rgb15to16);

  2561.     rgb15tobgr24       = RENAME(rgb15tobgr24);

  2562.     rgb15to32          = RENAME(rgb15to32);

  2563.     rgb16tobgr24       = RENAME(rgb16tobgr24);

  2564.     rgb16to32          = RENAME(rgb16to32);

  2565.     rgb16to15          = RENAME(rgb16to15);

  2566.     rgb24tobgr16       = RENAME(rgb24tobgr16);

  2567.     rgb24tobgr15       = RENAME(rgb24tobgr15);

  2568.     rgb24tobgr32       = RENAME(rgb24tobgr32);

  2569.     rgb32to16          = RENAME(rgb32to16);

  2570.     rgb32to15          = RENAME(rgb32to15);

  2571.     rgb32tobgr24       = RENAME(rgb32tobgr24);

  2572.     rgb24to15          = RENAME(rgb24to15);

  2573.     rgb24to16          = RENAME(rgb24to16);

  2574.     rgb24tobgr24       = RENAME(rgb24tobgr24);

  2575.     shuffle_bytes_2103 = RENAME(shuffle_bytes_2103);

  2576.     rgb32tobgr16       = RENAME(rgb32tobgr16);

  2577.     rgb32tobgr15       = RENAME(rgb32tobgr15);

  2578.     yv12toyuy2         = RENAME(yv12toyuy2);

  2579.     yv12touyvy         = RENAME(yv12touyvy);

  2580.     yuv422ptoyuy2      = RENAME(yuv422ptoyuy2);

  2581.     yuv422ptouyvy      = RENAME(yuv422ptouyvy);

  2582.     yuy2toyv12         = RENAME(yuy2toyv12);

  2583.     vu9_to_vu12        = RENAME(vu9_to_vu12);

  2584.     yvu9_to_yuy2       = RENAME(yvu9_to_yuy2);

  2585.     uyvytoyuv422       = RENAME(uyvytoyuv422);

  2586.     yuyvtoyuv422       = RENAME(yuyvtoyuv422);

  2587. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */

  2588. 

  2589. #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW

  2590.     planar2x           = RENAME(planar2x);

  2591. #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */

  2592. #if HAVE_7REGS

  2593.     ff_rgb24toyv12     = RENAME(rgb24toyv12);

  2594. #endif /* HAVE_7REGS */

  2595. 

  2596.     yuyvtoyuv420       = RENAME(yuyvtoyuv420);

  2597.     uyvytoyuv420       = RENAME(uyvytoyuv420);

  2598. #endif /* !COMPILE_TEMPLATE_SSE2 */

  2599. 

  2600. #if !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX

  2601.     interleaveBytes    = RENAME(interleaveBytes);

  2602. #endif /* !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX */

  2603. #if !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL

  2604. #if !COMPILE_TEMPLATE_AMD3DNOW && (ARCH_X86_32 || COMPILE_TEMPLATE_SSE2) && COMPILE_TEMPLATE_MMXEXT == COMPILE_TEMPLATE_SSE2 && HAVE_X86ASM

  2605.     deinterleaveBytes  = RENAME(deinterleaveBytes);

  2606. #endif

  2607. #endif

  2608. }