falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
8579 Commits
32 Branches
850MB
Tree: 024c7378c3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '024c7378c3'
${ noResults }
FFmpeg/libswscale/rgb2rgb_template.c

2689 lines
68KB
Raw Blame History 

  1. /*

  2.  *

  3.  *  rgb2rgb.c, Software RGB to RGB convertor

  4.  *  pluralize by Software PAL8 to RGB convertor

  5.  *               Software YUV to YUV convertor

  6.  *               Software YUV to RGB convertor

  7.  *  Written by Nick Kurshev.

  8.  *  palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)

  9.  *  lot of big-endian byteorder fixes by Alex Beregszaszi

  10.  *

  11.  * This file is part of FFmpeg.

  12.  *

  13.  * FFmpeg is free software; you can redistribute it and/or modify

  14.  * it under the terms of the GNU General Public License as published by

  15.  * the Free Software Foundation; either version 2 of the License, or

  16.  * (at your option) any later version.

  17.  *

  18.  * FFmpeg is distributed in the hope that it will be useful,

  19.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  20.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  21.  * GNU General Public License for more details.

  22.  *

  23.  * You should have received a copy of the GNU General Public License

  24.  * along with FFmpeg; if not, write to the Free Software

  25.  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

  26.  * 

  27.  * the C code (not assembly, mmx, ...) of this file can be used

  28.  * under the LGPL license too

  29.  */

  30. 

  31. #include <stddef.h>

  32. #include <inttypes.h> /* for __WORDSIZE */

  33. 

  34. #ifndef __WORDSIZE

  35. // #warning You have misconfigured system and probably will lose performance!

  36. #define __WORDSIZE MP_WORDSIZE

  37. #endif

  38. 

  39. #undef PREFETCH

  40. #undef MOVNTQ

  41. #undef EMMS

  42. #undef SFENCE

  43. #undef MMREG_SIZE

  44. #undef PREFETCHW

  45. #undef PAVGB

  46. 

  47. #ifdef HAVE_SSE2

  48. #define MMREG_SIZE 16

  49. #else

  50. #define MMREG_SIZE 8

  51. #endif

  52. 

  53. #ifdef HAVE_3DNOW

  54. #define PREFETCH  "prefetch"

  55. #define PREFETCHW "prefetchw"

  56. #define PAVGB	  "pavgusb"

  57. #elif defined ( HAVE_MMX2 )

  58. #define PREFETCH "prefetchnta"

  59. #define PREFETCHW "prefetcht0"

  60. #define PAVGB	  "pavgb"

  61. #else

  62. #ifdef __APPLE__

  63. #define PREFETCH "#"

  64. #define PREFETCHW "#"

  65. #else

  66. #define PREFETCH  " # nop"

  67. #define PREFETCHW " # nop"

  68. #endif

  69. #endif

  70. 

  71. #ifdef HAVE_3DNOW

  72. /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */

  73. #define EMMS     "femms"

  74. #else

  75. #define EMMS     "emms"

  76. #endif

  77. 

  78. #ifdef HAVE_MMX2

  79. #define MOVNTQ "movntq"

  80. #define SFENCE "sfence"

  81. #else

  82. #define MOVNTQ "movq"

  83. #define SFENCE " # nop"

  84. #endif

  85. 

  86. static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)

  87. {

  88.   uint8_t *dest = dst;

  89.   const uint8_t *s = src;

  90.   const uint8_t *end;

  91. #ifdef HAVE_MMX

  92.   const uint8_t *mm_end;

  93. #endif

  94.   end = s + src_size;

  95. #ifdef HAVE_MMX

  96.   __asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  97.   mm_end = end - 23;

  98.   __asm __volatile("movq	%0, %%mm7"::"m"(mask32):"memory");

  99.   while(s < mm_end)

  100.   {

  101.     __asm __volatile(

  102. 	PREFETCH"	32%1\n\t"

  103. 	"movd	%1, %%mm0\n\t"

  104. 	"punpckldq 3%1, %%mm0\n\t"

  105. 	"movd	6%1, %%mm1\n\t"

  106. 	"punpckldq 9%1, %%mm1\n\t"

  107. 	"movd	12%1, %%mm2\n\t"

  108. 	"punpckldq 15%1, %%mm2\n\t"

  109. 	"movd	18%1, %%mm3\n\t"

  110. 	"punpckldq 21%1, %%mm3\n\t"

  111. 	"pand	%%mm7, %%mm0\n\t"

  112. 	"pand	%%mm7, %%mm1\n\t"

  113. 	"pand	%%mm7, %%mm2\n\t"

  114. 	"pand	%%mm7, %%mm3\n\t"

  115. 	MOVNTQ"	%%mm0, %0\n\t"

  116. 	MOVNTQ"	%%mm1, 8%0\n\t"

  117. 	MOVNTQ"	%%mm2, 16%0\n\t"

  118. 	MOVNTQ"	%%mm3, 24%0"

  119. 	:"=m"(*dest)

  120. 	:"m"(*s)

  121. 	:"memory");

  122.     dest += 32;

  123.     s += 24;

  124.   }

  125.   __asm __volatile(SFENCE:::"memory");

  126.   __asm __volatile(EMMS:::"memory");

  127. #endif

  128.   while(s < end)

  129.   {

  130. #ifdef WORDS_BIGENDIAN

  131.     /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */

  132.     *dest++ = 0;

  133.     *dest++ = s[2];

  134.     *dest++ = s[1];

  135.     *dest++ = s[0];

  136.     s+=3;

  137. #else

  138.     *dest++ = *s++;

  139.     *dest++ = *s++;

  140.     *dest++ = *s++;

  141.     *dest++ = 0;

  142. #endif

  143.   }

  144. }

  145. 

  146. static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)

  147. {

  148.   uint8_t *dest = dst;

  149.   const uint8_t *s = src;

  150.   const uint8_t *end;

  151. #ifdef HAVE_MMX

  152.   const uint8_t *mm_end;

  153. #endif

  154.   end = s + src_size;

  155. #ifdef HAVE_MMX

  156.   __asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  157.   mm_end = end - 31;

  158.   while(s < mm_end)

  159.   {

  160.     __asm __volatile(

  161. 	PREFETCH"	32%1\n\t"

  162. 	"movq	%1, %%mm0\n\t"

  163. 	"movq	8%1, %%mm1\n\t"

  164. 	"movq	16%1, %%mm4\n\t"

  165. 	"movq	24%1, %%mm5\n\t"

  166. 	"movq	%%mm0, %%mm2\n\t"

  167. 	"movq	%%mm1, %%mm3\n\t"

  168. 	"movq	%%mm4, %%mm6\n\t"

  169. 	"movq	%%mm5, %%mm7\n\t"

  170. 	"psrlq	$8, %%mm2\n\t"

  171. 	"psrlq	$8, %%mm3\n\t"

  172. 	"psrlq	$8, %%mm6\n\t"

  173. 	"psrlq	$8, %%mm7\n\t"

  174. 	"pand	%2, %%mm0\n\t"

  175. 	"pand	%2, %%mm1\n\t"

  176. 	"pand	%2, %%mm4\n\t"

  177. 	"pand	%2, %%mm5\n\t"

  178. 	"pand	%3, %%mm2\n\t"

  179. 	"pand	%3, %%mm3\n\t"

  180. 	"pand	%3, %%mm6\n\t"

  181. 	"pand	%3, %%mm7\n\t"

  182. 	"por	%%mm2, %%mm0\n\t"

  183. 	"por	%%mm3, %%mm1\n\t"

  184. 	"por	%%mm6, %%mm4\n\t"

  185. 	"por	%%mm7, %%mm5\n\t"

  186. 

  187. 	"movq	%%mm1, %%mm2\n\t"

  188. 	"movq	%%mm4, %%mm3\n\t"

  189. 	"psllq	$48, %%mm2\n\t"

  190. 	"psllq	$32, %%mm3\n\t"

  191. 	"pand	%4, %%mm2\n\t"

  192. 	"pand	%5, %%mm3\n\t"

  193. 	"por	%%mm2, %%mm0\n\t"

  194. 	"psrlq	$16, %%mm1\n\t"

  195. 	"psrlq	$32, %%mm4\n\t"

  196. 	"psllq	$16, %%mm5\n\t"

  197. 	"por	%%mm3, %%mm1\n\t"

  198. 	"pand	%6, %%mm5\n\t"

  199. 	"por	%%mm5, %%mm4\n\t"

  200. 

  201. 	MOVNTQ"	%%mm0, %0\n\t"

  202. 	MOVNTQ"	%%mm1, 8%0\n\t"

  203. 	MOVNTQ"	%%mm4, 16%0"

  204. 	:"=m"(*dest)

  205. 	:"m"(*s),"m"(mask24l),

  206. 	 "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)

  207. 	:"memory");

  208.     dest += 24;

  209.     s += 32;

  210.   }

  211.   __asm __volatile(SFENCE:::"memory");

  212.   __asm __volatile(EMMS:::"memory");

  213. #endif

  214.   while(s < end)

  215.   {

  216. #ifdef WORDS_BIGENDIAN

  217.     /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */

  218.     s++;

  219.     dest[2] = *s++;

  220.     dest[1] = *s++;

  221.     dest[0] = *s++;

  222.     dest += 3;

  223. #else

  224.     *dest++ = *s++;

  225.     *dest++ = *s++;

  226.     *dest++ = *s++;

  227.     s++;

  228. #endif

  229.   }

  230. }

  231. 

  232. /*

  233.  Original by Strepto/Astral

  234.  ported to gcc & bugfixed : A'rpi

  235.  MMX2, 3DNOW optimization by Nick Kurshev

  236.  32bit c version, and and&add trick by Michael Niedermayer

  237. */

  238. static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)

  239. {

  240.   register const uint8_t* s=src;

  241.   register uint8_t* d=dst;

  242.   register const uint8_t *end;

  243.   const uint8_t *mm_end;

  244.   end = s + src_size;

  245. #ifdef HAVE_MMX

  246.   __asm __volatile(PREFETCH"	%0"::"m"(*s));

  247.   __asm __volatile("movq	%0, %%mm4"::"m"(mask15s));

  248.   mm_end = end - 15;

  249.   while(s<mm_end)

  250.   {

  251. 	__asm __volatile(

  252. 		PREFETCH"	32%1\n\t"

  253. 		"movq	%1, %%mm0\n\t"

  254. 		"movq	8%1, %%mm2\n\t"

  255. 		"movq	%%mm0, %%mm1\n\t"

  256. 		"movq	%%mm2, %%mm3\n\t"

  257. 		"pand	%%mm4, %%mm0\n\t"

  258. 		"pand	%%mm4, %%mm2\n\t"

  259. 		"paddw	%%mm1, %%mm0\n\t"

  260. 		"paddw	%%mm3, %%mm2\n\t"

  261. 		MOVNTQ"	%%mm0, %0\n\t"

  262. 		MOVNTQ"	%%mm2, 8%0"

  263. 		:"=m"(*d)

  264. 		:"m"(*s)

  265. 		);

  266. 	d+=16;

  267. 	s+=16;

  268.   }

  269.   __asm __volatile(SFENCE:::"memory");

  270.   __asm __volatile(EMMS:::"memory");

  271. #endif

  272.     mm_end = end - 3;

  273.     while(s < mm_end)

  274.     {

  275. 	register unsigned x= *((uint32_t *)s);

  276. 	*((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);

  277. 	d+=4;

  278. 	s+=4;

  279.     }

  280.     if(s < end)

  281.     {

  282. 	register unsigned short x= *((uint16_t *)s);

  283. 	*((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);

  284.     }

  285. }

  286. 

  287. static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)

  288. {

  289.   register const uint8_t* s=src;

  290.   register uint8_t* d=dst;

  291.   register const uint8_t *end;

  292.   const uint8_t *mm_end;

  293.   end = s + src_size;

  294. #ifdef HAVE_MMX

  295.   __asm __volatile(PREFETCH"	%0"::"m"(*s));

  296.   __asm __volatile("movq	%0, %%mm7"::"m"(mask15rg));

  297.   __asm __volatile("movq	%0, %%mm6"::"m"(mask15b));

  298.   mm_end = end - 15;

  299.   while(s<mm_end)

  300.   {

  301. 	__asm __volatile(

  302. 		PREFETCH"	32%1\n\t"

  303. 		"movq	%1, %%mm0\n\t"

  304. 		"movq	8%1, %%mm2\n\t"

  305. 		"movq	%%mm0, %%mm1\n\t"

  306. 		"movq	%%mm2, %%mm3\n\t"

  307. 		"psrlq	$1, %%mm0\n\t"

  308. 		"psrlq	$1, %%mm2\n\t"

  309. 		"pand	%%mm7, %%mm0\n\t"

  310. 		"pand	%%mm7, %%mm2\n\t"

  311. 		"pand	%%mm6, %%mm1\n\t"

  312. 		"pand	%%mm6, %%mm3\n\t"

  313. 		"por	%%mm1, %%mm0\n\t"

  314. 		"por	%%mm3, %%mm2\n\t"

  315. 		MOVNTQ"	%%mm0, %0\n\t"

  316. 		MOVNTQ"	%%mm2, 8%0"

  317. 		:"=m"(*d)

  318. 		:"m"(*s)

  319. 		);

  320. 	d+=16;

  321. 	s+=16;

  322.   }

  323.   __asm __volatile(SFENCE:::"memory");

  324.   __asm __volatile(EMMS:::"memory");

  325. #endif

  326.     mm_end = end - 3;

  327.     while(s < mm_end)

  328.     {

  329. 	register uint32_t x= *((uint32_t *)s);

  330. 	*((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);

  331. 	s+=4;

  332. 	d+=4;

  333.     }

  334.     if(s < end)

  335.     {

  336. 	register uint16_t x= *((uint16_t *)s);

  337. 	*((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);

  338. 	s+=2;

  339. 	d+=2;

  340.     }

  341. }

  342. 

  343. static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)

  344. {

  345. 	const uint8_t *s = src;

  346. 	const uint8_t *end;

  347. #ifdef HAVE_MMX

  348. 	const uint8_t *mm_end;

  349. #endif

  350. 	uint16_t *d = (uint16_t *)dst;

  351. 	end = s + src_size;

  352. #ifdef HAVE_MMX

  353. 	mm_end = end - 15;

  354. #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)

  355. 	asm volatile(

  356. 		"movq %3, %%mm5			\n\t"

  357. 		"movq %4, %%mm6			\n\t"

  358. 		"movq %5, %%mm7			\n\t"

  359. 		ASMALIGN(4)

  360. 		"1:				\n\t"

  361. 		PREFETCH" 32(%1)		\n\t"

  362. 		"movd	(%1), %%mm0		\n\t"

  363. 		"movd	4(%1), %%mm3		\n\t"

  364. 		"punpckldq 8(%1), %%mm0		\n\t"

  365. 		"punpckldq 12(%1), %%mm3	\n\t"

  366. 		"movq %%mm0, %%mm1		\n\t"

  367. 		"movq %%mm3, %%mm4		\n\t"

  368. 		"pand %%mm6, %%mm0		\n\t"

  369. 		"pand %%mm6, %%mm3		\n\t"

  370. 		"pmaddwd %%mm7, %%mm0		\n\t"

  371. 		"pmaddwd %%mm7, %%mm3		\n\t"

  372. 		"pand %%mm5, %%mm1		\n\t"

  373. 		"pand %%mm5, %%mm4		\n\t"

  374. 		"por %%mm1, %%mm0		\n\t"	

  375. 		"por %%mm4, %%mm3		\n\t"

  376. 		"psrld $5, %%mm0		\n\t"

  377. 		"pslld $11, %%mm3		\n\t"

  378. 		"por %%mm3, %%mm0		\n\t"

  379. 		MOVNTQ"	%%mm0, (%0)		\n\t"

  380. 		"add $16, %1			\n\t"

  381. 		"add $8, %0			\n\t"

  382. 		"cmp %2, %1			\n\t"

  383. 		" jb 1b				\n\t"

  384. 		: "+r" (d), "+r"(s)

  385. 		: "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)

  386. 	);

  387. #else

  388. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  389. 	__asm __volatile(

  390. 	    "movq	%0, %%mm7\n\t"

  391. 	    "movq	%1, %%mm6\n\t"

  392. 	    ::"m"(red_16mask),"m"(green_16mask));

  393. 	while(s < mm_end)

  394. 	{

  395. 	    __asm __volatile(

  396. 		PREFETCH" 32%1\n\t"

  397. 		"movd	%1, %%mm0\n\t"

  398. 		"movd	4%1, %%mm3\n\t"

  399. 		"punpckldq 8%1, %%mm0\n\t"

  400. 		"punpckldq 12%1, %%mm3\n\t"

  401. 		"movq	%%mm0, %%mm1\n\t"

  402. 		"movq	%%mm0, %%mm2\n\t"

  403. 		"movq	%%mm3, %%mm4\n\t"

  404. 		"movq	%%mm3, %%mm5\n\t"

  405. 		"psrlq	$3, %%mm0\n\t"

  406. 		"psrlq	$3, %%mm3\n\t"

  407. 		"pand	%2, %%mm0\n\t"

  408. 		"pand	%2, %%mm3\n\t"

  409. 		"psrlq	$5, %%mm1\n\t"

  410. 		"psrlq	$5, %%mm4\n\t"

  411. 		"pand	%%mm6, %%mm1\n\t"

  412. 		"pand	%%mm6, %%mm4\n\t"

  413. 		"psrlq	$8, %%mm2\n\t"

  414. 		"psrlq	$8, %%mm5\n\t"

  415. 		"pand	%%mm7, %%mm2\n\t"

  416. 		"pand	%%mm7, %%mm5\n\t"

  417. 		"por	%%mm1, %%mm0\n\t"

  418. 		"por	%%mm4, %%mm3\n\t"

  419. 		"por	%%mm2, %%mm0\n\t"

  420. 		"por	%%mm5, %%mm3\n\t"

  421. 		"psllq	$16, %%mm3\n\t"

  422. 		"por	%%mm3, %%mm0\n\t"

  423. 		MOVNTQ"	%%mm0, %0\n\t"

  424. 		:"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");

  425. 		d += 4;

  426. 		s += 16;

  427. 	}

  428. #endif

  429. 	__asm __volatile(SFENCE:::"memory");

  430. 	__asm __volatile(EMMS:::"memory");

  431. #endif

  432. 	while(s < end)

  433. 	{

  434. 		register int rgb = *(uint32_t*)s; s += 4;

  435. 		*d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);

  436. 	}

  437. }

  438. 

  439. static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)

  440. {

  441. 	const uint8_t *s = src;

  442. 	const uint8_t *end;

  443. #ifdef HAVE_MMX

  444. 	const uint8_t *mm_end;

  445. #endif

  446. 	uint16_t *d = (uint16_t *)dst;

  447. 	end = s + src_size;

  448. #ifdef HAVE_MMX

  449. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  450. 	__asm __volatile(

  451. 	    "movq	%0, %%mm7\n\t"

  452. 	    "movq	%1, %%mm6\n\t"

  453. 	    ::"m"(red_16mask),"m"(green_16mask));

  454. 	mm_end = end - 15;

  455. 	while(s < mm_end)

  456. 	{

  457. 	    __asm __volatile(

  458. 		PREFETCH" 32%1\n\t"

  459. 		"movd	%1, %%mm0\n\t"

  460. 		"movd	4%1, %%mm3\n\t"

  461. 		"punpckldq 8%1, %%mm0\n\t"

  462. 		"punpckldq 12%1, %%mm3\n\t"

  463. 		"movq	%%mm0, %%mm1\n\t"

  464. 		"movq	%%mm0, %%mm2\n\t"

  465. 		"movq	%%mm3, %%mm4\n\t"

  466. 		"movq	%%mm3, %%mm5\n\t"

  467. 		"psllq	$8, %%mm0\n\t"

  468. 		"psllq	$8, %%mm3\n\t"

  469. 		"pand	%%mm7, %%mm0\n\t"

  470. 		"pand	%%mm7, %%mm3\n\t"

  471. 		"psrlq	$5, %%mm1\n\t"

  472. 		"psrlq	$5, %%mm4\n\t"

  473. 		"pand	%%mm6, %%mm1\n\t"

  474. 		"pand	%%mm6, %%mm4\n\t"

  475. 		"psrlq	$19, %%mm2\n\t"

  476. 		"psrlq	$19, %%mm5\n\t"

  477. 		"pand	%2, %%mm2\n\t"

  478. 		"pand	%2, %%mm5\n\t"

  479. 		"por	%%mm1, %%mm0\n\t"

  480. 		"por	%%mm4, %%mm3\n\t"

  481. 		"por	%%mm2, %%mm0\n\t"

  482. 		"por	%%mm5, %%mm3\n\t"

  483. 		"psllq	$16, %%mm3\n\t"

  484. 		"por	%%mm3, %%mm0\n\t"

  485. 		MOVNTQ"	%%mm0, %0\n\t"

  486. 		:"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");

  487. 		d += 4;

  488. 		s += 16;

  489. 	}

  490. 	__asm __volatile(SFENCE:::"memory");

  491. 	__asm __volatile(EMMS:::"memory");

  492. #endif

  493. 	while(s < end)

  494. 	{

  495. 		register int rgb = *(uint32_t*)s; s += 4;

  496. 		*d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);

  497. 	}

  498. }

  499. 

  500. static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)

  501. {

  502. 	const uint8_t *s = src;

  503. 	const uint8_t *end;

  504. #ifdef HAVE_MMX

  505. 	const uint8_t *mm_end;

  506. #endif

  507. 	uint16_t *d = (uint16_t *)dst;

  508. 	end = s + src_size;

  509. #ifdef HAVE_MMX

  510. 	mm_end = end - 15;

  511. #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)

  512. 	asm volatile(

  513. 		"movq %3, %%mm5			\n\t"

  514. 		"movq %4, %%mm6			\n\t"

  515. 		"movq %5, %%mm7			\n\t"

  516. 		ASMALIGN(4)

  517. 		"1:				\n\t"

  518. 		PREFETCH" 32(%1)		\n\t"

  519. 		"movd	(%1), %%mm0		\n\t"

  520. 		"movd	4(%1), %%mm3		\n\t"

  521. 		"punpckldq 8(%1), %%mm0		\n\t"

  522. 		"punpckldq 12(%1), %%mm3	\n\t"

  523. 		"movq %%mm0, %%mm1		\n\t"

  524. 		"movq %%mm3, %%mm4		\n\t"

  525. 		"pand %%mm6, %%mm0		\n\t"

  526. 		"pand %%mm6, %%mm3		\n\t"

  527. 		"pmaddwd %%mm7, %%mm0		\n\t"

  528. 		"pmaddwd %%mm7, %%mm3		\n\t"

  529. 		"pand %%mm5, %%mm1		\n\t"

  530. 		"pand %%mm5, %%mm4		\n\t"

  531. 		"por %%mm1, %%mm0		\n\t"	

  532. 		"por %%mm4, %%mm3		\n\t"

  533. 		"psrld $6, %%mm0		\n\t"

  534. 		"pslld $10, %%mm3		\n\t"

  535. 		"por %%mm3, %%mm0		\n\t"

  536. 		MOVNTQ"	%%mm0, (%0)		\n\t"

  537. 		"add $16, %1			\n\t"

  538. 		"add $8, %0			\n\t"

  539. 		"cmp %2, %1			\n\t"

  540. 		" jb 1b				\n\t"

  541. 		: "+r" (d), "+r"(s)

  542. 		: "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)

  543. 	);

  544. #else

  545. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  546. 	__asm __volatile(

  547. 	    "movq	%0, %%mm7\n\t"

  548. 	    "movq	%1, %%mm6\n\t"

  549. 	    ::"m"(red_15mask),"m"(green_15mask));

  550. 	while(s < mm_end)

  551. 	{

  552. 	    __asm __volatile(

  553. 		PREFETCH" 32%1\n\t"

  554. 		"movd	%1, %%mm0\n\t"

  555. 		"movd	4%1, %%mm3\n\t"

  556. 		"punpckldq 8%1, %%mm0\n\t"

  557. 		"punpckldq 12%1, %%mm3\n\t"

  558. 		"movq	%%mm0, %%mm1\n\t"

  559. 		"movq	%%mm0, %%mm2\n\t"

  560. 		"movq	%%mm3, %%mm4\n\t"

  561. 		"movq	%%mm3, %%mm5\n\t"

  562. 		"psrlq	$3, %%mm0\n\t"

  563. 		"psrlq	$3, %%mm3\n\t"

  564. 		"pand	%2, %%mm0\n\t"

  565. 		"pand	%2, %%mm3\n\t"

  566. 		"psrlq	$6, %%mm1\n\t"

  567. 		"psrlq	$6, %%mm4\n\t"

  568. 		"pand	%%mm6, %%mm1\n\t"

  569. 		"pand	%%mm6, %%mm4\n\t"

  570. 		"psrlq	$9, %%mm2\n\t"

  571. 		"psrlq	$9, %%mm5\n\t"

  572. 		"pand	%%mm7, %%mm2\n\t"

  573. 		"pand	%%mm7, %%mm5\n\t"

  574. 		"por	%%mm1, %%mm0\n\t"

  575. 		"por	%%mm4, %%mm3\n\t"

  576. 		"por	%%mm2, %%mm0\n\t"

  577. 		"por	%%mm5, %%mm3\n\t"

  578. 		"psllq	$16, %%mm3\n\t"

  579. 		"por	%%mm3, %%mm0\n\t"

  580. 		MOVNTQ"	%%mm0, %0\n\t"

  581. 		:"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");

  582. 		d += 4;

  583. 		s += 16;

  584. 	}

  585. #endif

  586. 	__asm __volatile(SFENCE:::"memory");

  587. 	__asm __volatile(EMMS:::"memory");

  588. #endif

  589. 	while(s < end)

  590. 	{

  591. 		register int rgb = *(uint32_t*)s; s += 4;

  592. 		*d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);

  593. 	}

  594. }

  595. 

  596. static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)

  597. {

  598. 	const uint8_t *s = src;

  599. 	const uint8_t *end;

  600. #ifdef HAVE_MMX

  601. 	const uint8_t *mm_end;

  602. #endif

  603. 	uint16_t *d = (uint16_t *)dst;

  604. 	end = s + src_size;

  605. #ifdef HAVE_MMX

  606. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  607. 	__asm __volatile(

  608. 	    "movq	%0, %%mm7\n\t"

  609. 	    "movq	%1, %%mm6\n\t"

  610. 	    ::"m"(red_15mask),"m"(green_15mask));

  611. 	mm_end = end - 15;

  612. 	while(s < mm_end)

  613. 	{

  614. 	    __asm __volatile(

  615. 		PREFETCH" 32%1\n\t"

  616. 		"movd	%1, %%mm0\n\t"

  617. 		"movd	4%1, %%mm3\n\t"

  618. 		"punpckldq 8%1, %%mm0\n\t"

  619. 		"punpckldq 12%1, %%mm3\n\t"

  620. 		"movq	%%mm0, %%mm1\n\t"

  621. 		"movq	%%mm0, %%mm2\n\t"

  622. 		"movq	%%mm3, %%mm4\n\t"

  623. 		"movq	%%mm3, %%mm5\n\t"

  624. 		"psllq	$7, %%mm0\n\t"

  625. 		"psllq	$7, %%mm3\n\t"

  626. 		"pand	%%mm7, %%mm0\n\t"

  627. 		"pand	%%mm7, %%mm3\n\t"

  628. 		"psrlq	$6, %%mm1\n\t"

  629. 		"psrlq	$6, %%mm4\n\t"

  630. 		"pand	%%mm6, %%mm1\n\t"

  631. 		"pand	%%mm6, %%mm4\n\t"

  632. 		"psrlq	$19, %%mm2\n\t"

  633. 		"psrlq	$19, %%mm5\n\t"

  634. 		"pand	%2, %%mm2\n\t"

  635. 		"pand	%2, %%mm5\n\t"

  636. 		"por	%%mm1, %%mm0\n\t"

  637. 		"por	%%mm4, %%mm3\n\t"

  638. 		"por	%%mm2, %%mm0\n\t"

  639. 		"por	%%mm5, %%mm3\n\t"

  640. 		"psllq	$16, %%mm3\n\t"

  641. 		"por	%%mm3, %%mm0\n\t"

  642. 		MOVNTQ"	%%mm0, %0\n\t"

  643. 		:"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");

  644. 		d += 4;

  645. 		s += 16;

  646. 	}

  647. 	__asm __volatile(SFENCE:::"memory");

  648. 	__asm __volatile(EMMS:::"memory");

  649. #endif

  650. 	while(s < end)

  651. 	{

  652. 		register int rgb = *(uint32_t*)s; s += 4;

  653. 		*d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);

  654. 	}

  655. }

  656. 

  657. static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)

  658. {

  659. 	const uint8_t *s = src;

  660. 	const uint8_t *end;

  661. #ifdef HAVE_MMX

  662. 	const uint8_t *mm_end;

  663. #endif

  664. 	uint16_t *d = (uint16_t *)dst;

  665. 	end = s + src_size;

  666. #ifdef HAVE_MMX

  667. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  668. 	__asm __volatile(

  669. 	    "movq	%0, %%mm7\n\t"

  670. 	    "movq	%1, %%mm6\n\t"

  671. 	    ::"m"(red_16mask),"m"(green_16mask));

  672. 	mm_end = end - 11;

  673. 	while(s < mm_end)

  674. 	{

  675. 	    __asm __volatile(

  676. 		PREFETCH" 32%1\n\t"

  677. 		"movd	%1, %%mm0\n\t"

  678. 		"movd	3%1, %%mm3\n\t"

  679. 		"punpckldq 6%1, %%mm0\n\t"

  680. 		"punpckldq 9%1, %%mm3\n\t"

  681. 		"movq	%%mm0, %%mm1\n\t"

  682. 		"movq	%%mm0, %%mm2\n\t"

  683. 		"movq	%%mm3, %%mm4\n\t"

  684. 		"movq	%%mm3, %%mm5\n\t"

  685. 		"psrlq	$3, %%mm0\n\t"

  686. 		"psrlq	$3, %%mm3\n\t"

  687. 		"pand	%2, %%mm0\n\t"

  688. 		"pand	%2, %%mm3\n\t"

  689. 		"psrlq	$5, %%mm1\n\t"

  690. 		"psrlq	$5, %%mm4\n\t"

  691. 		"pand	%%mm6, %%mm1\n\t"

  692. 		"pand	%%mm6, %%mm4\n\t"

  693. 		"psrlq	$8, %%mm2\n\t"

  694. 		"psrlq	$8, %%mm5\n\t"

  695. 		"pand	%%mm7, %%mm2\n\t"

  696. 		"pand	%%mm7, %%mm5\n\t"

  697. 		"por	%%mm1, %%mm0\n\t"

  698. 		"por	%%mm4, %%mm3\n\t"

  699. 		"por	%%mm2, %%mm0\n\t"

  700. 		"por	%%mm5, %%mm3\n\t"

  701. 		"psllq	$16, %%mm3\n\t"

  702. 		"por	%%mm3, %%mm0\n\t"

  703. 		MOVNTQ"	%%mm0, %0\n\t"

  704. 		:"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");

  705. 		d += 4;

  706. 		s += 12;

  707. 	}

  708. 	__asm __volatile(SFENCE:::"memory");

  709. 	__asm __volatile(EMMS:::"memory");

  710. #endif

  711. 	while(s < end)

  712. 	{

  713. 		const int b= *s++;

  714. 		const int g= *s++;

  715. 		const int r= *s++;

  716. 		*d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);

  717. 	}

  718. }

  719. 

  720. static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)

  721. {

  722. 	const uint8_t *s = src;

  723. 	const uint8_t *end;

  724. #ifdef HAVE_MMX

  725. 	const uint8_t *mm_end;

  726. #endif

  727. 	uint16_t *d = (uint16_t *)dst;

  728. 	end = s + src_size;

  729. #ifdef HAVE_MMX

  730. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  731. 	__asm __volatile(

  732. 	    "movq	%0, %%mm7\n\t"

  733. 	    "movq	%1, %%mm6\n\t"

  734. 	    ::"m"(red_16mask),"m"(green_16mask));

  735. 	mm_end = end - 15;

  736. 	while(s < mm_end)

  737. 	{

  738. 	    __asm __volatile(

  739. 		PREFETCH" 32%1\n\t"

  740. 		"movd	%1, %%mm0\n\t"

  741. 		"movd	3%1, %%mm3\n\t"

  742. 		"punpckldq 6%1, %%mm0\n\t"

  743. 		"punpckldq 9%1, %%mm3\n\t"

  744. 		"movq	%%mm0, %%mm1\n\t"

  745. 		"movq	%%mm0, %%mm2\n\t"

  746. 		"movq	%%mm3, %%mm4\n\t"

  747. 		"movq	%%mm3, %%mm5\n\t"

  748. 		"psllq	$8, %%mm0\n\t"

  749. 		"psllq	$8, %%mm3\n\t"

  750. 		"pand	%%mm7, %%mm0\n\t"

  751. 		"pand	%%mm7, %%mm3\n\t"

  752. 		"psrlq	$5, %%mm1\n\t"

  753. 		"psrlq	$5, %%mm4\n\t"

  754. 		"pand	%%mm6, %%mm1\n\t"

  755. 		"pand	%%mm6, %%mm4\n\t"

  756. 		"psrlq	$19, %%mm2\n\t"

  757. 		"psrlq	$19, %%mm5\n\t"

  758. 		"pand	%2, %%mm2\n\t"

  759. 		"pand	%2, %%mm5\n\t"

  760. 		"por	%%mm1, %%mm0\n\t"

  761. 		"por	%%mm4, %%mm3\n\t"

  762. 		"por	%%mm2, %%mm0\n\t"

  763. 		"por	%%mm5, %%mm3\n\t"

  764. 		"psllq	$16, %%mm3\n\t"

  765. 		"por	%%mm3, %%mm0\n\t"

  766. 		MOVNTQ"	%%mm0, %0\n\t"

  767. 		:"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");

  768. 		d += 4;

  769. 		s += 12;

  770. 	}

  771. 	__asm __volatile(SFENCE:::"memory");

  772. 	__asm __volatile(EMMS:::"memory");

  773. #endif

  774. 	while(s < end)

  775. 	{

  776. 		const int r= *s++;

  777. 		const int g= *s++;

  778. 		const int b= *s++;

  779. 		*d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);

  780. 	}

  781. }

  782. 

  783. static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)

  784. {

  785. 	const uint8_t *s = src;

  786. 	const uint8_t *end;

  787. #ifdef HAVE_MMX

  788. 	const uint8_t *mm_end;

  789. #endif

  790. 	uint16_t *d = (uint16_t *)dst;

  791. 	end = s + src_size;

  792. #ifdef HAVE_MMX

  793. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  794. 	__asm __volatile(

  795. 	    "movq	%0, %%mm7\n\t"

  796. 	    "movq	%1, %%mm6\n\t"

  797. 	    ::"m"(red_15mask),"m"(green_15mask));

  798. 	mm_end = end - 11;

  799. 	while(s < mm_end)

  800. 	{

  801. 	    __asm __volatile(

  802. 		PREFETCH" 32%1\n\t"

  803. 		"movd	%1, %%mm0\n\t"

  804. 		"movd	3%1, %%mm3\n\t"

  805. 		"punpckldq 6%1, %%mm0\n\t"

  806. 		"punpckldq 9%1, %%mm3\n\t"

  807. 		"movq	%%mm0, %%mm1\n\t"

  808. 		"movq	%%mm0, %%mm2\n\t"

  809. 		"movq	%%mm3, %%mm4\n\t"

  810. 		"movq	%%mm3, %%mm5\n\t"

  811. 		"psrlq	$3, %%mm0\n\t"

  812. 		"psrlq	$3, %%mm3\n\t"

  813. 		"pand	%2, %%mm0\n\t"

  814. 		"pand	%2, %%mm3\n\t"

  815. 		"psrlq	$6, %%mm1\n\t"

  816. 		"psrlq	$6, %%mm4\n\t"

  817. 		"pand	%%mm6, %%mm1\n\t"

  818. 		"pand	%%mm6, %%mm4\n\t"

  819. 		"psrlq	$9, %%mm2\n\t"

  820. 		"psrlq	$9, %%mm5\n\t"

  821. 		"pand	%%mm7, %%mm2\n\t"

  822. 		"pand	%%mm7, %%mm5\n\t"

  823. 		"por	%%mm1, %%mm0\n\t"

  824. 		"por	%%mm4, %%mm3\n\t"

  825. 		"por	%%mm2, %%mm0\n\t"

  826. 		"por	%%mm5, %%mm3\n\t"

  827. 		"psllq	$16, %%mm3\n\t"

  828. 		"por	%%mm3, %%mm0\n\t"

  829. 		MOVNTQ"	%%mm0, %0\n\t"

  830. 		:"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");

  831. 		d += 4;

  832. 		s += 12;

  833. 	}

  834. 	__asm __volatile(SFENCE:::"memory");

  835. 	__asm __volatile(EMMS:::"memory");

  836. #endif

  837. 	while(s < end)

  838. 	{

  839. 		const int b= *s++;

  840. 		const int g= *s++;

  841. 		const int r= *s++;

  842. 		*d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);

  843. 	}

  844. }

  845. 

  846. static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)

  847. {

  848. 	const uint8_t *s = src;

  849. 	const uint8_t *end;

  850. #ifdef HAVE_MMX

  851. 	const uint8_t *mm_end;

  852. #endif

  853. 	uint16_t *d = (uint16_t *)dst;

  854. 	end = s + src_size;

  855. #ifdef HAVE_MMX

  856. 	__asm __volatile(PREFETCH"	%0"::"m"(*src):"memory");

  857. 	__asm __volatile(

  858. 	    "movq	%0, %%mm7\n\t"

  859. 	    "movq	%1, %%mm6\n\t"

  860. 	    ::"m"(red_15mask),"m"(green_15mask));

  861. 	mm_end = end - 15;

  862. 	while(s < mm_end)

  863. 	{

  864. 	    __asm __volatile(

  865. 		PREFETCH" 32%1\n\t"

  866. 		"movd	%1, %%mm0\n\t"

  867. 		"movd	3%1, %%mm3\n\t"

  868. 		"punpckldq 6%1, %%mm0\n\t"

  869. 		"punpckldq 9%1, %%mm3\n\t"

  870. 		"movq	%%mm0, %%mm1\n\t"

  871. 		"movq	%%mm0, %%mm2\n\t"

  872. 		"movq	%%mm3, %%mm4\n\t"

  873. 		"movq	%%mm3, %%mm5\n\t"

  874. 		"psllq	$7, %%mm0\n\t"

  875. 		"psllq	$7, %%mm3\n\t"

  876. 		"pand	%%mm7, %%mm0\n\t"

  877. 		"pand	%%mm7, %%mm3\n\t"

  878. 		"psrlq	$6, %%mm1\n\t"

  879. 		"psrlq	$6, %%mm4\n\t"

  880. 		"pand	%%mm6, %%mm1\n\t"

  881. 		"pand	%%mm6, %%mm4\n\t"

  882. 		"psrlq	$19, %%mm2\n\t"

  883. 		"psrlq	$19, %%mm5\n\t"

  884. 		"pand	%2, %%mm2\n\t"

  885. 		"pand	%2, %%mm5\n\t"

  886. 		"por	%%mm1, %%mm0\n\t"

  887. 		"por	%%mm4, %%mm3\n\t"

  888. 		"por	%%mm2, %%mm0\n\t"

  889. 		"por	%%mm5, %%mm3\n\t"

  890. 		"psllq	$16, %%mm3\n\t"

  891. 		"por	%%mm3, %%mm0\n\t"

  892. 		MOVNTQ"	%%mm0, %0\n\t"

  893. 		:"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");

  894. 		d += 4;

  895. 		s += 12;

  896. 	}

  897. 	__asm __volatile(SFENCE:::"memory");

  898. 	__asm __volatile(EMMS:::"memory");

  899. #endif

  900. 	while(s < end)

  901. 	{

  902. 		const int r= *s++;

  903. 		const int g= *s++;

  904. 		const int b= *s++;

  905. 		*d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);

  906. 	}

  907. }

  908. 

  909. /*

  910.   I use here less accurate approximation by simply

  911.  left-shifting the input

  912.   value and filling the low order bits with

  913.  zeroes. This method improves png's

  914.   compression but this scheme cannot reproduce white exactly, since it does not

  915.   generate an all-ones maximum value; the net effect is to darken the

  916.   image slightly.

  917. 

  918.   The better method should be "left bit replication":

  919. 

  920.    4 3 2 1 0

  921.    ---------

  922.    1 1 0 1 1

  923. 

  924.    7 6 5 4 3  2 1 0

  925.    ----------------

  926.    1 1 0 1 1  1 1 0

  927.    |=======|  |===|

  928.        |      Leftmost Bits Repeated to Fill Open Bits

  929.        |

  930.    Original Bits

  931. */

  932. static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)

  933. {

  934. 	const uint16_t *end;

  935. #ifdef HAVE_MMX

  936. 	const uint16_t *mm_end;

  937. #endif

  938. 	uint8_t *d = (uint8_t *)dst;

  939. 	const uint16_t *s = (uint16_t *)src;

  940. 	end = s + src_size/2;

  941. #ifdef HAVE_MMX

  942. 	__asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  943. 	mm_end = end - 7;

  944. 	while(s < mm_end)

  945. 	{

  946. 	    __asm __volatile(

  947. 		PREFETCH" 32%1\n\t"

  948. 		"movq	%1, %%mm0\n\t"

  949. 		"movq	%1, %%mm1\n\t"

  950. 		"movq	%1, %%mm2\n\t"

  951. 		"pand	%2, %%mm0\n\t"

  952. 		"pand	%3, %%mm1\n\t"

  953. 		"pand	%4, %%mm2\n\t"

  954. 		"psllq	$3, %%mm0\n\t"

  955. 		"psrlq	$2, %%mm1\n\t"

  956. 		"psrlq	$7, %%mm2\n\t"

  957. 		"movq	%%mm0, %%mm3\n\t"

  958. 		"movq	%%mm1, %%mm4\n\t"

  959. 		"movq	%%mm2, %%mm5\n\t"

  960. 		"punpcklwd %5, %%mm0\n\t"

  961. 		"punpcklwd %5, %%mm1\n\t"

  962. 		"punpcklwd %5, %%mm2\n\t"

  963. 		"punpckhwd %5, %%mm3\n\t"

  964. 		"punpckhwd %5, %%mm4\n\t"

  965. 		"punpckhwd %5, %%mm5\n\t"

  966. 		"psllq	$8, %%mm1\n\t"

  967. 		"psllq	$16, %%mm2\n\t"

  968. 		"por	%%mm1, %%mm0\n\t"

  969. 		"por	%%mm2, %%mm0\n\t"

  970. 		"psllq	$8, %%mm4\n\t"

  971. 		"psllq	$16, %%mm5\n\t"

  972. 		"por	%%mm4, %%mm3\n\t"

  973. 		"por	%%mm5, %%mm3\n\t"

  974. 

  975. 		"movq	%%mm0, %%mm6\n\t"

  976. 		"movq	%%mm3, %%mm7\n\t"

  977. 		

  978. 		"movq	8%1, %%mm0\n\t"

  979. 		"movq	8%1, %%mm1\n\t"

  980. 		"movq	8%1, %%mm2\n\t"

  981. 		"pand	%2, %%mm0\n\t"

  982. 		"pand	%3, %%mm1\n\t"

  983. 		"pand	%4, %%mm2\n\t"

  984. 		"psllq	$3, %%mm0\n\t"

  985. 		"psrlq	$2, %%mm1\n\t"

  986. 		"psrlq	$7, %%mm2\n\t"

  987. 		"movq	%%mm0, %%mm3\n\t"

  988. 		"movq	%%mm1, %%mm4\n\t"

  989. 		"movq	%%mm2, %%mm5\n\t"

  990. 		"punpcklwd %5, %%mm0\n\t"

  991. 		"punpcklwd %5, %%mm1\n\t"

  992. 		"punpcklwd %5, %%mm2\n\t"

  993. 		"punpckhwd %5, %%mm3\n\t"

  994. 		"punpckhwd %5, %%mm4\n\t"

  995. 		"punpckhwd %5, %%mm5\n\t"

  996. 		"psllq	$8, %%mm1\n\t"

  997. 		"psllq	$16, %%mm2\n\t"

  998. 		"por	%%mm1, %%mm0\n\t"

  999. 		"por	%%mm2, %%mm0\n\t"

  1000. 		"psllq	$8, %%mm4\n\t"

  1001. 		"psllq	$16, %%mm5\n\t"

  1002. 		"por	%%mm4, %%mm3\n\t"

  1003. 		"por	%%mm5, %%mm3\n\t"

  1004. 

  1005. 		:"=m"(*d)

  1006. 		:"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)

  1007. 		:"memory");

  1008. 	    /* Borrowed 32 to 24 */

  1009. 	    __asm __volatile(

  1010. 		"movq	%%mm0, %%mm4\n\t"

  1011. 		"movq	%%mm3, %%mm5\n\t"

  1012. 		"movq	%%mm6, %%mm0\n\t"

  1013. 		"movq	%%mm7, %%mm1\n\t"

  1014. 		

  1015. 		"movq	%%mm4, %%mm6\n\t"

  1016. 		"movq	%%mm5, %%mm7\n\t"

  1017. 		"movq	%%mm0, %%mm2\n\t"

  1018. 		"movq	%%mm1, %%mm3\n\t"

  1019. 

  1020. 		"psrlq	$8, %%mm2\n\t"

  1021. 		"psrlq	$8, %%mm3\n\t"

  1022. 		"psrlq	$8, %%mm6\n\t"

  1023. 		"psrlq	$8, %%mm7\n\t"

  1024. 		"pand	%2, %%mm0\n\t"

  1025. 		"pand	%2, %%mm1\n\t"

  1026. 		"pand	%2, %%mm4\n\t"

  1027. 		"pand	%2, %%mm5\n\t"

  1028. 		"pand	%3, %%mm2\n\t"

  1029. 		"pand	%3, %%mm3\n\t"

  1030. 		"pand	%3, %%mm6\n\t"

  1031. 		"pand	%3, %%mm7\n\t"

  1032. 		"por	%%mm2, %%mm0\n\t"

  1033. 		"por	%%mm3, %%mm1\n\t"

  1034. 		"por	%%mm6, %%mm4\n\t"

  1035. 		"por	%%mm7, %%mm5\n\t"

  1036. 

  1037. 		"movq	%%mm1, %%mm2\n\t"

  1038. 		"movq	%%mm4, %%mm3\n\t"

  1039. 		"psllq	$48, %%mm2\n\t"

  1040. 		"psllq	$32, %%mm3\n\t"

  1041. 		"pand	%4, %%mm2\n\t"

  1042. 		"pand	%5, %%mm3\n\t"

  1043. 		"por	%%mm2, %%mm0\n\t"

  1044. 		"psrlq	$16, %%mm1\n\t"

  1045. 		"psrlq	$32, %%mm4\n\t"

  1046. 		"psllq	$16, %%mm5\n\t"

  1047. 		"por	%%mm3, %%mm1\n\t"

  1048. 		"pand	%6, %%mm5\n\t"

  1049. 		"por	%%mm5, %%mm4\n\t"

  1050. 

  1051. 		MOVNTQ"	%%mm0, %0\n\t"

  1052. 		MOVNTQ"	%%mm1, 8%0\n\t"

  1053. 		MOVNTQ"	%%mm4, 16%0"

  1054. 

  1055. 		:"=m"(*d)

  1056. 		:"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)

  1057. 		:"memory");

  1058. 		d += 24;

  1059. 		s += 8;

  1060. 	}

  1061. 	__asm __volatile(SFENCE:::"memory");

  1062. 	__asm __volatile(EMMS:::"memory");

  1063. #endif

  1064. 	while(s < end)

  1065. 	{

  1066. 		register uint16_t bgr;

  1067. 		bgr = *s++;

  1068. 		*d++ = (bgr&0x1F)<<3;

  1069. 		*d++ = (bgr&0x3E0)>>2;

  1070. 		*d++ = (bgr&0x7C00)>>7;

  1071. 	}

  1072. }

  1073. 

  1074. static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)

  1075. {

  1076. 	const uint16_t *end;

  1077. #ifdef HAVE_MMX

  1078. 	const uint16_t *mm_end;

  1079. #endif

  1080. 	uint8_t *d = (uint8_t *)dst;

  1081. 	const uint16_t *s = (const uint16_t *)src;

  1082. 	end = s + src_size/2;

  1083. #ifdef HAVE_MMX

  1084. 	__asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  1085. 	mm_end = end - 7;

  1086. 	while(s < mm_end)

  1087. 	{

  1088. 	    __asm __volatile(

  1089. 		PREFETCH" 32%1\n\t"

  1090. 		"movq	%1, %%mm0\n\t"

  1091. 		"movq	%1, %%mm1\n\t"

  1092. 		"movq	%1, %%mm2\n\t"

  1093. 		"pand	%2, %%mm0\n\t"

  1094. 		"pand	%3, %%mm1\n\t"

  1095. 		"pand	%4, %%mm2\n\t"

  1096. 		"psllq	$3, %%mm0\n\t"

  1097. 		"psrlq	$3, %%mm1\n\t"

  1098. 		"psrlq	$8, %%mm2\n\t"

  1099. 		"movq	%%mm0, %%mm3\n\t"

  1100. 		"movq	%%mm1, %%mm4\n\t"

  1101. 		"movq	%%mm2, %%mm5\n\t"

  1102. 		"punpcklwd %5, %%mm0\n\t"

  1103. 		"punpcklwd %5, %%mm1\n\t"

  1104. 		"punpcklwd %5, %%mm2\n\t"

  1105. 		"punpckhwd %5, %%mm3\n\t"

  1106. 		"punpckhwd %5, %%mm4\n\t"

  1107. 		"punpckhwd %5, %%mm5\n\t"

  1108. 		"psllq	$8, %%mm1\n\t"

  1109. 		"psllq	$16, %%mm2\n\t"

  1110. 		"por	%%mm1, %%mm0\n\t"

  1111. 		"por	%%mm2, %%mm0\n\t"

  1112. 		"psllq	$8, %%mm4\n\t"

  1113. 		"psllq	$16, %%mm5\n\t"

  1114. 		"por	%%mm4, %%mm3\n\t"

  1115. 		"por	%%mm5, %%mm3\n\t"

  1116. 		

  1117. 		"movq	%%mm0, %%mm6\n\t"

  1118. 		"movq	%%mm3, %%mm7\n\t"

  1119. 

  1120. 		"movq	8%1, %%mm0\n\t"

  1121. 		"movq	8%1, %%mm1\n\t"

  1122. 		"movq	8%1, %%mm2\n\t"

  1123. 		"pand	%2, %%mm0\n\t"

  1124. 		"pand	%3, %%mm1\n\t"

  1125. 		"pand	%4, %%mm2\n\t"

  1126. 		"psllq	$3, %%mm0\n\t"

  1127. 		"psrlq	$3, %%mm1\n\t"

  1128. 		"psrlq	$8, %%mm2\n\t"

  1129. 		"movq	%%mm0, %%mm3\n\t"

  1130. 		"movq	%%mm1, %%mm4\n\t"

  1131. 		"movq	%%mm2, %%mm5\n\t"

  1132. 		"punpcklwd %5, %%mm0\n\t"

  1133. 		"punpcklwd %5, %%mm1\n\t"

  1134. 		"punpcklwd %5, %%mm2\n\t"

  1135. 		"punpckhwd %5, %%mm3\n\t"

  1136. 		"punpckhwd %5, %%mm4\n\t"

  1137. 		"punpckhwd %5, %%mm5\n\t"

  1138. 		"psllq	$8, %%mm1\n\t"

  1139. 		"psllq	$16, %%mm2\n\t"

  1140. 		"por	%%mm1, %%mm0\n\t"

  1141. 		"por	%%mm2, %%mm0\n\t"

  1142. 		"psllq	$8, %%mm4\n\t"

  1143. 		"psllq	$16, %%mm5\n\t"

  1144. 		"por	%%mm4, %%mm3\n\t"

  1145. 		"por	%%mm5, %%mm3\n\t"

  1146. 		:"=m"(*d)

  1147. 		:"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)		

  1148. 		:"memory");

  1149. 	    /* Borrowed 32 to 24 */

  1150. 	    __asm __volatile(

  1151. 		"movq	%%mm0, %%mm4\n\t"

  1152. 		"movq	%%mm3, %%mm5\n\t"

  1153. 		"movq	%%mm6, %%mm0\n\t"

  1154. 		"movq	%%mm7, %%mm1\n\t"

  1155. 		

  1156. 		"movq	%%mm4, %%mm6\n\t"

  1157. 		"movq	%%mm5, %%mm7\n\t"

  1158. 		"movq	%%mm0, %%mm2\n\t"

  1159. 		"movq	%%mm1, %%mm3\n\t"

  1160. 

  1161. 		"psrlq	$8, %%mm2\n\t"

  1162. 		"psrlq	$8, %%mm3\n\t"

  1163. 		"psrlq	$8, %%mm6\n\t"

  1164. 		"psrlq	$8, %%mm7\n\t"

  1165. 		"pand	%2, %%mm0\n\t"

  1166. 		"pand	%2, %%mm1\n\t"

  1167. 		"pand	%2, %%mm4\n\t"

  1168. 		"pand	%2, %%mm5\n\t"

  1169. 		"pand	%3, %%mm2\n\t"

  1170. 		"pand	%3, %%mm3\n\t"

  1171. 		"pand	%3, %%mm6\n\t"

  1172. 		"pand	%3, %%mm7\n\t"

  1173. 		"por	%%mm2, %%mm0\n\t"

  1174. 		"por	%%mm3, %%mm1\n\t"

  1175. 		"por	%%mm6, %%mm4\n\t"

  1176. 		"por	%%mm7, %%mm5\n\t"

  1177. 

  1178. 		"movq	%%mm1, %%mm2\n\t"

  1179. 		"movq	%%mm4, %%mm3\n\t"

  1180. 		"psllq	$48, %%mm2\n\t"

  1181. 		"psllq	$32, %%mm3\n\t"

  1182. 		"pand	%4, %%mm2\n\t"

  1183. 		"pand	%5, %%mm3\n\t"

  1184. 		"por	%%mm2, %%mm0\n\t"

  1185. 		"psrlq	$16, %%mm1\n\t"

  1186. 		"psrlq	$32, %%mm4\n\t"

  1187. 		"psllq	$16, %%mm5\n\t"

  1188. 		"por	%%mm3, %%mm1\n\t"

  1189. 		"pand	%6, %%mm5\n\t"

  1190. 		"por	%%mm5, %%mm4\n\t"

  1191. 

  1192. 		MOVNTQ"	%%mm0, %0\n\t"

  1193. 		MOVNTQ"	%%mm1, 8%0\n\t"

  1194. 		MOVNTQ"	%%mm4, 16%0"

  1195. 

  1196. 		:"=m"(*d)

  1197. 		:"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)

  1198. 		:"memory");

  1199. 		d += 24;

  1200. 		s += 8;

  1201. 	}

  1202. 	__asm __volatile(SFENCE:::"memory");

  1203. 	__asm __volatile(EMMS:::"memory");

  1204. #endif

  1205. 	while(s < end)

  1206. 	{

  1207. 		register uint16_t bgr;

  1208. 		bgr = *s++;

  1209. 		*d++ = (bgr&0x1F)<<3;

  1210. 		*d++ = (bgr&0x7E0)>>3;

  1211. 		*d++ = (bgr&0xF800)>>8;

  1212. 	}

  1213. }

  1214. 

  1215. static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)

  1216. {

  1217. 	const uint16_t *end;

  1218. #ifdef HAVE_MMX

  1219. 	const uint16_t *mm_end;

  1220. #endif

  1221. 	uint8_t *d = (uint8_t *)dst;

  1222. 	const uint16_t *s = (const uint16_t *)src;

  1223. 	end = s + src_size/2;

  1224. #ifdef HAVE_MMX

  1225. 	__asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  1226. 	__asm __volatile("pxor	%%mm7,%%mm7\n\t":::"memory");

  1227. 	mm_end = end - 3;

  1228. 	while(s < mm_end)

  1229. 	{

  1230. 	    __asm __volatile(

  1231. 		PREFETCH" 32%1\n\t"

  1232. 		"movq	%1, %%mm0\n\t"

  1233. 		"movq	%1, %%mm1\n\t"

  1234. 		"movq	%1, %%mm2\n\t"

  1235. 		"pand	%2, %%mm0\n\t"

  1236. 		"pand	%3, %%mm1\n\t"

  1237. 		"pand	%4, %%mm2\n\t"

  1238. 		"psllq	$3, %%mm0\n\t"

  1239. 		"psrlq	$2, %%mm1\n\t"

  1240. 		"psrlq	$7, %%mm2\n\t"

  1241. 		"movq	%%mm0, %%mm3\n\t"

  1242. 		"movq	%%mm1, %%mm4\n\t"

  1243. 		"movq	%%mm2, %%mm5\n\t"

  1244. 		"punpcklwd %%mm7, %%mm0\n\t"

  1245. 		"punpcklwd %%mm7, %%mm1\n\t"

  1246. 		"punpcklwd %%mm7, %%mm2\n\t"

  1247. 		"punpckhwd %%mm7, %%mm3\n\t"

  1248. 		"punpckhwd %%mm7, %%mm4\n\t"

  1249. 		"punpckhwd %%mm7, %%mm5\n\t"

  1250. 		"psllq	$8, %%mm1\n\t"

  1251. 		"psllq	$16, %%mm2\n\t"

  1252. 		"por	%%mm1, %%mm0\n\t"

  1253. 		"por	%%mm2, %%mm0\n\t"

  1254. 		"psllq	$8, %%mm4\n\t"

  1255. 		"psllq	$16, %%mm5\n\t"

  1256. 		"por	%%mm4, %%mm3\n\t"

  1257. 		"por	%%mm5, %%mm3\n\t"

  1258. 		MOVNTQ"	%%mm0, %0\n\t"

  1259. 		MOVNTQ"	%%mm3, 8%0\n\t"

  1260. 		:"=m"(*d)

  1261. 		:"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)

  1262. 		:"memory");

  1263. 		d += 16;

  1264. 		s += 4;

  1265. 	}

  1266. 	__asm __volatile(SFENCE:::"memory");

  1267. 	__asm __volatile(EMMS:::"memory");

  1268. #endif

  1269. 	while(s < end)

  1270. 	{

  1271. #if 0 //slightly slower on athlon

  1272. 		int bgr= *s++;

  1273. 		*((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);

  1274. #else

  1275. 		register uint16_t bgr;

  1276. 		bgr = *s++;

  1277. #ifdef WORDS_BIGENDIAN

  1278. 		*d++ = 0;

  1279. 		*d++ = (bgr&0x7C00)>>7;

  1280. 		*d++ = (bgr&0x3E0)>>2;

  1281. 		*d++ = (bgr&0x1F)<<3;

  1282. #else

  1283. 		*d++ = (bgr&0x1F)<<3;

  1284. 		*d++ = (bgr&0x3E0)>>2;

  1285. 		*d++ = (bgr&0x7C00)>>7;

  1286. 		*d++ = 0;

  1287. #endif

  1288. 

  1289. #endif

  1290. 	}

  1291. }

  1292. 

  1293. static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)

  1294. {

  1295. 	const uint16_t *end;

  1296. #ifdef HAVE_MMX

  1297. 	const uint16_t *mm_end;

  1298. #endif

  1299. 	uint8_t *d = (uint8_t *)dst;

  1300. 	const uint16_t *s = (uint16_t *)src;

  1301. 	end = s + src_size/2;

  1302. #ifdef HAVE_MMX

  1303. 	__asm __volatile(PREFETCH"	%0"::"m"(*s):"memory");

  1304. 	__asm __volatile("pxor	%%mm7,%%mm7\n\t":::"memory");

  1305. 	mm_end = end - 3;

  1306. 	while(s < mm_end)

  1307. 	{

  1308. 	    __asm __volatile(

  1309. 		PREFETCH" 32%1\n\t"

  1310. 		"movq	%1, %%mm0\n\t"

  1311. 		"movq	%1, %%mm1\n\t"

  1312. 		"movq	%1, %%mm2\n\t"

  1313. 		"pand	%2, %%mm0\n\t"

  1314. 		"pand	%3, %%mm1\n\t"

  1315. 		"pand	%4, %%mm2\n\t"

  1316. 		"psllq	$3, %%mm0\n\t"

  1317. 		"psrlq	$3, %%mm1\n\t"

  1318. 		"psrlq	$8, %%mm2\n\t"

  1319. 		"movq	%%mm0, %%mm3\n\t"

  1320. 		"movq	%%mm1, %%mm4\n\t"

  1321. 		"movq	%%mm2, %%mm5\n\t"

  1322. 		"punpcklwd %%mm7, %%mm0\n\t"

  1323. 		"punpcklwd %%mm7, %%mm1\n\t"

  1324. 		"punpcklwd %%mm7, %%mm2\n\t"

  1325. 		"punpckhwd %%mm7, %%mm3\n\t"

  1326. 		"punpckhwd %%mm7, %%mm4\n\t"

  1327. 		"punpckhwd %%mm7, %%mm5\n\t"

  1328. 		"psllq	$8, %%mm1\n\t"

  1329. 		"psllq	$16, %%mm2\n\t"

  1330. 		"por	%%mm1, %%mm0\n\t"

  1331. 		"por	%%mm2, %%mm0\n\t"

  1332. 		"psllq	$8, %%mm4\n\t"

  1333. 		"psllq	$16, %%mm5\n\t"

  1334. 		"por	%%mm4, %%mm3\n\t"

  1335. 		"por	%%mm5, %%mm3\n\t"

  1336. 		MOVNTQ"	%%mm0, %0\n\t"

  1337. 		MOVNTQ"	%%mm3, 8%0\n\t"

  1338. 		:"=m"(*d)

  1339. 		:"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)

  1340. 		:"memory");

  1341. 		d += 16;

  1342. 		s += 4;

  1343. 	}

  1344. 	__asm __volatile(SFENCE:::"memory");

  1345. 	__asm __volatile(EMMS:::"memory");

  1346. #endif

  1347. 	while(s < end)

  1348. 	{

  1349. 		register uint16_t bgr;

  1350. 		bgr = *s++;

  1351. #ifdef WORDS_BIGENDIAN

  1352. 		*d++ = 0;

  1353. 		*d++ = (bgr&0xF800)>>8;

  1354. 		*d++ = (bgr&0x7E0)>>3;

  1355. 		*d++ = (bgr&0x1F)<<3;

  1356. #else

  1357. 		*d++ = (bgr&0x1F)<<3;

  1358. 		*d++ = (bgr&0x7E0)>>3;

  1359. 		*d++ = (bgr&0xF800)>>8;

  1360. 		*d++ = 0;

  1361. #endif

  1362. 	}

  1363. }

  1364. 

  1365. static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)

  1366. {

  1367. #ifdef HAVE_MMX

  1368. /* TODO: unroll this loop */

  1369. 	asm volatile (

  1370. 		"xor %%"REG_a", %%"REG_a"	\n\t"

  1371. 		ASMALIGN(4)

  1372. 		"1:				\n\t"

  1373. 		PREFETCH" 32(%0, %%"REG_a")	\n\t"

  1374. 		"movq (%0, %%"REG_a"), %%mm0	\n\t"

  1375. 		"movq %%mm0, %%mm1		\n\t"

  1376. 		"movq %%mm0, %%mm2		\n\t"

  1377. 		"pslld $16, %%mm0		\n\t"

  1378. 		"psrld $16, %%mm1		\n\t"

  1379. 		"pand "MANGLE(mask32r)", %%mm0	\n\t"

  1380. 		"pand "MANGLE(mask32g)", %%mm2	\n\t"

  1381. 		"pand "MANGLE(mask32b)", %%mm1	\n\t"

  1382. 		"por %%mm0, %%mm2		\n\t"

  1383. 		"por %%mm1, %%mm2		\n\t"

  1384. 		MOVNTQ" %%mm2, (%1, %%"REG_a")	\n\t"

  1385. 		"add $8, %%"REG_a"		\n\t"

  1386. 		"cmp %2, %%"REG_a"		\n\t"

  1387. 		" jb 1b				\n\t"

  1388. 		:: "r" (src), "r"(dst), "r" (src_size-7)

  1389. 		: "%"REG_a

  1390. 	);

  1391. 

  1392. 	__asm __volatile(SFENCE:::"memory");

  1393. 	__asm __volatile(EMMS:::"memory");

  1394. #else

  1395. 	unsigned i;

  1396. 	unsigned num_pixels = src_size >> 2;

  1397. 	for(i=0; i<num_pixels; i++)

  1398. 	{

  1399. #ifdef WORDS_BIGENDIAN  

  1400. 	  dst[4*i + 1] = src[4*i + 3];

  1401. 	  dst[4*i + 2] = src[4*i + 2];

  1402. 	  dst[4*i + 3] = src[4*i + 1];

  1403. #else

  1404. 	  dst[4*i + 0] = src[4*i + 2];

  1405. 	  dst[4*i + 1] = src[4*i + 1];

  1406. 	  dst[4*i + 2] = src[4*i + 0];

  1407. #endif

  1408. 	}

  1409. #endif

  1410. }

  1411. 

  1412. static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)

  1413. {

  1414. 	unsigned i;

  1415. #ifdef HAVE_MMX

  1416. 	long mmx_size= 23 - src_size;

  1417. 	asm volatile (

  1418. 		"movq "MANGLE(mask24r)", %%mm5	\n\t"

  1419. 		"movq "MANGLE(mask24g)", %%mm6	\n\t"

  1420. 		"movq "MANGLE(mask24b)", %%mm7	\n\t"

  1421. 		ASMALIGN(4)

  1422. 		"1:				\n\t"

  1423. 		PREFETCH" 32(%1, %%"REG_a")	\n\t"

  1424. 		"movq   (%1, %%"REG_a"), %%mm0	\n\t" // BGR BGR BG

  1425. 		"movq   (%1, %%"REG_a"), %%mm1	\n\t" // BGR BGR BG

  1426. 		"movq  2(%1, %%"REG_a"), %%mm2	\n\t" // R BGR BGR B

  1427. 		"psllq $16, %%mm0		\n\t" // 00 BGR BGR

  1428. 		"pand %%mm5, %%mm0		\n\t"

  1429. 		"pand %%mm6, %%mm1		\n\t"

  1430. 		"pand %%mm7, %%mm2		\n\t"

  1431. 		"por %%mm0, %%mm1		\n\t"

  1432. 		"por %%mm2, %%mm1		\n\t"                

  1433. 		"movq  6(%1, %%"REG_a"), %%mm0	\n\t" // BGR BGR BG

  1434. 		MOVNTQ" %%mm1,   (%2, %%"REG_a")\n\t" // RGB RGB RG

  1435. 		"movq  8(%1, %%"REG_a"), %%mm1	\n\t" // R BGR BGR B

  1436. 		"movq 10(%1, %%"REG_a"), %%mm2	\n\t" // GR BGR BGR

  1437. 		"pand %%mm7, %%mm0		\n\t"

  1438. 		"pand %%mm5, %%mm1		\n\t"

  1439. 		"pand %%mm6, %%mm2		\n\t"

  1440. 		"por %%mm0, %%mm1		\n\t"

  1441. 		"por %%mm2, %%mm1		\n\t"                

  1442. 		"movq 14(%1, %%"REG_a"), %%mm0	\n\t" // R BGR BGR B

  1443. 		MOVNTQ" %%mm1,  8(%2, %%"REG_a")\n\t" // B RGB RGB R

  1444. 		"movq 16(%1, %%"REG_a"), %%mm1	\n\t" // GR BGR BGR

  1445. 		"movq 18(%1, %%"REG_a"), %%mm2	\n\t" // BGR BGR BG

  1446. 		"pand %%mm6, %%mm0		\n\t"

  1447. 		"pand %%mm7, %%mm1		\n\t"

  1448. 		"pand %%mm5, %%mm2		\n\t"

  1449. 		"por %%mm0, %%mm1		\n\t"

  1450. 		"por %%mm2, %%mm1		\n\t"                

  1451. 		MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"

  1452. 		"add $24, %%"REG_a"		\n\t"

  1453. 		" js 1b				\n\t"

  1454. 		: "+a" (mmx_size)

  1455. 		: "r" (src-mmx_size), "r"(dst-mmx_size)

  1456. 	);

  1457. 

  1458. 	__asm __volatile(SFENCE:::"memory");

  1459. 	__asm __volatile(EMMS:::"memory");

  1460. 

  1461. 	if(mmx_size==23) return; //finihsed, was multiple of 8

  1462. 

  1463. 	src+= src_size;

  1464. 	dst+= src_size;

  1465. 	src_size= 23-mmx_size;

  1466. 	src-= src_size;

  1467. 	dst-= src_size;

  1468. #endif

  1469. 	for(i=0; i<src_size; i+=3)

  1470. 	{

  1471. 		register uint8_t x;

  1472. 		x          = src[i + 2];

  1473. 		dst[i + 1] = src[i + 1];

  1474. 		dst[i + 2] = src[i + 0];

  1475. 		dst[i + 0] = x;

  1476. 	}

  1477. }

  1478. 

  1479. static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1480. 	long width, long height,

  1481. 	long lumStride, long chromStride, long dstStride, long vertLumPerChroma)

  1482. {

  1483. 	long y;

  1484. 	const long chromWidth= width>>1;

  1485. 	for(y=0; y<height; y++)

  1486. 	{

  1487. #ifdef HAVE_MMX

  1488. //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)

  1489. 		asm volatile(

  1490. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  1491. 			ASMALIGN(4)

  1492. 			"1:				\n\t"

  1493. 			PREFETCH" 32(%1, %%"REG_a", 2)	\n\t"

  1494. 			PREFETCH" 32(%2, %%"REG_a")	\n\t"

  1495. 			PREFETCH" 32(%3, %%"REG_a")	\n\t"

  1496. 			"movq (%2, %%"REG_a"), %%mm0	\n\t" // U(0)

  1497. 			"movq %%mm0, %%mm2		\n\t" // U(0)

  1498. 			"movq (%3, %%"REG_a"), %%mm1	\n\t" // V(0)

  1499. 			"punpcklbw %%mm1, %%mm0		\n\t" // UVUV UVUV(0)

  1500. 			"punpckhbw %%mm1, %%mm2		\n\t" // UVUV UVUV(8)

  1501. 

  1502. 			"movq (%1, %%"REG_a",2), %%mm3	\n\t" // Y(0)

  1503. 			"movq 8(%1, %%"REG_a",2), %%mm5	\n\t" // Y(8)

  1504. 			"movq %%mm3, %%mm4		\n\t" // Y(0)

  1505. 			"movq %%mm5, %%mm6		\n\t" // Y(8)

  1506. 			"punpcklbw %%mm0, %%mm3		\n\t" // YUYV YUYV(0)

  1507. 			"punpckhbw %%mm0, %%mm4		\n\t" // YUYV YUYV(4)

  1508. 			"punpcklbw %%mm2, %%mm5		\n\t" // YUYV YUYV(8)

  1509. 			"punpckhbw %%mm2, %%mm6		\n\t" // YUYV YUYV(12)

  1510. 

  1511. 			MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"

  1512. 			MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"

  1513. 			MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"

  1514. 			MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"

  1515. 

  1516. 			"add $8, %%"REG_a"		\n\t"

  1517. 			"cmp %4, %%"REG_a"		\n\t"

  1518. 			" jb 1b				\n\t"

  1519. 			::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)

  1520. 			: "%"REG_a

  1521. 		);

  1522. #else

  1523. 

  1524. #if defined ARCH_ALPHA && defined HAVE_MVI

  1525. #define pl2yuy2(n)					\

  1526. 	y1 = yc[n];					\

  1527. 	y2 = yc2[n];					\

  1528. 	u = uc[n];					\

  1529. 	v = vc[n];					\

  1530. 	asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1));	\

  1531. 	asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2));	\

  1532. 	asm("unpkbl %1, %0" : "=r"(u) : "r"(u));	\

  1533. 	asm("unpkbl %1, %0" : "=r"(v) : "r"(v));	\

  1534. 	yuv1 = (u << 8) + (v << 24);			\

  1535. 	yuv2 = yuv1 + y2;				\

  1536. 	yuv1 += y1;					\

  1537. 	qdst[n] = yuv1;					\

  1538. 	qdst2[n] = yuv2;

  1539. 

  1540. 		int i;

  1541. 		uint64_t *qdst = (uint64_t *) dst;

  1542. 		uint64_t *qdst2 = (uint64_t *) (dst + dstStride);

  1543. 		const uint32_t *yc = (uint32_t *) ysrc;

  1544. 		const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);

  1545. 		const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;

  1546. 		for(i = 0; i < chromWidth; i += 8){

  1547. 			uint64_t y1, y2, yuv1, yuv2;

  1548. 			uint64_t u, v;

  1549. 			/* Prefetch */

  1550. 			asm("ldq $31,64(%0)" :: "r"(yc));

  1551. 			asm("ldq $31,64(%0)" :: "r"(yc2));

  1552. 			asm("ldq $31,64(%0)" :: "r"(uc));

  1553. 			asm("ldq $31,64(%0)" :: "r"(vc));

  1554. 

  1555. 			pl2yuy2(0);

  1556. 			pl2yuy2(1);

  1557. 			pl2yuy2(2);

  1558. 			pl2yuy2(3);

  1559. 

  1560. 			yc += 4;

  1561. 			yc2 += 4;

  1562. 			uc += 4;

  1563. 			vc += 4;

  1564. 			qdst += 4;

  1565. 			qdst2 += 4;

  1566. 		}

  1567. 		y++;

  1568. 		ysrc += lumStride;

  1569. 		dst += dstStride;

  1570. 

  1571. #elif __WORDSIZE >= 64

  1572. 		int i;

  1573. 		uint64_t *ldst = (uint64_t *) dst;

  1574. 		const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;

  1575. 		for(i = 0; i < chromWidth; i += 2){

  1576. 			uint64_t k, l;

  1577. 			k = yc[0] + (uc[0] << 8) +

  1578. 			    (yc[1] << 16) + (vc[0] << 24);

  1579. 			l = yc[2] + (uc[1] << 8) +

  1580. 			    (yc[3] << 16) + (vc[1] << 24);

  1581. 			*ldst++ = k + (l << 32);

  1582. 			yc += 4;

  1583. 			uc += 2;

  1584. 			vc += 2;

  1585. 		}

  1586. 

  1587. #else

  1588. 		int i, *idst = (int32_t *) dst;

  1589. 		const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;

  1590. 		for(i = 0; i < chromWidth; i++){

  1591. #ifdef WORDS_BIGENDIAN

  1592. 			*idst++ = (yc[0] << 24)+ (uc[0] << 16) +

  1593. 			    (yc[1] << 8) + (vc[0] << 0);

  1594. #else

  1595. 			*idst++ = yc[0] + (uc[0] << 8) +

  1596. 			    (yc[1] << 16) + (vc[0] << 24);

  1597. #endif

  1598. 			yc += 2;

  1599. 			uc++;

  1600. 			vc++;

  1601. 		}

  1602. #endif

  1603. #endif

  1604. 		if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )

  1605. 		{

  1606. 			usrc += chromStride;

  1607. 			vsrc += chromStride;

  1608. 		}

  1609. 		ysrc += lumStride;

  1610. 		dst += dstStride;

  1611. 	}

  1612. #ifdef HAVE_MMX

  1613. asm(    EMMS" \n\t"

  1614.         SFENCE" \n\t"

  1615.         :::"memory");

  1616. #endif

  1617. }

  1618. 

  1619. /**

  1620.  *

  1621.  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a

  1622.  * problem for anyone then tell me, and ill fix it)

  1623.  */

  1624. static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1625. 	long width, long height,

  1626. 	long lumStride, long chromStride, long dstStride)

  1627. {

  1628. 	//FIXME interpolate chroma

  1629. 	RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);

  1630. }

  1631. 

  1632. static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1633. 	long width, long height,

  1634. 	long lumStride, long chromStride, long dstStride, long vertLumPerChroma)

  1635. {

  1636. 	long y;

  1637. 	const long chromWidth= width>>1;

  1638. 	for(y=0; y<height; y++)

  1639. 	{

  1640. #ifdef HAVE_MMX

  1641. //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)

  1642. 		asm volatile(

  1643. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  1644. 			ASMALIGN(4)

  1645. 			"1:				\n\t"

  1646. 			PREFETCH" 32(%1, %%"REG_a", 2)	\n\t"

  1647. 			PREFETCH" 32(%2, %%"REG_a")	\n\t"

  1648. 			PREFETCH" 32(%3, %%"REG_a")	\n\t"

  1649. 			"movq (%2, %%"REG_a"), %%mm0	\n\t" // U(0)

  1650. 			"movq %%mm0, %%mm2		\n\t" // U(0)

  1651. 			"movq (%3, %%"REG_a"), %%mm1	\n\t" // V(0)

  1652. 			"punpcklbw %%mm1, %%mm0		\n\t" // UVUV UVUV(0)

  1653. 			"punpckhbw %%mm1, %%mm2		\n\t" // UVUV UVUV(8)

  1654. 

  1655. 			"movq (%1, %%"REG_a",2), %%mm3	\n\t" // Y(0)

  1656. 			"movq 8(%1, %%"REG_a",2), %%mm5	\n\t" // Y(8)

  1657. 			"movq %%mm0, %%mm4		\n\t" // Y(0)

  1658. 			"movq %%mm2, %%mm6		\n\t" // Y(8)

  1659. 			"punpcklbw %%mm3, %%mm0		\n\t" // YUYV YUYV(0)

  1660. 			"punpckhbw %%mm3, %%mm4		\n\t" // YUYV YUYV(4)

  1661. 			"punpcklbw %%mm5, %%mm2		\n\t" // YUYV YUYV(8)

  1662. 			"punpckhbw %%mm5, %%mm6		\n\t" // YUYV YUYV(12)

  1663. 

  1664. 			MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"

  1665. 			MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"

  1666. 			MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"

  1667. 			MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"

  1668. 

  1669. 			"add $8, %%"REG_a"		\n\t"

  1670. 			"cmp %4, %%"REG_a"		\n\t"

  1671. 			" jb 1b				\n\t"

  1672. 			::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)

  1673. 			: "%"REG_a

  1674. 		);

  1675. #else

  1676. //FIXME adapt the alpha asm code from yv12->yuy2

  1677. 

  1678. #if __WORDSIZE >= 64

  1679. 		int i;

  1680. 		uint64_t *ldst = (uint64_t *) dst;

  1681. 		const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;

  1682. 		for(i = 0; i < chromWidth; i += 2){

  1683. 			uint64_t k, l;

  1684. 			k = uc[0] + (yc[0] << 8) +

  1685. 			    (vc[0] << 16) + (yc[1] << 24);

  1686. 			l = uc[1] + (yc[2] << 8) +

  1687. 			    (vc[1] << 16) + (yc[3] << 24);

  1688. 			*ldst++ = k + (l << 32);

  1689. 			yc += 4;

  1690. 			uc += 2;

  1691. 			vc += 2;

  1692. 		}

  1693. 

  1694. #else

  1695. 		int i, *idst = (int32_t *) dst;

  1696. 		const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;

  1697. 		for(i = 0; i < chromWidth; i++){

  1698. #ifdef WORDS_BIGENDIAN

  1699. 			*idst++ = (uc[0] << 24)+ (yc[0] << 16) +

  1700. 			    (vc[0] << 8) + (yc[1] << 0);

  1701. #else

  1702. 			*idst++ = uc[0] + (yc[0] << 8) +

  1703. 			    (vc[0] << 16) + (yc[1] << 24);

  1704. #endif

  1705. 			yc += 2;

  1706. 			uc++;

  1707. 			vc++;

  1708. 		}

  1709. #endif

  1710. #endif

  1711. 		if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )

  1712. 		{

  1713. 			usrc += chromStride;

  1714. 			vsrc += chromStride;

  1715. 		}

  1716. 		ysrc += lumStride;

  1717. 		dst += dstStride;

  1718. 	}

  1719. #ifdef HAVE_MMX

  1720. asm(    EMMS" \n\t"

  1721.         SFENCE" \n\t"

  1722.         :::"memory");

  1723. #endif

  1724. }

  1725. 

  1726. /**

  1727.  *

  1728.  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a

  1729.  * problem for anyone then tell me, and ill fix it)

  1730.  */

  1731. static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1732. 	long width, long height,

  1733. 	long lumStride, long chromStride, long dstStride)

  1734. {

  1735. 	//FIXME interpolate chroma

  1736. 	RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);

  1737. }

  1738. 

  1739. /**

  1740.  *

  1741.  * width should be a multiple of 16

  1742.  */

  1743. static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,

  1744. 	long width, long height,

  1745. 	long lumStride, long chromStride, long dstStride)

  1746. {

  1747. 	RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);

  1748. }

  1749. 

  1750. /**

  1751.  *

  1752.  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a

  1753.  * problem for anyone then tell me, and ill fix it)

  1754.  */

  1755. static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1756. 	long width, long height,

  1757. 	long lumStride, long chromStride, long srcStride)

  1758. {

  1759. 	long y;

  1760. 	const long chromWidth= width>>1;

  1761. 	for(y=0; y<height; y+=2)

  1762. 	{

  1763. #ifdef HAVE_MMX

  1764. 		asm volatile(

  1765. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  1766. 			"pcmpeqw %%mm7, %%mm7		\n\t"

  1767. 			"psrlw $8, %%mm7		\n\t" // FF,00,FF,00...

  1768. 			ASMALIGN(4)

  1769. 			"1:				\n\t"

  1770. 			PREFETCH" 64(%0, %%"REG_a", 4)	\n\t"

  1771. 			"movq (%0, %%"REG_a", 4), %%mm0	\n\t" // YUYV YUYV(0)

  1772. 			"movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)

  1773. 			"movq %%mm0, %%mm2		\n\t" // YUYV YUYV(0)

  1774. 			"movq %%mm1, %%mm3		\n\t" // YUYV YUYV(4)

  1775. 			"psrlw $8, %%mm0		\n\t" // U0V0 U0V0(0)

  1776. 			"psrlw $8, %%mm1		\n\t" // U0V0 U0V0(4)

  1777. 			"pand %%mm7, %%mm2		\n\t" // Y0Y0 Y0Y0(0)

  1778. 			"pand %%mm7, %%mm3		\n\t" // Y0Y0 Y0Y0(4)

  1779. 			"packuswb %%mm1, %%mm0		\n\t" // UVUV UVUV(0)

  1780. 			"packuswb %%mm3, %%mm2		\n\t" // YYYY YYYY(0)

  1781. 

  1782. 			MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"

  1783. 

  1784. 			"movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)

  1785. 			"movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)

  1786. 			"movq %%mm1, %%mm3		\n\t" // YUYV YUYV(8)

  1787. 			"movq %%mm2, %%mm4		\n\t" // YUYV YUYV(12)

  1788. 			"psrlw $8, %%mm1		\n\t" // U0V0 U0V0(8)

  1789. 			"psrlw $8, %%mm2		\n\t" // U0V0 U0V0(12)

  1790. 			"pand %%mm7, %%mm3		\n\t" // Y0Y0 Y0Y0(8)

  1791. 			"pand %%mm7, %%mm4		\n\t" // Y0Y0 Y0Y0(12)

  1792. 			"packuswb %%mm2, %%mm1		\n\t" // UVUV UVUV(8)

  1793. 			"packuswb %%mm4, %%mm3		\n\t" // YYYY YYYY(8)

  1794. 

  1795. 			MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"

  1796. 

  1797. 			"movq %%mm0, %%mm2		\n\t" // UVUV UVUV(0)

  1798. 			"movq %%mm1, %%mm3		\n\t" // UVUV UVUV(8)

  1799. 			"psrlw $8, %%mm0		\n\t" // V0V0 V0V0(0)

  1800. 			"psrlw $8, %%mm1		\n\t" // V0V0 V0V0(8)

  1801. 			"pand %%mm7, %%mm2		\n\t" // U0U0 U0U0(0)

  1802. 			"pand %%mm7, %%mm3		\n\t" // U0U0 U0U0(8)

  1803. 			"packuswb %%mm1, %%mm0		\n\t" // VVVV VVVV(0)

  1804. 			"packuswb %%mm3, %%mm2		\n\t" // UUUU UUUU(0)

  1805. 

  1806. 			MOVNTQ" %%mm0, (%3, %%"REG_a")	\n\t"

  1807. 			MOVNTQ" %%mm2, (%2, %%"REG_a")	\n\t"

  1808. 

  1809. 			"add $8, %%"REG_a"		\n\t"

  1810. 			"cmp %4, %%"REG_a"		\n\t"

  1811. 			" jb 1b				\n\t"

  1812. 			::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1813. 			: "memory", "%"REG_a

  1814. 		);

  1815. 

  1816. 		ydst += lumStride;

  1817. 		src  += srcStride;

  1818. 

  1819. 		asm volatile(

  1820. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  1821. 			ASMALIGN(4)

  1822. 			"1:				\n\t"

  1823. 			PREFETCH" 64(%0, %%"REG_a", 4)	\n\t"

  1824. 			"movq (%0, %%"REG_a", 4), %%mm0	\n\t" // YUYV YUYV(0)

  1825. 			"movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)

  1826. 			"movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)

  1827. 			"movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)

  1828. 			"pand %%mm7, %%mm0		\n\t" // Y0Y0 Y0Y0(0)

  1829. 			"pand %%mm7, %%mm1		\n\t" // Y0Y0 Y0Y0(4)

  1830. 			"pand %%mm7, %%mm2		\n\t" // Y0Y0 Y0Y0(8)

  1831. 			"pand %%mm7, %%mm3		\n\t" // Y0Y0 Y0Y0(12)

  1832. 			"packuswb %%mm1, %%mm0		\n\t" // YYYY YYYY(0)

  1833. 			"packuswb %%mm3, %%mm2		\n\t" // YYYY YYYY(8)

  1834. 

  1835. 			MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"

  1836. 			MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"

  1837. 

  1838. 			"add $8, %%"REG_a"		\n\t"

  1839. 			"cmp %4, %%"REG_a"		\n\t"

  1840. 			" jb 1b				\n\t"

  1841. 

  1842. 			::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  1843. 			: "memory", "%"REG_a

  1844. 		);

  1845. #else

  1846. 		long i;

  1847. 		for(i=0; i<chromWidth; i++)

  1848. 		{

  1849. 			ydst[2*i+0] 	= src[4*i+0];

  1850. 			udst[i] 	= src[4*i+1];

  1851. 			ydst[2*i+1] 	= src[4*i+2];

  1852. 			vdst[i] 	= src[4*i+3];

  1853. 		}

  1854. 		ydst += lumStride;

  1855. 		src  += srcStride;

  1856. 

  1857. 		for(i=0; i<chromWidth; i++)

  1858. 		{

  1859. 			ydst[2*i+0] 	= src[4*i+0];

  1860. 			ydst[2*i+1] 	= src[4*i+2];

  1861. 		}

  1862. #endif

  1863. 		udst += chromStride;

  1864. 		vdst += chromStride;

  1865. 		ydst += lumStride;

  1866. 		src  += srcStride;

  1867. 	}

  1868. #ifdef HAVE_MMX

  1869. asm volatile(   EMMS" \n\t"

  1870.         	SFENCE" \n\t"

  1871.         	:::"memory");

  1872. #endif

  1873. }

  1874. 

  1875. static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,

  1876. 	uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1877. 	long width, long height, long lumStride, long chromStride)

  1878. {

  1879. 	/* Y Plane */

  1880. 	memcpy(ydst, ysrc, width*height);

  1881. 

  1882. 	/* XXX: implement upscaling for U,V */

  1883. }

  1884. 

  1885. static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)

  1886. {

  1887. 	long x,y;

  1888. 	

  1889. 	dst[0]= src[0];

  1890.         

  1891. 	// first line

  1892. 	for(x=0; x<srcWidth-1; x++){

  1893. 		dst[2*x+1]= (3*src[x] +   src[x+1])>>2;

  1894. 		dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;

  1895. 	}

  1896. 	dst[2*srcWidth-1]= src[srcWidth-1];

  1897. 	

  1898.         dst+= dstStride;

  1899. 

  1900. 	for(y=1; y<srcHeight; y++){

  1901. #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)

  1902. 		const long mmxSize= srcWidth&~15;

  1903. 		asm volatile(

  1904. 			"mov %4, %%"REG_a"		\n\t"

  1905. 			"1:				\n\t"

  1906. 			"movq (%0, %%"REG_a"), %%mm0	\n\t"

  1907. 			"movq (%1, %%"REG_a"), %%mm1	\n\t"

  1908. 			"movq 1(%0, %%"REG_a"), %%mm2	\n\t"

  1909. 			"movq 1(%1, %%"REG_a"), %%mm3	\n\t"

  1910. 			"movq -1(%0, %%"REG_a"), %%mm4	\n\t"

  1911. 			"movq -1(%1, %%"REG_a"), %%mm5	\n\t"

  1912. 			PAVGB" %%mm0, %%mm5		\n\t"

  1913. 			PAVGB" %%mm0, %%mm3		\n\t"

  1914. 			PAVGB" %%mm0, %%mm5		\n\t"

  1915. 			PAVGB" %%mm0, %%mm3		\n\t"

  1916. 			PAVGB" %%mm1, %%mm4		\n\t"

  1917. 			PAVGB" %%mm1, %%mm2		\n\t"

  1918. 			PAVGB" %%mm1, %%mm4		\n\t"

  1919. 			PAVGB" %%mm1, %%mm2		\n\t"

  1920. 			"movq %%mm5, %%mm7		\n\t"

  1921. 			"movq %%mm4, %%mm6		\n\t"

  1922. 			"punpcklbw %%mm3, %%mm5		\n\t"

  1923. 			"punpckhbw %%mm3, %%mm7		\n\t"

  1924. 			"punpcklbw %%mm2, %%mm4		\n\t"

  1925. 			"punpckhbw %%mm2, %%mm6		\n\t"

  1926. #if 1

  1927. 			MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"

  1928. 			MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"

  1929. 			MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"

  1930. 			MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"

  1931. #else

  1932. 			"movq %%mm5, (%2, %%"REG_a", 2)	\n\t"

  1933. 			"movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"

  1934. 			"movq %%mm4, (%3, %%"REG_a", 2)	\n\t"

  1935. 			"movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"

  1936. #endif

  1937. 			"add $8, %%"REG_a"		\n\t"

  1938. 			" js 1b				\n\t"

  1939. 			:: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),

  1940. 			   "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),

  1941. 			   "g" (-mmxSize)

  1942. 			: "%"REG_a

  1943. 

  1944. 		);

  1945. #else

  1946. 		const long mmxSize=1;

  1947. #endif

  1948. 		dst[0        ]= (3*src[0] +   src[srcStride])>>2;

  1949. 		dst[dstStride]= (  src[0] + 3*src[srcStride])>>2;

  1950. 

  1951. 		for(x=mmxSize-1; x<srcWidth-1; x++){

  1952. 			dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;

  1953. 			dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;

  1954. 			dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;

  1955. 			dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;

  1956. 		}

  1957. 		dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;

  1958. 		dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;

  1959. 

  1960. 		dst+=dstStride*2;

  1961. 		src+=srcStride;

  1962. 	}

  1963. 	

  1964. 	// last line

  1965. #if 1

  1966. 	dst[0]= src[0];

  1967.         

  1968. 	for(x=0; x<srcWidth-1; x++){

  1969. 		dst[2*x+1]= (3*src[x] +   src[x+1])>>2;

  1970. 		dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;

  1971. 	}

  1972. 	dst[2*srcWidth-1]= src[srcWidth-1];

  1973. #else

  1974. 	for(x=0; x<srcWidth; x++){

  1975. 		dst[2*x+0]=

  1976. 		dst[2*x+1]= src[x];

  1977. 	}

  1978. #endif

  1979. 

  1980. #ifdef HAVE_MMX

  1981. asm volatile(   EMMS" \n\t"

  1982.         	SFENCE" \n\t"

  1983.         	:::"memory");

  1984. #endif

  1985. }

  1986. 

  1987. /**

  1988.  *

  1989.  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a

  1990.  * problem for anyone then tell me, and ill fix it)

  1991.  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version

  1992.  */

  1993. static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  1994. 	long width, long height,

  1995. 	long lumStride, long chromStride, long srcStride)

  1996. {

  1997. 	long y;

  1998. 	const long chromWidth= width>>1;

  1999. 	for(y=0; y<height; y+=2)

  2000. 	{

  2001. #ifdef HAVE_MMX

  2002. 		asm volatile(

  2003. 			"xorl %%eax, %%eax		\n\t"

  2004. 			"pcmpeqw %%mm7, %%mm7		\n\t"

  2005. 			"psrlw $8, %%mm7		\n\t" // FF,00,FF,00...

  2006. 			ASMALIGN(4)

  2007. 			"1:				\n\t"

  2008. 			PREFETCH" 64(%0, %%eax, 4)	\n\t"

  2009. 			"movq (%0, %%eax, 4), %%mm0	\n\t" // UYVY UYVY(0)

  2010. 			"movq 8(%0, %%eax, 4), %%mm1	\n\t" // UYVY UYVY(4)

  2011. 			"movq %%mm0, %%mm2		\n\t" // UYVY UYVY(0)

  2012. 			"movq %%mm1, %%mm3		\n\t" // UYVY UYVY(4)

  2013. 			"pand %%mm7, %%mm0		\n\t" // U0V0 U0V0(0)

  2014. 			"pand %%mm7, %%mm1		\n\t" // U0V0 U0V0(4)

  2015. 			"psrlw $8, %%mm2		\n\t" // Y0Y0 Y0Y0(0)

  2016. 			"psrlw $8, %%mm3		\n\t" // Y0Y0 Y0Y0(4)

  2017. 			"packuswb %%mm1, %%mm0		\n\t" // UVUV UVUV(0)

  2018. 			"packuswb %%mm3, %%mm2		\n\t" // YYYY YYYY(0)

  2019. 

  2020. 			MOVNTQ" %%mm2, (%1, %%eax, 2)	\n\t"

  2021. 

  2022. 			"movq 16(%0, %%eax, 4), %%mm1	\n\t" // UYVY UYVY(8)

  2023. 			"movq 24(%0, %%eax, 4), %%mm2	\n\t" // UYVY UYVY(12)

  2024. 			"movq %%mm1, %%mm3		\n\t" // UYVY UYVY(8)

  2025. 			"movq %%mm2, %%mm4		\n\t" // UYVY UYVY(12)

  2026. 			"pand %%mm7, %%mm1		\n\t" // U0V0 U0V0(8)

  2027. 			"pand %%mm7, %%mm2		\n\t" // U0V0 U0V0(12)

  2028. 			"psrlw $8, %%mm3		\n\t" // Y0Y0 Y0Y0(8)

  2029. 			"psrlw $8, %%mm4		\n\t" // Y0Y0 Y0Y0(12)

  2030. 			"packuswb %%mm2, %%mm1		\n\t" // UVUV UVUV(8)

  2031. 			"packuswb %%mm4, %%mm3		\n\t" // YYYY YYYY(8)

  2032. 

  2033. 			MOVNTQ" %%mm3, 8(%1, %%eax, 2)	\n\t"

  2034. 

  2035. 			"movq %%mm0, %%mm2		\n\t" // UVUV UVUV(0)

  2036. 			"movq %%mm1, %%mm3		\n\t" // UVUV UVUV(8)

  2037. 			"psrlw $8, %%mm0		\n\t" // V0V0 V0V0(0)

  2038. 			"psrlw $8, %%mm1		\n\t" // V0V0 V0V0(8)

  2039. 			"pand %%mm7, %%mm2		\n\t" // U0U0 U0U0(0)

  2040. 			"pand %%mm7, %%mm3		\n\t" // U0U0 U0U0(8)

  2041. 			"packuswb %%mm1, %%mm0		\n\t" // VVVV VVVV(0)

  2042. 			"packuswb %%mm3, %%mm2		\n\t" // UUUU UUUU(0)

  2043. 

  2044. 			MOVNTQ" %%mm0, (%3, %%eax)	\n\t"

  2045. 			MOVNTQ" %%mm2, (%2, %%eax)	\n\t"

  2046. 

  2047. 			"addl $8, %%eax			\n\t"

  2048. 			"cmpl %4, %%eax			\n\t"

  2049. 			" jb 1b				\n\t"

  2050. 			::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  2051. 			: "memory", "%eax"

  2052. 		);

  2053. 

  2054. 		ydst += lumStride;

  2055. 		src  += srcStride;

  2056. 

  2057. 		asm volatile(

  2058. 			"xorl %%eax, %%eax		\n\t"

  2059. 			ASMALIGN(4)

  2060. 			"1:				\n\t"

  2061. 			PREFETCH" 64(%0, %%eax, 4)	\n\t"

  2062. 			"movq (%0, %%eax, 4), %%mm0	\n\t" // YUYV YUYV(0)

  2063. 			"movq 8(%0, %%eax, 4), %%mm1	\n\t" // YUYV YUYV(4)

  2064. 			"movq 16(%0, %%eax, 4), %%mm2	\n\t" // YUYV YUYV(8)

  2065. 			"movq 24(%0, %%eax, 4), %%mm3	\n\t" // YUYV YUYV(12)

  2066. 			"psrlw $8, %%mm0		\n\t" // Y0Y0 Y0Y0(0)

  2067. 			"psrlw $8, %%mm1		\n\t" // Y0Y0 Y0Y0(4)

  2068. 			"psrlw $8, %%mm2		\n\t" // Y0Y0 Y0Y0(8)

  2069. 			"psrlw $8, %%mm3		\n\t" // Y0Y0 Y0Y0(12)

  2070. 			"packuswb %%mm1, %%mm0		\n\t" // YYYY YYYY(0)

  2071. 			"packuswb %%mm3, %%mm2		\n\t" // YYYY YYYY(8)

  2072. 

  2073. 			MOVNTQ" %%mm0, (%1, %%eax, 2)	\n\t"

  2074. 			MOVNTQ" %%mm2, 8(%1, %%eax, 2)	\n\t"

  2075. 

  2076. 			"addl $8, %%eax			\n\t"

  2077. 			"cmpl %4, %%eax			\n\t"

  2078. 			" jb 1b				\n\t"

  2079. 

  2080. 			::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)

  2081. 			: "memory", "%eax"

  2082. 		);

  2083. #else

  2084. 		long i;

  2085. 		for(i=0; i<chromWidth; i++)

  2086. 		{

  2087. 			udst[i] 	= src[4*i+0];

  2088. 			ydst[2*i+0] 	= src[4*i+1];

  2089. 			vdst[i] 	= src[4*i+2];

  2090. 			ydst[2*i+1] 	= src[4*i+3];

  2091. 		}

  2092. 		ydst += lumStride;

  2093. 		src  += srcStride;

  2094. 

  2095. 		for(i=0; i<chromWidth; i++)

  2096. 		{

  2097. 			ydst[2*i+0] 	= src[4*i+1];

  2098. 			ydst[2*i+1] 	= src[4*i+3];

  2099. 		}

  2100. #endif

  2101. 		udst += chromStride;

  2102. 		vdst += chromStride;

  2103. 		ydst += lumStride;

  2104. 		src  += srcStride;

  2105. 	}

  2106. #ifdef HAVE_MMX

  2107. asm volatile(   EMMS" \n\t"

  2108.         	SFENCE" \n\t"

  2109.         	:::"memory");

  2110. #endif

  2111. }

  2112. 

  2113. /**

  2114.  *

  2115.  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a

  2116.  * problem for anyone then tell me, and ill fix it)

  2117.  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version

  2118.  */

  2119. static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,

  2120. 	long width, long height,

  2121. 	long lumStride, long chromStride, long srcStride)

  2122. {

  2123. 	long y;

  2124. 	const long chromWidth= width>>1;

  2125. #ifdef HAVE_MMX

  2126. 	for(y=0; y<height-2; y+=2)

  2127. 	{

  2128. 		long i;

  2129. 		for(i=0; i<2; i++)

  2130. 		{

  2131. 			asm volatile(

  2132. 				"mov %2, %%"REG_a"		\n\t"

  2133. 				"movq "MANGLE(bgr2YCoeff)", %%mm6		\n\t"

  2134. 				"movq "MANGLE(w1111)", %%mm5		\n\t"

  2135. 				"pxor %%mm7, %%mm7		\n\t"

  2136. 				"lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"

  2137. 				ASMALIGN(4)

  2138. 				"1:				\n\t"

  2139. 				PREFETCH" 64(%0, %%"REG_d")	\n\t"

  2140. 				"movd (%0, %%"REG_d"), %%mm0	\n\t"

  2141. 				"movd 3(%0, %%"REG_d"), %%mm1	\n\t"

  2142. 				"punpcklbw %%mm7, %%mm0		\n\t"

  2143. 				"punpcklbw %%mm7, %%mm1		\n\t"

  2144. 				"movd 6(%0, %%"REG_d"), %%mm2	\n\t"

  2145. 				"movd 9(%0, %%"REG_d"), %%mm3	\n\t"

  2146. 				"punpcklbw %%mm7, %%mm2		\n\t"

  2147. 				"punpcklbw %%mm7, %%mm3		\n\t"

  2148. 				"pmaddwd %%mm6, %%mm0		\n\t"

  2149. 				"pmaddwd %%mm6, %%mm1		\n\t"

  2150. 				"pmaddwd %%mm6, %%mm2		\n\t"

  2151. 				"pmaddwd %%mm6, %%mm3		\n\t"

  2152. #ifndef FAST_BGR2YV12

  2153. 				"psrad $8, %%mm0		\n\t"

  2154. 				"psrad $8, %%mm1		\n\t"

  2155. 				"psrad $8, %%mm2		\n\t"

  2156. 				"psrad $8, %%mm3		\n\t"

  2157. #endif

  2158. 				"packssdw %%mm1, %%mm0		\n\t"

  2159. 				"packssdw %%mm3, %%mm2		\n\t"

  2160. 				"pmaddwd %%mm5, %%mm0		\n\t"

  2161. 				"pmaddwd %%mm5, %%mm2		\n\t"

  2162. 				"packssdw %%mm2, %%mm0		\n\t"

  2163. 				"psraw $7, %%mm0		\n\t"

  2164. 

  2165. 				"movd 12(%0, %%"REG_d"), %%mm4	\n\t"

  2166. 				"movd 15(%0, %%"REG_d"), %%mm1	\n\t"

  2167. 				"punpcklbw %%mm7, %%mm4		\n\t"

  2168. 				"punpcklbw %%mm7, %%mm1		\n\t"

  2169. 				"movd 18(%0, %%"REG_d"), %%mm2	\n\t"

  2170. 				"movd 21(%0, %%"REG_d"), %%mm3	\n\t"

  2171. 				"punpcklbw %%mm7, %%mm2		\n\t"

  2172. 				"punpcklbw %%mm7, %%mm3		\n\t"

  2173. 				"pmaddwd %%mm6, %%mm4		\n\t"

  2174. 				"pmaddwd %%mm6, %%mm1		\n\t"

  2175. 				"pmaddwd %%mm6, %%mm2		\n\t"

  2176. 				"pmaddwd %%mm6, %%mm3		\n\t"

  2177. #ifndef FAST_BGR2YV12

  2178. 				"psrad $8, %%mm4		\n\t"

  2179. 				"psrad $8, %%mm1		\n\t"

  2180. 				"psrad $8, %%mm2		\n\t"

  2181. 				"psrad $8, %%mm3		\n\t"

  2182. #endif

  2183. 				"packssdw %%mm1, %%mm4		\n\t"

  2184. 				"packssdw %%mm3, %%mm2		\n\t"

  2185. 				"pmaddwd %%mm5, %%mm4		\n\t"

  2186. 				"pmaddwd %%mm5, %%mm2		\n\t"

  2187. 				"add $24, %%"REG_d"		\n\t"

  2188. 				"packssdw %%mm2, %%mm4		\n\t"

  2189. 				"psraw $7, %%mm4		\n\t"

  2190. 

  2191. 				"packuswb %%mm4, %%mm0		\n\t"

  2192. 				"paddusb "MANGLE(bgr2YOffset)", %%mm0	\n\t"

  2193. 

  2194. 				MOVNTQ" %%mm0, (%1, %%"REG_a")	\n\t"

  2195. 				"add $8, %%"REG_a"		\n\t"

  2196. 				" js 1b				\n\t"

  2197. 				: : "r" (src+width*3), "r" (ydst+width), "g" (-width)

  2198. 				: "%"REG_a, "%"REG_d

  2199. 			);

  2200. 			ydst += lumStride;

  2201. 			src  += srcStride;

  2202. 		}

  2203. 		src -= srcStride*2;

  2204. 		asm volatile(

  2205. 			"mov %4, %%"REG_a"		\n\t"

  2206. 			"movq "MANGLE(w1111)", %%mm5		\n\t"

  2207. 			"movq "MANGLE(bgr2UCoeff)", %%mm6		\n\t"

  2208. 			"pxor %%mm7, %%mm7		\n\t"

  2209. 			"lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"

  2210. 			"add %%"REG_d", %%"REG_d"	\n\t"

  2211. 			ASMALIGN(4)

  2212. 			"1:				\n\t"

  2213. 			PREFETCH" 64(%0, %%"REG_d")	\n\t"

  2214. 			PREFETCH" 64(%1, %%"REG_d")	\n\t"

  2215. #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)

  2216. 			"movq (%0, %%"REG_d"), %%mm0	\n\t"

  2217. 			"movq (%1, %%"REG_d"), %%mm1	\n\t"

  2218. 			"movq 6(%0, %%"REG_d"), %%mm2	\n\t"

  2219. 			"movq 6(%1, %%"REG_d"), %%mm3	\n\t"

  2220. 			PAVGB" %%mm1, %%mm0		\n\t"

  2221. 			PAVGB" %%mm3, %%mm2		\n\t"

  2222. 			"movq %%mm0, %%mm1		\n\t"

  2223. 			"movq %%mm2, %%mm3		\n\t"

  2224. 			"psrlq $24, %%mm0		\n\t"

  2225. 			"psrlq $24, %%mm2		\n\t"

  2226. 			PAVGB" %%mm1, %%mm0		\n\t"

  2227. 			PAVGB" %%mm3, %%mm2		\n\t"

  2228. 			"punpcklbw %%mm7, %%mm0		\n\t"

  2229. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2230. #else

  2231. 			"movd (%0, %%"REG_d"), %%mm0	\n\t"

  2232. 			"movd (%1, %%"REG_d"), %%mm1	\n\t"

  2233. 			"movd 3(%0, %%"REG_d"), %%mm2	\n\t"

  2234. 			"movd 3(%1, %%"REG_d"), %%mm3	\n\t"

  2235. 			"punpcklbw %%mm7, %%mm0		\n\t"

  2236. 			"punpcklbw %%mm7, %%mm1		\n\t"

  2237. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2238. 			"punpcklbw %%mm7, %%mm3		\n\t"

  2239. 			"paddw %%mm1, %%mm0		\n\t"

  2240. 			"paddw %%mm3, %%mm2		\n\t"

  2241. 			"paddw %%mm2, %%mm0		\n\t"

  2242. 			"movd 6(%0, %%"REG_d"), %%mm4	\n\t"

  2243. 			"movd 6(%1, %%"REG_d"), %%mm1	\n\t"

  2244. 			"movd 9(%0, %%"REG_d"), %%mm2	\n\t"

  2245. 			"movd 9(%1, %%"REG_d"), %%mm3	\n\t"

  2246. 			"punpcklbw %%mm7, %%mm4		\n\t"

  2247. 			"punpcklbw %%mm7, %%mm1		\n\t"

  2248. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2249. 			"punpcklbw %%mm7, %%mm3		\n\t"

  2250. 			"paddw %%mm1, %%mm4		\n\t"

  2251. 			"paddw %%mm3, %%mm2		\n\t"

  2252. 			"paddw %%mm4, %%mm2		\n\t"

  2253. 			"psrlw $2, %%mm0		\n\t"

  2254. 			"psrlw $2, %%mm2		\n\t"

  2255. #endif

  2256. 			"movq "MANGLE(bgr2VCoeff)", %%mm1		\n\t"

  2257. 			"movq "MANGLE(bgr2VCoeff)", %%mm3		\n\t"

  2258. 

  2259. 			"pmaddwd %%mm0, %%mm1		\n\t"

  2260. 			"pmaddwd %%mm2, %%mm3		\n\t"

  2261. 			"pmaddwd %%mm6, %%mm0		\n\t"

  2262. 			"pmaddwd %%mm6, %%mm2		\n\t"

  2263. #ifndef FAST_BGR2YV12

  2264. 			"psrad $8, %%mm0		\n\t"

  2265. 			"psrad $8, %%mm1		\n\t"

  2266. 			"psrad $8, %%mm2		\n\t"

  2267. 			"psrad $8, %%mm3		\n\t"

  2268. #endif

  2269. 			"packssdw %%mm2, %%mm0		\n\t"

  2270. 			"packssdw %%mm3, %%mm1		\n\t"

  2271. 			"pmaddwd %%mm5, %%mm0		\n\t"

  2272. 			"pmaddwd %%mm5, %%mm1		\n\t"

  2273. 			"packssdw %%mm1, %%mm0		\n\t" // V1 V0 U1 U0

  2274. 			"psraw $7, %%mm0		\n\t"

  2275. 

  2276. #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)

  2277. 			"movq 12(%0, %%"REG_d"), %%mm4	\n\t"

  2278. 			"movq 12(%1, %%"REG_d"), %%mm1	\n\t"

  2279. 			"movq 18(%0, %%"REG_d"), %%mm2	\n\t"

  2280. 			"movq 18(%1, %%"REG_d"), %%mm3	\n\t"

  2281. 			PAVGB" %%mm1, %%mm4		\n\t"

  2282. 			PAVGB" %%mm3, %%mm2		\n\t"

  2283. 			"movq %%mm4, %%mm1		\n\t"

  2284. 			"movq %%mm2, %%mm3		\n\t"

  2285. 			"psrlq $24, %%mm4		\n\t"

  2286. 			"psrlq $24, %%mm2		\n\t"

  2287. 			PAVGB" %%mm1, %%mm4		\n\t"

  2288. 			PAVGB" %%mm3, %%mm2		\n\t"

  2289. 			"punpcklbw %%mm7, %%mm4		\n\t"

  2290. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2291. #else

  2292. 			"movd 12(%0, %%"REG_d"), %%mm4	\n\t"

  2293. 			"movd 12(%1, %%"REG_d"), %%mm1	\n\t"

  2294. 			"movd 15(%0, %%"REG_d"), %%mm2	\n\t"

  2295. 			"movd 15(%1, %%"REG_d"), %%mm3	\n\t"

  2296. 			"punpcklbw %%mm7, %%mm4		\n\t"

  2297. 			"punpcklbw %%mm7, %%mm1		\n\t"

  2298. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2299. 			"punpcklbw %%mm7, %%mm3		\n\t"

  2300. 			"paddw %%mm1, %%mm4		\n\t"

  2301. 			"paddw %%mm3, %%mm2		\n\t"

  2302. 			"paddw %%mm2, %%mm4		\n\t"

  2303. 			"movd 18(%0, %%"REG_d"), %%mm5	\n\t"

  2304. 			"movd 18(%1, %%"REG_d"), %%mm1	\n\t"

  2305. 			"movd 21(%0, %%"REG_d"), %%mm2	\n\t"

  2306. 			"movd 21(%1, %%"REG_d"), %%mm3	\n\t"

  2307. 			"punpcklbw %%mm7, %%mm5		\n\t"

  2308. 			"punpcklbw %%mm7, %%mm1		\n\t"

  2309. 			"punpcklbw %%mm7, %%mm2		\n\t"

  2310. 			"punpcklbw %%mm7, %%mm3		\n\t"

  2311. 			"paddw %%mm1, %%mm5		\n\t"

  2312. 			"paddw %%mm3, %%mm2		\n\t"

  2313. 			"paddw %%mm5, %%mm2		\n\t"

  2314. 			"movq "MANGLE(w1111)", %%mm5		\n\t"

  2315. 			"psrlw $2, %%mm4		\n\t"

  2316. 			"psrlw $2, %%mm2		\n\t"

  2317. #endif

  2318. 			"movq "MANGLE(bgr2VCoeff)", %%mm1		\n\t"

  2319. 			"movq "MANGLE(bgr2VCoeff)", %%mm3		\n\t"

  2320. 

  2321. 			"pmaddwd %%mm4, %%mm1		\n\t"

  2322. 			"pmaddwd %%mm2, %%mm3		\n\t"

  2323. 			"pmaddwd %%mm6, %%mm4		\n\t"

  2324. 			"pmaddwd %%mm6, %%mm2		\n\t"

  2325. #ifndef FAST_BGR2YV12

  2326. 			"psrad $8, %%mm4		\n\t"

  2327. 			"psrad $8, %%mm1		\n\t"

  2328. 			"psrad $8, %%mm2		\n\t"

  2329. 			"psrad $8, %%mm3		\n\t"

  2330. #endif

  2331. 			"packssdw %%mm2, %%mm4		\n\t"

  2332. 			"packssdw %%mm3, %%mm1		\n\t"

  2333. 			"pmaddwd %%mm5, %%mm4		\n\t"

  2334. 			"pmaddwd %%mm5, %%mm1		\n\t"

  2335. 			"add $24, %%"REG_d"		\n\t"

  2336. 			"packssdw %%mm1, %%mm4		\n\t" // V3 V2 U3 U2

  2337. 			"psraw $7, %%mm4		\n\t"

  2338. 

  2339. 			"movq %%mm0, %%mm1		\n\t"

  2340. 			"punpckldq %%mm4, %%mm0		\n\t"

  2341. 			"punpckhdq %%mm4, %%mm1		\n\t"

  2342. 			"packsswb %%mm1, %%mm0		\n\t"

  2343. 			"paddb "MANGLE(bgr2UVOffset)", %%mm0	\n\t"

  2344. 			"movd %%mm0, (%2, %%"REG_a")	\n\t"

  2345. 			"punpckhdq %%mm0, %%mm0		\n\t"

  2346. 			"movd %%mm0, (%3, %%"REG_a")	\n\t"

  2347. 			"add $4, %%"REG_a"		\n\t"

  2348. 			" js 1b				\n\t"

  2349. 			: : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)

  2350. 			: "%"REG_a, "%"REG_d

  2351. 		);

  2352. 

  2353. 		udst += chromStride;

  2354. 		vdst += chromStride;

  2355. 		src  += srcStride*2;

  2356. 	}

  2357. 

  2358. 	asm volatile(   EMMS" \n\t"

  2359. 			SFENCE" \n\t"

  2360. 			:::"memory");

  2361. #else

  2362. 	y=0;

  2363. #endif

  2364. 	for(; y<height; y+=2)

  2365. 	{

  2366. 		long i;

  2367. 		for(i=0; i<chromWidth; i++)

  2368. 		{

  2369. 			unsigned int b= src[6*i+0];

  2370. 			unsigned int g= src[6*i+1];

  2371. 			unsigned int r= src[6*i+2];

  2372. 

  2373. 			unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;

  2374. 			unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;

  2375. 			unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;

  2376. 

  2377. 			udst[i] 	= U;

  2378. 			vdst[i] 	= V;

  2379. 			ydst[2*i] 	= Y;

  2380. 

  2381. 			b= src[6*i+3];

  2382. 			g= src[6*i+4];

  2383. 			r= src[6*i+5];

  2384. 

  2385. 			Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;

  2386. 			ydst[2*i+1] 	= Y;

  2387. 		}

  2388. 		ydst += lumStride;

  2389. 		src  += srcStride;

  2390. 

  2391. 		for(i=0; i<chromWidth; i++)

  2392. 		{

  2393. 			unsigned int b= src[6*i+0];

  2394. 			unsigned int g= src[6*i+1];

  2395. 			unsigned int r= src[6*i+2];

  2396. 

  2397. 			unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;

  2398. 

  2399. 			ydst[2*i] 	= Y;

  2400. 

  2401. 			b= src[6*i+3];

  2402. 			g= src[6*i+4];

  2403. 			r= src[6*i+5];

  2404. 

  2405. 			Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;

  2406. 			ydst[2*i+1] 	= Y;

  2407. 		}

  2408. 		udst += chromStride;

  2409. 		vdst += chromStride;

  2410. 		ydst += lumStride;

  2411. 		src  += srcStride;

  2412. 	}

  2413. }

  2414. 

  2415. void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,

  2416. 			    long width, long height, long src1Stride,

  2417. 			    long src2Stride, long dstStride){

  2418. 	long h;

  2419. 

  2420. 	for(h=0; h < height; h++)

  2421. 	{

  2422. 		long w;

  2423. 

  2424. #ifdef HAVE_MMX

  2425. #ifdef HAVE_SSE2

  2426. 		asm(

  2427. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  2428. 			"1:				\n\t"

  2429. 			PREFETCH" 64(%1, %%"REG_a")	\n\t"

  2430. 			PREFETCH" 64(%2, %%"REG_a")	\n\t"

  2431. 			"movdqa (%1, %%"REG_a"), %%xmm0	\n\t"

  2432. 			"movdqa (%1, %%"REG_a"), %%xmm1	\n\t"

  2433. 			"movdqa (%2, %%"REG_a"), %%xmm2	\n\t"

  2434. 			"punpcklbw %%xmm2, %%xmm0	\n\t"

  2435. 			"punpckhbw %%xmm2, %%xmm1	\n\t"

  2436. 			"movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"

  2437. 			"movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"

  2438. 			"add $16, %%"REG_a"		\n\t"

  2439. 			"cmp %3, %%"REG_a"		\n\t"

  2440. 			" jb 1b				\n\t"

  2441. 			::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)

  2442. 			: "memory", "%"REG_a""

  2443. 		);

  2444. #else

  2445. 		asm(

  2446. 			"xor %%"REG_a", %%"REG_a"	\n\t"

  2447. 			"1:				\n\t"

  2448. 			PREFETCH" 64(%1, %%"REG_a")	\n\t"

  2449. 			PREFETCH" 64(%2, %%"REG_a")	\n\t"

  2450. 			"movq (%1, %%"REG_a"), %%mm0	\n\t"

  2451. 			"movq 8(%1, %%"REG_a"), %%mm2	\n\t"

  2452. 			"movq %%mm0, %%mm1		\n\t"

  2453. 			"movq %%mm2, %%mm3		\n\t"

  2454. 			"movq (%2, %%"REG_a"), %%mm4	\n\t"

  2455. 			"movq 8(%2, %%"REG_a"), %%mm5	\n\t"

  2456. 			"punpcklbw %%mm4, %%mm0		\n\t"

  2457. 			"punpckhbw %%mm4, %%mm1		\n\t"

  2458. 			"punpcklbw %%mm5, %%mm2		\n\t"

  2459. 			"punpckhbw %%mm5, %%mm3		\n\t"

  2460. 			MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"

  2461. 			MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"

  2462. 			MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"

  2463. 			MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"

  2464. 			"add $16, %%"REG_a"		\n\t"

  2465. 			"cmp %3, %%"REG_a"		\n\t"

  2466. 			" jb 1b				\n\t"

  2467. 			::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)

  2468. 			: "memory", "%"REG_a

  2469. 		);

  2470. #endif

  2471. 		for(w= (width&(~15)); w < width; w++)

  2472. 		{

  2473. 			dest[2*w+0] = src1[w];

  2474. 			dest[2*w+1] = src2[w];

  2475. 		}

  2476. #else

  2477. 		for(w=0; w < width; w++)

  2478. 		{

  2479. 			dest[2*w+0] = src1[w];

  2480. 			dest[2*w+1] = src2[w];

  2481. 		}

  2482. #endif

  2483. 		dest += dstStride;

  2484.                 src1 += src1Stride;

  2485.                 src2 += src2Stride;

  2486. 	}

  2487. #ifdef HAVE_MMX

  2488. 	asm(

  2489. 		EMMS" \n\t"

  2490. 		SFENCE" \n\t"

  2491. 		::: "memory"

  2492. 		);

  2493. #endif

  2494. }

  2495. 

  2496. static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,

  2497. 			uint8_t *dst1, uint8_t *dst2,

  2498. 			long width, long height,

  2499. 			long srcStride1, long srcStride2,

  2500. 			long dstStride1, long dstStride2)

  2501. {

  2502.     long y,x,w,h;

  2503.     w=width/2; h=height/2;

  2504. #ifdef HAVE_MMX

  2505.     asm volatile(

  2506. 	PREFETCH" %0\n\t"

  2507. 	PREFETCH" %1\n\t"

  2508. 	::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");

  2509. #endif

  2510.     for(y=0;y<h;y++){

  2511. 	const uint8_t* s1=src1+srcStride1*(y>>1);

  2512. 	uint8_t* d=dst1+dstStride1*y;

  2513. 	x=0;

  2514. #ifdef HAVE_MMX

  2515. 	for(;x<w-31;x+=32)

  2516. 	{

  2517. 	    asm volatile(

  2518. 		PREFETCH" 32%1\n\t"

  2519. 	        "movq	%1, %%mm0\n\t"

  2520. 	        "movq	8%1, %%mm2\n\t"

  2521. 	        "movq	16%1, %%mm4\n\t"

  2522. 	        "movq	24%1, %%mm6\n\t"

  2523. 	        "movq	%%mm0, %%mm1\n\t"

  2524. 	        "movq	%%mm2, %%mm3\n\t"

  2525. 	        "movq	%%mm4, %%mm5\n\t"

  2526. 	        "movq	%%mm6, %%mm7\n\t"

  2527. 		"punpcklbw %%mm0, %%mm0\n\t"

  2528. 		"punpckhbw %%mm1, %%mm1\n\t"

  2529. 		"punpcklbw %%mm2, %%mm2\n\t"

  2530. 		"punpckhbw %%mm3, %%mm3\n\t"

  2531. 		"punpcklbw %%mm4, %%mm4\n\t"

  2532. 		"punpckhbw %%mm5, %%mm5\n\t"

  2533. 		"punpcklbw %%mm6, %%mm6\n\t"

  2534. 		"punpckhbw %%mm7, %%mm7\n\t"

  2535. 		MOVNTQ"	%%mm0, %0\n\t"

  2536. 		MOVNTQ"	%%mm1, 8%0\n\t"

  2537. 		MOVNTQ"	%%mm2, 16%0\n\t"

  2538. 		MOVNTQ"	%%mm3, 24%0\n\t"

  2539. 		MOVNTQ"	%%mm4, 32%0\n\t"

  2540. 		MOVNTQ"	%%mm5, 40%0\n\t"

  2541. 		MOVNTQ"	%%mm6, 48%0\n\t"

  2542. 		MOVNTQ"	%%mm7, 56%0"

  2543. 		:"=m"(d[2*x])

  2544. 		:"m"(s1[x])

  2545. 		:"memory");

  2546. 	}

  2547. #endif

  2548. 	for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];

  2549.     }

  2550.     for(y=0;y<h;y++){

  2551. 	const uint8_t* s2=src2+srcStride2*(y>>1);

  2552. 	uint8_t* d=dst2+dstStride2*y;

  2553. 	x=0;

  2554. #ifdef HAVE_MMX

  2555. 	for(;x<w-31;x+=32)

  2556. 	{

  2557. 	    asm volatile(

  2558. 		PREFETCH" 32%1\n\t"

  2559. 	        "movq	%1, %%mm0\n\t"

  2560. 	        "movq	8%1, %%mm2\n\t"

  2561. 	        "movq	16%1, %%mm4\n\t"

  2562. 	        "movq	24%1, %%mm6\n\t"

  2563. 	        "movq	%%mm0, %%mm1\n\t"

  2564. 	        "movq	%%mm2, %%mm3\n\t"

  2565. 	        "movq	%%mm4, %%mm5\n\t"

  2566. 	        "movq	%%mm6, %%mm7\n\t"

  2567. 		"punpcklbw %%mm0, %%mm0\n\t"

  2568. 		"punpckhbw %%mm1, %%mm1\n\t"

  2569. 		"punpcklbw %%mm2, %%mm2\n\t"

  2570. 		"punpckhbw %%mm3, %%mm3\n\t"

  2571. 		"punpcklbw %%mm4, %%mm4\n\t"

  2572. 		"punpckhbw %%mm5, %%mm5\n\t"

  2573. 		"punpcklbw %%mm6, %%mm6\n\t"

  2574. 		"punpckhbw %%mm7, %%mm7\n\t"

  2575. 		MOVNTQ"	%%mm0, %0\n\t"

  2576. 		MOVNTQ"	%%mm1, 8%0\n\t"

  2577. 		MOVNTQ"	%%mm2, 16%0\n\t"

  2578. 		MOVNTQ"	%%mm3, 24%0\n\t"

  2579. 		MOVNTQ"	%%mm4, 32%0\n\t"

  2580. 		MOVNTQ"	%%mm5, 40%0\n\t"

  2581. 		MOVNTQ"	%%mm6, 48%0\n\t"

  2582. 		MOVNTQ"	%%mm7, 56%0"

  2583. 		:"=m"(d[2*x])

  2584. 		:"m"(s2[x])

  2585. 		:"memory");

  2586. 	}

  2587. #endif

  2588. 	for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];

  2589.     }

  2590. #ifdef HAVE_MMX

  2591. 	asm(

  2592. 		EMMS" \n\t"

  2593. 		SFENCE" \n\t"

  2594. 		::: "memory"

  2595. 		);

  2596. #endif

  2597. }

  2598. 

  2599. static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,

  2600. 			uint8_t *dst,

  2601. 			long width, long height,

  2602. 			long srcStride1, long srcStride2,

  2603. 			long srcStride3, long dstStride)

  2604. {

  2605.     long y,x,w,h;

  2606.     w=width/2; h=height;

  2607.     for(y=0;y<h;y++){

  2608. 	const uint8_t* yp=src1+srcStride1*y;

  2609. 	const uint8_t* up=src2+srcStride2*(y>>2);

  2610. 	const uint8_t* vp=src3+srcStride3*(y>>2);

  2611. 	uint8_t* d=dst+dstStride*y;

  2612. 	x=0;

  2613. #ifdef HAVE_MMX

  2614. 	for(;x<w-7;x+=8)

  2615. 	{

  2616. 	    asm volatile(

  2617. 		PREFETCH" 32(%1, %0)\n\t"

  2618. 		PREFETCH" 32(%2, %0)\n\t"

  2619. 		PREFETCH" 32(%3, %0)\n\t"

  2620. 		"movq	(%1, %0, 4), %%mm0\n\t"       /* Y0Y1Y2Y3Y4Y5Y6Y7 */

  2621. 		"movq	(%2, %0), %%mm1\n\t"       /* U0U1U2U3U4U5U6U7 */

  2622. 		"movq	(%3, %0), %%mm2\n\t"	     /* V0V1V2V3V4V5V6V7 */

  2623. 		"movq	%%mm0, %%mm3\n\t"    /* Y0Y1Y2Y3Y4Y5Y6Y7 */

  2624. 		"movq	%%mm1, %%mm4\n\t"    /* U0U1U2U3U4U5U6U7 */

  2625. 		"movq	%%mm2, %%mm5\n\t"    /* V0V1V2V3V4V5V6V7 */

  2626. 		"punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */

  2627. 		"punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */

  2628. 		"punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */

  2629. 		"punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */

  2630. 

  2631. 		"movq	%%mm1, %%mm6\n\t"

  2632. 		"punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/

  2633. 		"punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/

  2634. 		"punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/

  2635. 		MOVNTQ"	%%mm0, (%4, %0, 8)\n\t"

  2636. 		MOVNTQ"	%%mm3, 8(%4, %0, 8)\n\t"

  2637. 		

  2638. 		"punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/

  2639. 		"movq	8(%1, %0, 4), %%mm0\n\t"

  2640. 		"movq	%%mm0, %%mm3\n\t"

  2641. 		"punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/

  2642. 		"punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/

  2643. 		MOVNTQ"	%%mm0, 16(%4, %0, 8)\n\t"

  2644. 		MOVNTQ"	%%mm3, 24(%4, %0, 8)\n\t"

  2645. 

  2646. 		"movq	%%mm4, %%mm6\n\t"

  2647. 		"movq	16(%1, %0, 4), %%mm0\n\t"

  2648. 		"movq	%%mm0, %%mm3\n\t"

  2649. 		"punpcklbw %%mm5, %%mm4\n\t"

  2650. 		"punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/

  2651. 		"punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/

  2652. 		MOVNTQ"	%%mm0, 32(%4, %0, 8)\n\t"

  2653. 		MOVNTQ"	%%mm3, 40(%4, %0, 8)\n\t"

  2654. 		

  2655. 		"punpckhbw %%mm5, %%mm6\n\t"

  2656. 		"movq	24(%1, %0, 4), %%mm0\n\t"

  2657. 		"movq	%%mm0, %%mm3\n\t"

  2658. 		"punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/

  2659. 		"punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/

  2660. 		MOVNTQ"	%%mm0, 48(%4, %0, 8)\n\t"

  2661. 		MOVNTQ"	%%mm3, 56(%4, %0, 8)\n\t"

  2662. 

  2663. 		: "+r" (x)

  2664.                 : "r"(yp), "r" (up), "r"(vp), "r"(d)

  2665. 		:"memory");

  2666. 	}

  2667. #endif

  2668. 	for(; x<w; x++)

  2669. 	{

  2670. 	    const long x2= x<<2;

  2671. 	    d[8*x+0]=yp[x2];

  2672. 	    d[8*x+1]=up[x];

  2673. 	    d[8*x+2]=yp[x2+1];

  2674. 	    d[8*x+3]=vp[x];

  2675. 	    d[8*x+4]=yp[x2+2];

  2676. 	    d[8*x+5]=up[x];

  2677. 	    d[8*x+6]=yp[x2+3];

  2678. 	    d[8*x+7]=vp[x];

  2679. 	}

  2680.     }

  2681. #ifdef HAVE_MMX

  2682. 	asm(

  2683. 		EMMS" \n\t"

  2684. 		SFENCE" \n\t"

  2685. 		::: "memory"

  2686. 		);

  2687. #endif

  2688. }