falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
152 Commits
32 Branches
283MB
Tree: 13e0052810
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '13e0052810'
${ noResults }
FFmpeg/postproc/postprocess.c

1823 lines
49KB
Raw Blame History 

  1. /*

  2.     Copyright (C) 2001 Michael Niedermayer (michaelni@gmx.at)

  3. 

  4.     This program is free software; you can redistribute it and/or modify

  5.     it under the terms of the GNU General Public License as published by

  6.     the Free Software Foundation; either version 2 of the License, or

  7.     (at your option) any later version.

  8. 

  9.     This program is distributed in the hope that it will be useful,

  10.     but WITHOUT ANY WARRANTY; without even the implied warranty of

  11.     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  12.     GNU General Public License for more details.

  13. 

  14.     You should have received a copy of the GNU General Public License

  15.     along with this program; if not, write to the Free Software

  16.     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

  17. */

  18. 

  19. /*

  20. 			C	MMX	MMX2	3DNow*

  21. isVertDC		Ec	Ec

  22. isVertMinMaxOk		Ec	Ec

  23. doVertLowPass		E		e	e*

  24. doVertDefFilter		Ec	Ec	Ec

  25. isHorizDC		Ec	Ec

  26. isHorizMinMaxOk		a

  27. doHorizLowPass		E		a	a*

  28. doHorizDefFilter	E	ac	ac

  29. deRing

  30. 

  31. * i dont have a 3dnow CPU -> its untested

  32. E = Exact implementation

  33. e = allmost exact implementation

  34. a = alternative / approximate impl

  35. c = checked against the other implementations (-vo md5)

  36. */

  37. 

  38. /*

  39. TODO:

  40. verify that everything workes as it should

  41. reduce the time wasted on the mem transfer

  42. implement dering

  43. implement everything in C at least (done at the moment but ...)

  44. figure range of QP out (assuming <256 for now)

  45. unroll stuff if instructions depend too much on the prior one

  46. we use 8x8 blocks for the horizontal filters, opendivx seems to use 8x4?

  47. move YScale thing to the end instead of fixing QP

  48. write a faster and higher quality deblocking filter :)

  49. ...

  50. 

  51. Notes:

  52. 

  53. */

  54. 

  55. /*

  56. Changelog:

  57. 0.1.2

  58. 	fixed a bug in the horizontal default filter

  59. 	3dnow version of the Horizontal & Vertical Lowpass filters

  60. 	mmx version of the Horizontal Default filter

  61. 	mmx2 & C versions of a simple filter described in a paper from ramkishor & karandikar

  62. 	added mode flags & quality2mode function

  63. 0.1.1

  64. */

  65. 

  66. 

  67. #include <inttypes.h>

  68. #include <stdio.h>

  69. #include "../config.h"

  70. //#undef HAVE_MMX2

  71. //#define HAVE_3DNOW

  72. //#undef HAVE_MMX

  73. #include "postprocess.h"

  74. 

  75. 

  76. static uint64_t packedYOffset=	0x0000000000000000LL;

  77. static uint64_t packedYScale=	0x0100010001000100LL;

  78. static uint64_t w05=		0x0005000500050005LL;

  79. static uint64_t w20=		0x0020002000200020LL;

  80. static uint64_t w1400=		0x1400140014001400LL;

  81. static uint64_t bm00000001=	0x00000000000000FFLL;

  82. static uint64_t bm00010000=	0x000000FF00000000LL;

  83. static uint64_t bm00001000=	0x00000000FF000000LL;

  84. static uint64_t bm10000000=	0xFF00000000000000LL;

  85. static uint64_t bm10000001=	0xFF000000000000FFLL;

  86. static uint64_t bm11000011=	0xFFFF00000000FFFFLL;

  87. static uint64_t bm00000011=	0x000000000000FFFFLL;

  88. static uint64_t bm11000000=	0xFFFF000000000000LL;

  89. static uint64_t bm00011000=	0x000000FFFF000000LL;

  90. static uint64_t bm00110011=	0x0000FFFF0000FFFFLL;

  91. static uint64_t bm11001100=	0xFFFF0000FFFF0000LL;

  92. static uint64_t b00= 		0x0000000000000000LL;

  93. static uint64_t b02= 		0x0202020202020202LL;

  94. static uint64_t b0F= 		0x0F0F0F0F0F0F0F0FLL;

  95. static uint64_t bFF= 		0xFFFFFFFFFFFFFFFFLL;

  96. static uint64_t b20= 		0x2020202020202020LL;

  97. static uint64_t b80= 		0x8080808080808080LL;

  98. static uint64_t b7E= 		0x7E7E7E7E7E7E7E7ELL;

  99. static uint64_t b7C= 		0x7C7C7C7C7C7C7C7CLL;

  100. static uint64_t b3F= 		0x3F3F3F3F3F3F3F3FLL;

  101. static uint64_t temp0=0;

  102. static uint64_t temp1=0;

  103. static uint64_t temp2=0;

  104. static uint64_t temp3=0;

  105. static uint64_t temp4=0;

  106. static uint64_t temp5=0;

  107. static uint64_t pQPb=0;

  108. static uint8_t tempBlock[16*16];

  109. 

  110. int hFlatnessThreshold= 56 - 16;

  111. int vFlatnessThreshold= 56 - 16;

  112. 

  113. //amount of "black" u r willing to loose to get a brightness corrected picture

  114. double maxClippedThreshold= 0.01;

  115. 

  116. int maxAllowedY=255;

  117. //FIXME can never make a movie´s black brighter (anyone needs that?)

  118. int minAllowedY=0;

  119. 

  120. 

  121. static inline long long rdtsc()

  122. {

  123. 	long long l;

  124. 	asm volatile(	"rdtsc\n\t"

  125. 		: "=A" (l)

  126. 	);

  127. //	printf("%d\n", int(l/1000));

  128. 	return l;

  129. }

  130. 

  131. static inline void prefetchnta(void *p)

  132. {

  133. 	asm volatile(	"prefetchnta (%0)\n\t"

  134. 		: : "r" (p)

  135. 	);

  136. }

  137. 

  138. static inline void prefetcht0(void *p)

  139. {

  140. 	asm volatile(	"prefetcht0 (%0)\n\t"

  141. 		: : "r" (p)

  142. 	);

  143. }

  144. 

  145. static inline void prefetcht1(void *p)

  146. {

  147. 	asm volatile(	"prefetcht1 (%0)\n\t"

  148. 		: : "r" (p)

  149. 	);

  150. }

  151. 

  152. static inline void prefetcht2(void *p)

  153. {

  154. 	asm volatile(	"prefetcht2 (%0)\n\t"

  155. 		: : "r" (p)

  156. 	);

  157. }

  158. 

  159. //FIXME? |255-0| = 1 (shouldnt be a problem ...)

  160. /**

  161.  * Check if the middle 8x8 Block in the given 8x10 block is flat

  162.  */

  163. static inline bool isVertDC(uint8_t src[], int stride){

  164. //	return true;

  165. 	int numEq= 0;

  166. 	src+= stride; // src points to begin of the 8x8 Block

  167. #ifdef HAVE_MMX

  168. 	asm volatile(

  169. //		"int $3 \n\t"

  170. 		"pushl %1\n\t"

  171. 		"movq b7E, %%mm7					\n\t" // mm7 = 0x7F

  172. 		"movq b7C, %%mm6					\n\t" // mm6 = 0x7D

  173. 		"movq (%1), %%mm0				\n\t"

  174. 		"addl %2, %1					\n\t"

  175. 		"movq (%1), %%mm1				\n\t"

  176. 		"psubb %%mm1, %%mm0				\n\t" // mm0 = differnece

  177. 		"paddb %%mm7, %%mm0				\n\t"

  178. 		"pcmpgtb %%mm6, %%mm0				\n\t"

  179. 

  180. 		"addl %2, %1					\n\t"

  181. 		"movq (%1), %%mm2				\n\t"

  182. 		"psubb %%mm2, %%mm1				\n\t"

  183. 		"paddb %%mm7, %%mm1				\n\t"

  184. 		"pcmpgtb %%mm6, %%mm1				\n\t"

  185. 		"paddb %%mm1, %%mm0				\n\t"

  186. 

  187. 		"addl %2, %1					\n\t"

  188. 		"movq (%1), %%mm1				\n\t"

  189. 		"psubb %%mm1, %%mm2				\n\t"

  190. 		"paddb %%mm7, %%mm2				\n\t"

  191. 		"pcmpgtb %%mm6, %%mm2				\n\t"

  192. 		"paddb %%mm2, %%mm0				\n\t"

  193. 

  194. 		"addl %2, %1					\n\t"

  195. 		"movq (%1), %%mm2				\n\t"

  196. 		"psubb %%mm2, %%mm1				\n\t"

  197. 		"paddb %%mm7, %%mm1				\n\t"

  198. 		"pcmpgtb %%mm6, %%mm1				\n\t"

  199. 		"paddb %%mm1, %%mm0				\n\t"

  200. 

  201. 		"addl %2, %1					\n\t"

  202. 		"movq (%1), %%mm1				\n\t"

  203. 		"psubb %%mm1, %%mm2				\n\t"

  204. 		"paddb %%mm7, %%mm2				\n\t"

  205. 		"pcmpgtb %%mm6, %%mm2				\n\t"

  206. 		"paddb %%mm2, %%mm0				\n\t"

  207. 

  208. 		"addl %2, %1					\n\t"

  209. 		"movq (%1), %%mm2				\n\t"

  210. 		"psubb %%mm2, %%mm1				\n\t"

  211. 		"paddb %%mm7, %%mm1				\n\t"

  212. 		"pcmpgtb %%mm6, %%mm1				\n\t"

  213. 		"paddb %%mm1, %%mm0				\n\t"

  214. 

  215. 		"addl %2, %1					\n\t"

  216. 		"movq (%1), %%mm1				\n\t"

  217. 		"psubb %%mm1, %%mm2				\n\t"

  218. 		"paddb %%mm7, %%mm2				\n\t"

  219. 		"pcmpgtb %%mm6, %%mm2				\n\t"

  220. 		"paddb %%mm2, %%mm0				\n\t"

  221. 

  222. 		"						\n\t"

  223. 		"movq %%mm0, %%mm1				\n\t"

  224. 		"psrlw $8, %%mm0				\n\t"

  225. 		"paddb %%mm1, %%mm0				\n\t"

  226. 		"movq %%mm0, %%mm1				\n\t"

  227. 		"psrlq $16, %%mm0				\n\t"

  228. 		"paddb %%mm1, %%mm0				\n\t"

  229. 		"movq %%mm0, %%mm1				\n\t"

  230. 		"psrlq $32, %%mm0				\n\t"

  231. 		"paddb %%mm1, %%mm0				\n\t"

  232. 		"popl %1\n\t"

  233. 		"movd %%mm0, %0					\n\t"

  234. 		: "=r" (numEq)

  235. 		: "r" (src), "r" (stride)

  236. 		);

  237. //	printf("%d\n", numEq);

  238. 	numEq= (256 - (numEq & 0xFF)) &0xFF;

  239. 

  240. //	int asmEq= numEq;

  241. //	numEq=0;

  242. //	uint8_t *temp= src;

  243. 

  244. #else

  245. 	for(int y=0; y<BLOCK_SIZE-1; y++)

  246. 	{

  247. 		if(((src[0] - src[0+stride] + 1)&0xFFFF) < 3) numEq++;

  248. 		if(((src[1] - src[1+stride] + 1)&0xFFFF) < 3) numEq++;

  249. 		if(((src[2] - src[2+stride] + 1)&0xFFFF) < 3) numEq++;

  250. 		if(((src[3] - src[3+stride] + 1)&0xFFFF) < 3) numEq++;

  251. 		if(((src[4] - src[4+stride] + 1)&0xFFFF) < 3) numEq++;

  252. 		if(((src[5] - src[5+stride] + 1)&0xFFFF) < 3) numEq++;

  253. 		if(((src[6] - src[6+stride] + 1)&0xFFFF) < 3) numEq++;

  254. 		if(((src[7] - src[7+stride] + 1)&0xFFFF) < 3) numEq++;

  255. 		src+= stride;

  256. 	}

  257. #endif

  258. /*	if(abs(numEq - asmEq) > 0)

  259. 	{

  260. 		printf("\nasm:%d  c:%d\n", asmEq, numEq);

  261. 		for(int y=0; y<8; y++)

  262. 		{

  263. 			for(int x=0; x<8; x++)

  264. 			{

  265. 				printf("%d ", temp[x + y*stride]);

  266. 			}

  267. 			printf("\n");

  268. 		}

  269. 	}

  270. */

  271. 	return numEq > vFlatnessThreshold;

  272. }

  273. 

  274. static inline bool isVertMinMaxOk(uint8_t src[], int stride, int QP)

  275. {

  276. #ifdef HAVE_MMX

  277. 	int isOk;

  278. 	asm volatile(

  279. //		"int $3 \n\t"

  280. 		"movq (%1, %2), %%mm0				\n\t"

  281. 		"movq (%1, %2, 8), %%mm1			\n\t"

  282. 		"movq %%mm0, %%mm2				\n\t"

  283. 		"psubusb %%mm1, %%mm0				\n\t"

  284. 		"psubusb %%mm2, %%mm1				\n\t"

  285. 		"por %%mm1, %%mm0				\n\t" // ABS Diff

  286. 

  287. 		"movq pQPb, %%mm7				\n\t" // QP,..., QP

  288. 		"paddusb %%mm7, %%mm7				\n\t" // 2QP ... 2QP

  289. 		"psubusb %%mm7, %%mm0				\n\t" // Diff <= 2QP -> 0

  290. 		"pcmpeqd b00, %%mm0				\n\t"

  291. 		"psrlq $16, %%mm0				\n\t"

  292. 		"pcmpeqd bFF, %%mm0				\n\t"

  293. //		"movd %%mm0, (%1, %2, 4)\n\t"

  294. 		"movd %%mm0, %0					\n\t"

  295. 		: "=r" (isOk)

  296. 		: "r" (src), "r" (stride)

  297. 		);

  298. 	return isOk;

  299. #else

  300. 

  301. 	int isOk2= true;

  302. 	for(int x=0; x<BLOCK_SIZE; x++)

  303. 	{

  304. 		if(abs((int)src[x + stride] - (int)src[x + (stride<<3)]) > 2*QP) isOk2=false;

  305. 	}

  306. /*	if(isOk && !isOk2 || !isOk && isOk2)

  307. 	{

  308. 		printf("\nasm:%d  c:%d QP:%d\n", isOk, isOk2, QP);

  309. 		for(int y=0; y<9; y++)

  310. 		{

  311. 			for(int x=0; x<8; x++)

  312. 			{

  313. 				printf("%d ", src[x + y*stride]);

  314. 			}

  315. 			printf("\n");

  316. 		}

  317. 	} */

  318. 

  319. 	return isOk2;

  320. #endif

  321. 

  322. }

  323. 

  324. /**

  325.  * Do a vertical low pass filter on the 8x10 block (only write to the 8x8 block in the middle)

  326.  * useing the 9-Tap Filter (1,1,2,2,4,2,2,1,1)/16

  327.  */

  328. static inline void doVertLowPass(uint8_t *src, int stride, int QP)

  329. {

  330. //	QP= 64;

  331. 

  332. #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)

  333. //#ifdef HAVE_MMX2

  334. 	asm volatile(	//"movv %0 %1 %2\n\t"

  335. 		"pushl %0 \n\t"

  336. 		"movq pQPb, %%mm0				\n\t"  // QP,..., QP

  337. //		"movq bFF  , %%mm0				\n\t"  // QP,..., QP

  338. 

  339. 		"movq (%0), %%mm6				\n\t"

  340. 		"movq (%0, %1), %%mm5				\n\t"

  341. 		"movq %%mm5, %%mm1				\n\t"

  342. 		"movq %%mm6, %%mm2				\n\t"

  343. 		"psubusb %%mm6, %%mm5				\n\t"

  344. 		"psubusb %%mm1, %%mm2				\n\t"

  345. 		"por %%mm5, %%mm2				\n\t" // ABS Diff of lines

  346. 		"psubusb %%mm0, %%mm2				\n\t" // diff <= QP -> 0

  347. 		"pcmpeqb b00, %%mm2				\n\t" // diff <= QP -> FF

  348. 

  349. 		"pand %%mm2, %%mm6				\n\t"

  350. 		"pandn %%mm1, %%mm2				\n\t"

  351. 		"por %%mm2, %%mm6				\n\t"// First Line to Filter

  352. 

  353. 		"movq (%0, %1, 8), %%mm5			\n\t"

  354. 		"leal (%0, %1, 4), %%eax			\n\t"

  355. 		"leal (%0, %1, 8), %%ebx			\n\t"

  356. 		"subl %1, %%ebx					\n\t"

  357. 		"addl %1, %0					\n\t" // %0 points to line 1 not 0

  358. 		"movq (%0, %1, 8), %%mm7			\n\t"

  359. 		"movq %%mm5, %%mm1				\n\t"

  360. 		"movq %%mm7, %%mm2				\n\t"

  361. 		"psubusb %%mm7, %%mm5				\n\t"

  362. 		"psubusb %%mm1, %%mm2				\n\t"

  363. 		"por %%mm5, %%mm2				\n\t" // ABS Diff of lines

  364. 		"psubusb %%mm0, %%mm2				\n\t" // diff <= QP -> 0

  365. 		"pcmpeqb b00, %%mm2				\n\t" // diff <= QP -> FF

  366. 

  367. 		"pand %%mm2, %%mm7				\n\t"

  368. 		"pandn %%mm1, %%mm2				\n\t"

  369. 		"por %%mm2, %%mm7				\n\t" // First Line to Filter

  370. 

  371. 

  372. 		// 	1	2	3	4	5	6	7	8

  373. 		//	%0	%0+%1	%0+2%1	eax	%0+4%1	eax+2%1	ebx	eax+4%1

  374. 		// 6 4 2 2 1 1

  375. 		// 6 4 4 2

  376. 		// 6 8 2

  377. /*

  378. 		"movq %%mm6, %%mm2				\n\t" //1

  379. 		"movq %%mm6, %%mm3				\n\t" //1

  380. 		"paddusb b02, %%mm3 				\n\t"

  381. 		"psrlw $2, %%mm3				\n\t" //1	/4

  382. 		"pand b3F, %%mm3				\n\t"

  383. 		"psubb %%mm3, %%mm2				\n\t"

  384. 		"movq (%0, %1), %%mm0				\n\t" //  1

  385. 		"movq %%mm0, %%mm1				\n\t" //  1

  386. 		"paddusb b02, %%mm0				\n\t"

  387. 		"psrlw $2, %%mm0				\n\t" //  1	/4

  388. 		"pand b3F, %%mm0				\n\t"

  389. 		"paddusb %%mm2, %%mm0				\n\t" //3 1	/4

  390. */

  391. 		"movq (%0, %1), %%mm0				\n\t" //  1

  392. 		"movq %%mm0, %%mm1				\n\t" //  1

  393. 		PAVGB(%%mm6, %%mm0)				      //1 1	/2

  394. 		PAVGB(%%mm6, %%mm0)				      //3 1	/4

  395. 

  396. 		"movq (%0, %1, 4), %%mm2			\n\t" //     1

  397. 		"movq %%mm2, %%mm5				\n\t" //     1

  398. 		PAVGB((%%eax), %%mm2)				      //    11	/2

  399. 		PAVGB((%0, %1, 2), %%mm2)			      //   211	/4

  400. 		"movq %%mm2, %%mm3				\n\t" //   211	/4

  401. 		"movq (%0), %%mm4				\n\t" // 1

  402. 		PAVGB(%%mm4, %%mm3)				      // 4 211	/8

  403. 		PAVGB(%%mm0, %%mm3)				      //642211	/16

  404. 		"movq %%mm3, (%0)				\n\t" // X

  405. 		// mm1=2 mm2=3(211) mm4=1 mm5=5 mm6=0 mm7=9

  406. 		"movq %%mm1, %%mm0				\n\t" //  1

  407. 		PAVGB(%%mm6, %%mm0)				      //1 1	/2

  408. 		"movq %%mm4, %%mm3				\n\t" // 1

  409. 		PAVGB((%0,%1,2), %%mm3)				      // 1 1	/2

  410. 		PAVGB((%%eax,%1,2), %%mm5)			      //     11	/2

  411. 		PAVGB((%%eax), %%mm5)				      //    211 /4

  412. 		PAVGB(%%mm5, %%mm3)				      // 2 2211 /8

  413. 		PAVGB(%%mm0, %%mm3)				      //4242211 /16

  414. 		"movq %%mm3, (%0,%1)				\n\t" //  X

  415. 		// mm1=2 mm2=3(211) mm4=1 mm5=4(211) mm6=0 mm7=9

  416. 		PAVGB(%%mm4, %%mm6)				      //11	/2

  417. 		"movq (%%ebx), %%mm0				\n\t" //       1

  418. 		PAVGB((%%eax, %1, 2), %%mm0)			      //      11/2

  419. 		"movq %%mm0, %%mm3				\n\t" //      11/2

  420. 		PAVGB(%%mm1, %%mm0)				      //  2   11/4

  421. 		PAVGB(%%mm6, %%mm0)				      //222   11/8

  422. 		PAVGB(%%mm2, %%mm0)				      //22242211/16

  423. 		"movq (%0, %1, 2), %%mm2			\n\t" //   1

  424. 		"movq %%mm0, (%0, %1, 2)			\n\t" //   X

  425. 		// mm1=2 mm2=3 mm3=6(11) mm4=1 mm5=4(211) mm6=0(11) mm7=9

  426. 		"movq (%%eax, %1, 4), %%mm0			\n\t" //        1

  427. 		PAVGB((%%ebx), %%mm0)				      //       11	/2

  428. 		PAVGB(%%mm0, %%mm6)				      //11     11	/4

  429. 		PAVGB(%%mm1, %%mm4)				      // 11		/2

  430. 		PAVGB(%%mm2, %%mm1)				      //  11		/2

  431. 		PAVGB(%%mm1, %%mm6)				      //1122   11	/8

  432. 		PAVGB(%%mm5, %%mm6)				      //112242211	/16

  433. 		"movq (%%eax), %%mm5				\n\t" //    1

  434. 		"movq %%mm6, (%%eax)				\n\t" //    X

  435. 		// mm0=7(11) mm1=2(11) mm2=3 mm3=6(11) mm4=1(11) mm5=4 mm7=9

  436. 		"movq (%%eax, %1, 4), %%mm6			\n\t" //        1

  437. 		PAVGB(%%mm7, %%mm6)				      //        11	/2

  438. 		PAVGB(%%mm4, %%mm6)				      // 11     11	/4

  439. 		PAVGB(%%mm3, %%mm6)				      // 11   2211	/8

  440. 		PAVGB(%%mm5, %%mm2)				      //   11		/2

  441. 		"movq (%0, %1, 4), %%mm4			\n\t" //     1

  442. 		PAVGB(%%mm4, %%mm2)				      //   112		/4

  443. 		PAVGB(%%mm2, %%mm6)				      // 112242211	/16

  444. 		"movq %%mm6, (%0, %1, 4)			\n\t" //     X

  445. 		// mm0=7(11) mm1=2(11) mm2=3(112) mm3=6(11) mm4=5 mm5=4 mm7=9

  446. 		PAVGB(%%mm7, %%mm1)				      //  11     2	/4

  447. 		PAVGB(%%mm4, %%mm5)				      //    11		/2

  448. 		PAVGB(%%mm5, %%mm0)				      //    11 11	/4

  449. 		"movq (%%eax, %1, 2), %%mm6			\n\t" //      1

  450. 		PAVGB(%%mm6, %%mm1)				      //  11  4  2	/8

  451. 		PAVGB(%%mm0, %%mm1)				      //  11224222	/16

  452. //		"pxor %%mm1, %%mm1 \n\t"

  453. 		"movq %%mm1, (%%eax, %1, 2)			\n\t" //      X

  454. 		// mm2=3(112) mm3=6(11) mm4=5 mm5=4(11) mm6=6 mm7=9

  455. 		PAVGB((%%ebx), %%mm2)				      //   112 4	/8

  456. 		"movq (%%eax, %1, 4), %%mm0			\n\t" //        1

  457. 		PAVGB(%%mm0, %%mm6)				      //      1 1	/2

  458. 		PAVGB(%%mm7, %%mm6)				      //      1 12	/4

  459. 		PAVGB(%%mm2, %%mm6)				      //   1122424	/4

  460. //		"pxor %%mm6, %%mm6 \n\t"

  461. 		"movq %%mm6, (%%ebx)				\n\t" //       X

  462. 		// mm0=8 mm3=6(11) mm4=5 mm5=4(11) mm7=9

  463. 		PAVGB(%%mm7, %%mm5)				      //    11   2	/4

  464. 		PAVGB(%%mm7, %%mm5)				      //    11   6	/8

  465. 

  466. 		PAVGB(%%mm3, %%mm0)				      //      112	/4

  467. 		PAVGB(%%mm0, %%mm5)				      //    112246	/16

  468. //		"pxor %%mm5, %%mm5 \n\t"

  469. //		"movq pQPb, %%mm5 \n\t"

  470. 		"movq %%mm5, (%%eax, %1, 4)			\n\t" //        X

  471. 		"popl %0\n\t"

  472. 

  473. 		:

  474. 		: "r" (src), "r" (stride)

  475. 		: "%eax", "%ebx"

  476. 	);

  477. #else

  478. 	const int l1= stride;

  479. 	const int l2= stride + l1;

  480. 	const int l3= stride + l2;

  481. 	const int l4= stride + l3;

  482. 	const int l5= stride + l4;

  483. 	const int l6= stride + l5;

  484. 	const int l7= stride + l6;

  485. 	const int l8= stride + l7;

  486. 	const int l9= stride + l8;

  487. 

  488. 	for(int x=0; x<BLOCK_SIZE; x++)

  489. 	{

  490. 		const int first= ABS(src[0] - src[l1]) < QP ? src[0] : src[l1];

  491. 		const int last= ABS(src[l8] - src[l9]) < QP ? src[l9] : src[l8];

  492. 

  493. 		int sums[9];

  494. 		sums[0] = first + src[l1];

  495. 		sums[1] = src[l1] + src[l2];

  496. 		sums[2] = src[l2] + src[l3];

  497. 		sums[3] = src[l3] + src[l4];

  498. 		sums[4] = src[l4] + src[l5];

  499. 		sums[5] = src[l5] + src[l6];

  500. 		sums[6] = src[l6] + src[l7];

  501. 		sums[7] = src[l7] + src[l8];

  502. 		sums[8] = src[l8] + last;

  503. 

  504. 		src[l1]= ((sums[0]<<2) + ((first + sums[2])<<1) + sums[4] + 8)>>4;

  505. 		src[l2]= ((src[l2]<<2) + (first + sums[0] + sums[3]<<1) + sums[5] + 8)>>4;

  506. 		src[l3]= ((src[l3]<<2) + (first + sums[1] + sums[4]<<1) + sums[6] + 8)>>4;

  507. 		src[l4]= ((src[l4]<<2) + (sums[2] + sums[5]<<1) + sums[0] + sums[7] + 8)>>4;

  508. 		src[l5]= ((src[l5]<<2) + (sums[3] + sums[6]<<1) + sums[1] + sums[8] + 8)>>4;

  509. 		src[l6]= ((src[l6]<<2) + (last + sums[7] + sums[4]<<1) + sums[2] + 8)>>4;

  510. 		src[l7]= ((last + src[l7]<<2) + (src[l8] + sums[5]<<1) + sums[3] + 8)>>4;

  511. 		src[l8]= ((sums[8]<<2) + (last + sums[6]<<1) + sums[4] + 8)>>4;

  512. 

  513. 		src++;

  514. 	}

  515. 

  516. #endif

  517. }

  518. 

  519. /**

  520.  * Experimental implementation of the filter (Algorithm 1) described in a paper from Ramkishor & Karandikar

  521.  * values are correctly clipped (MMX2)

  522.  * values are wraparound (C)

  523.  * conclusion: its fast, but introduces ugly horizontal patterns if there is a continious gradient

  524. 	0 8 16 24

  525. 	x = 8

  526. 	x/2 = 4

  527. 	x/8 = 1

  528. 	1 12 12 23

  529.  */

  530. static inline void vertRKFilter(uint8_t *src, int stride, int QP)

  531. {

  532. #ifdef HAVE_MMX2

  533. // FIXME rounding

  534. 	asm volatile(

  535. 		"pxor %%mm7, %%mm7				\n\t" // 0

  536. 		"movq b80, %%mm6				\n\t" // MIN_SIGNED_BYTE

  537. 		"leal (%0, %1), %%eax				\n\t"

  538. 		"leal (%%eax, %1, 4), %%ebx			\n\t"

  539. //	0	1	2	3	4	5	6	7	8	9

  540. //	%0	eax	eax+%1	eax+2%1	%0+4%1	ebx	ebx+%1	ebx+2%1	%0+8%1	ebx+4%1

  541. 		"movq pQPb, %%mm0				\n\t" // QP,..., QP

  542. 		"movq %%mm0, %%mm1				\n\t" // QP,..., QP

  543. 		"paddusb b02, %%mm0				\n\t"

  544. 		"psrlw $2, %%mm0				\n\t"

  545. 		"pand b3F, %%mm0				\n\t" // QP/4,..., QP/4

  546. 		"paddusb %%mm1, %%mm0				\n\t" // QP*1.25 ...

  547. 		"movq (%0, %1, 4), %%mm2			\n\t" // line 4

  548. 		"movq (%%ebx), %%mm3				\n\t" // line 5

  549. 		"movq %%mm2, %%mm4				\n\t" // line 4

  550. 		"pcmpeqb %%mm5, %%mm5				\n\t" // -1

  551. 		"pxor %%mm2, %%mm5				\n\t" // -line 4 - 1

  552. 		"pavgb %%mm3, %%mm5				\n\t"

  553. 		"paddb %%mm6, %%mm5				\n\t" // (l5-l4)/2

  554. 		"psubusb %%mm3, %%mm4				\n\t"

  555. 		"psubusb %%mm2, %%mm3				\n\t"

  556. 		"por %%mm3, %%mm4				\n\t" // |l4 - l5|

  557. 		"psubusb %%mm0, %%mm4				\n\t"

  558. 		"pcmpeqb %%mm7, %%mm4				\n\t"

  559. 		"pand %%mm4, %%mm5				\n\t" // d/2

  560. 

  561. //		"paddb %%mm6, %%mm2				\n\t" // line 4 + 0x80

  562. 		"paddb %%mm5, %%mm2				\n\t"

  563. //		"psubb %%mm6, %%mm2				\n\t"

  564. 		"movq %%mm2, (%0,%1, 4)				\n\t"

  565. 

  566. 		"movq (%%ebx), %%mm2				\n\t"

  567. //		"paddb %%mm6, %%mm2				\n\t" // line 5 + 0x80

  568. 		"psubb %%mm5, %%mm2				\n\t"

  569. //		"psubb %%mm6, %%mm2				\n\t"

  570. 		"movq %%mm2, (%%ebx)				\n\t"

  571. 

  572. 		"paddb %%mm6, %%mm5				\n\t"

  573. 		"psrlw $2, %%mm5				\n\t"

  574. 		"pand b3F, %%mm5				\n\t"

  575. 		"psubb b20, %%mm5				\n\t" // (l5-l4)/8

  576. 

  577. 		"movq (%%eax, %1, 2), %%mm2			\n\t"

  578. 		"paddb %%mm6, %%mm2				\n\t" // line 3 + 0x80

  579. 		"paddsb %%mm5, %%mm2				\n\t"

  580. 		"psubb %%mm6, %%mm2				\n\t"

  581. 		"movq %%mm2, (%%eax, %1, 2)			\n\t"

  582. 

  583. 		"movq (%%ebx, %1), %%mm2			\n\t"

  584. 		"paddb %%mm6, %%mm2				\n\t" // line 6 + 0x80

  585. 		"psubsb %%mm5, %%mm2				\n\t"

  586. 		"psubb %%mm6, %%mm2				\n\t"

  587. 		"movq %%mm2, (%%ebx, %1)			\n\t"

  588. 

  589. 		:

  590. 		: "r" (src), "r" (stride)

  591. 		: "%eax", "%ebx"

  592. 	);

  593. #else

  594.  	const int l1= stride;

  595. 	const int l2= stride + l1;

  596. 	const int l3= stride + l2;

  597. 	const int l4= stride + l3;

  598. 	const int l5= stride + l4;

  599. 	const int l6= stride + l5;

  600. 	const int l7= stride + l6;

  601. 	const int l8= stride + l7;

  602. 	const int l9= stride + l8;

  603. 	for(int x=0; x<BLOCK_SIZE; x++)

  604. 	{

  605. 		if(ABS(src[l4]-src[l5]) < QP + QP/4)

  606. 		{

  607. 			int x = src[l5] - src[l4];

  608. 

  609. 			src[l3] +=x/8;

  610. 			src[l4] +=x/2;

  611. 			src[l5] -=x/2;

  612. 			src[l6] -=x/8;

  613. 		}

  614. 		src++;

  615. 	}

  616. 

  617. #endif

  618. }

  619. 

  620. /**

  621.  * Experimental Filter 1

  622.  */

  623. static inline void vertX1Filter(uint8_t *src, int stride, int QP)

  624. {

  625. #ifdef HAVE_MMX2X

  626. // FIXME

  627. 	asm volatile(

  628. 

  629. 		:

  630. 		: "r" (src), "r" (stride)

  631. 		: "%eax", "%ebx"

  632. 	);

  633. #else

  634.  	const int l1= stride;

  635. 	const int l2= stride + l1;

  636. 	const int l3= stride + l2;

  637. 	const int l4= stride + l3;

  638. 	const int l5= stride + l4;

  639. 	const int l6= stride + l5;

  640. 	const int l7= stride + l6;

  641. 	const int l8= stride + l7;

  642. 	const int l9= stride + l8;

  643. 	for(int x=0; x<BLOCK_SIZE; x++)

  644. 	{

  645. 		int v2= src[l2];

  646. 		int v3= src[l3];

  647. 		int v4= src[l4];

  648. 		int v5= src[l5];

  649. 		int v6= src[l6];

  650. 		int v7= src[l7];

  651. 

  652. 		if(ABS(v4-v5)<QP &&  ABS(v4-v5) - (ABS(v3-v4) + ABS(v5-v6))>0 )

  653. 		{

  654. 			src[l3] = (6*v2 + 4*v3 + 3*v4 + 2*v5 + v6         )/16;

  655. 			src[l4] = (3*v2 + 3*v3 + 4*v4 + 3*v5 + 2*v6 + v7  )/16;

  656. 			src[l5] = (1*v2 + 2*v3 + 3*v4 + 4*v5 + 3*v6 + 3*v7)/16;

  657. 			src[l6] = (       1*v3 + 2*v4 + 3*v5 + 4*v6 + 6*v7)/16;

  658. 		}

  659. 		src++;

  660. 	}

  661. 

  662. #endif

  663. }

  664. 

  665. 

  666. static inline void doVertDefFilter(uint8_t src[], int stride, int QP)

  667. {

  668. #ifdef HAVE_MMX

  669. 	src+= stride;

  670. 	//FIXME try pmul for *5 stuff

  671. //	src[0]=0;

  672. 	asm volatile(

  673. 		"pxor %%mm7, %%mm7				\n\t"

  674. 		"leal (%0, %1), %%eax				\n\t"

  675. 		"leal (%%eax, %1, 4), %%ebx			\n\t"

  676. //	0	1	2	3	4	5	6	7

  677. //	%0	%0+%1	%0+2%1	eax+2%1	%0+4%1	eax+4%1	ebx+%1	ebx+2%1

  678. //	%0	eax	eax+%1	eax+2%1	%0+4%1	ebx	ebx+%1	ebx+2%1

  679. 

  680. 		"movq (%0), %%mm0				\n\t"

  681. 		"movq %%mm0, %%mm1				\n\t"

  682. 		"punpcklbw %%mm7, %%mm0				\n\t" // low part of line 0

  683. 		"punpckhbw %%mm7, %%mm1				\n\t" // high part of line 0

  684. 

  685. 		"movq (%%eax), %%mm2				\n\t"

  686. 		"movq %%mm2, %%mm3				\n\t"

  687. 		"punpcklbw %%mm7, %%mm2				\n\t" // low part of line 1

  688. 		"punpckhbw %%mm7, %%mm3				\n\t" // high part of line 1

  689. 

  690. 		"movq (%%eax, %1), %%mm4			\n\t"

  691. 		"movq %%mm4, %%mm5				\n\t"

  692. 		"punpcklbw %%mm7, %%mm4				\n\t" // low part of line 2

  693. 		"punpckhbw %%mm7, %%mm5				\n\t" // high part of line 2

  694. 

  695. 		"paddw %%mm0, %%mm0				\n\t" // 2L0

  696. 		"paddw %%mm1, %%mm1				\n\t" // 2H0

  697. 		"psubw %%mm4, %%mm2				\n\t" // L1 - L2

  698. 		"psubw %%mm5, %%mm3				\n\t" // H1 - H2

  699. 		"psubw %%mm2, %%mm0				\n\t" // 2L0 - L1 + L2

  700. 		"psubw %%mm3, %%mm1				\n\t" // 2H0 - H1 + H2

  701. 

  702. 		"psllw $2, %%mm2				\n\t" // 4L1 - 4L2

  703. 		"psllw $2, %%mm3				\n\t" // 4H1 - 4H2

  704. 		"psubw %%mm2, %%mm0				\n\t" // 2L0 - 5L1 + 5L2

  705. 		"psubw %%mm3, %%mm1				\n\t" // 2H0 - 5H1 + 5H2

  706. 

  707. 		"movq (%%eax, %1, 2), %%mm2			\n\t"

  708. 		"movq %%mm2, %%mm3				\n\t"

  709. 		"punpcklbw %%mm7, %%mm2				\n\t" // L3

  710. 		"punpckhbw %%mm7, %%mm3				\n\t" // H3

  711. 

  712. 		"psubw %%mm2, %%mm0				\n\t" // 2L0 - 5L1 + 5L2 - L3

  713. 		"psubw %%mm3, %%mm1				\n\t" // 2H0 - 5H1 + 5H2 - H3

  714. 		"psubw %%mm2, %%mm0				\n\t" // 2L0 - 5L1 + 5L2 - 2L3

  715. 		"psubw %%mm3, %%mm1				\n\t" // 2H0 - 5H1 + 5H2 - 2H3

  716. 		"movq %%mm0, temp0				\n\t" // 2L0 - 5L1 + 5L2 - 2L3

  717. 		"movq %%mm1, temp1				\n\t" // 2H0 - 5H1 + 5H2 - 2H3

  718. 

  719. 		"movq (%0, %1, 4), %%mm0			\n\t"

  720. 		"movq %%mm0, %%mm1				\n\t"

  721. 		"punpcklbw %%mm7, %%mm0				\n\t" // L4

  722. 		"punpckhbw %%mm7, %%mm1				\n\t" // H4

  723. 

  724. 		"psubw %%mm0, %%mm2				\n\t" // L3 - L4

  725. 		"psubw %%mm1, %%mm3				\n\t" // H3 - H4

  726. 		"movq %%mm2, temp2				\n\t" // L3 - L4

  727. 		"movq %%mm3, temp3				\n\t" // H3 - H4

  728. 		"paddw %%mm4, %%mm4				\n\t" // 2L2

  729. 		"paddw %%mm5, %%mm5				\n\t" // 2H2

  730. 		"psubw %%mm2, %%mm4				\n\t" // 2L2 - L3 + L4

  731. 		"psubw %%mm3, %%mm5				\n\t" // 2H2 - H3 + H4

  732. 

  733. 		"psllw $2, %%mm2				\n\t" // 4L3 - 4L4

  734. 		"psllw $2, %%mm3				\n\t" // 4H3 - 4H4

  735. 		"psubw %%mm2, %%mm4				\n\t" // 2L2 - 5L3 + 5L4

  736. 		"psubw %%mm3, %%mm5				\n\t" // 2H2 - 5H3 + 5H4

  737. //50 opcodes so far

  738. 		"movq (%%ebx), %%mm2				\n\t"

  739. 		"movq %%mm2, %%mm3				\n\t"

  740. 		"punpcklbw %%mm7, %%mm2				\n\t" // L5

  741. 		"punpckhbw %%mm7, %%mm3				\n\t" // H5

  742. 		"psubw %%mm2, %%mm4				\n\t" // 2L2 - 5L3 + 5L4 - L5

  743. 		"psubw %%mm3, %%mm5				\n\t" // 2H2 - 5H3 + 5H4 - H5

  744. 		"psubw %%mm2, %%mm4				\n\t" // 2L2 - 5L3 + 5L4 - 2L5

  745. 		"psubw %%mm3, %%mm5				\n\t" // 2H2 - 5H3 + 5H4 - 2H5

  746. 

  747. 		"movq (%%ebx, %1), %%mm6			\n\t"

  748. 		"punpcklbw %%mm7, %%mm6				\n\t" // L6

  749. 		"psubw %%mm6, %%mm2				\n\t" // L5 - L6

  750. 		"movq (%%ebx, %1), %%mm6			\n\t"

  751. 		"punpckhbw %%mm7, %%mm6				\n\t" // H6

  752. 		"psubw %%mm6, %%mm3				\n\t" // H5 - H6

  753. 

  754. 		"paddw %%mm0, %%mm0				\n\t" // 2L4

  755. 		"paddw %%mm1, %%mm1				\n\t" // 2H4

  756. 		"psubw %%mm2, %%mm0				\n\t" // 2L4 - L5 + L6

  757. 		"psubw %%mm3, %%mm1				\n\t" // 2H4 - H5 + H6

  758. 

  759. 		"psllw $2, %%mm2				\n\t" // 4L5 - 4L6

  760. 		"psllw $2, %%mm3				\n\t" // 4H5 - 4H6

  761. 		"psubw %%mm2, %%mm0				\n\t" // 2L4 - 5L5 + 5L6

  762. 		"psubw %%mm3, %%mm1				\n\t" // 2H4 - 5H5 + 5H6

  763. 

  764. 		"movq (%%ebx, %1, 2), %%mm2			\n\t"

  765. 		"movq %%mm2, %%mm3				\n\t"

  766. 		"punpcklbw %%mm7, %%mm2				\n\t" // L7

  767. 		"punpckhbw %%mm7, %%mm3				\n\t" // H7

  768. 

  769. 		"paddw %%mm2, %%mm2				\n\t" // 2L7

  770. 		"paddw %%mm3, %%mm3				\n\t" // 2H7

  771. 		"psubw %%mm2, %%mm0				\n\t" // 2L4 - 5L5 + 5L6 - 2L7

  772. 		"psubw %%mm3, %%mm1				\n\t" // 2H4 - 5H5 + 5H6 - 2H7

  773. 

  774. 		"movq temp0, %%mm2				\n\t" // 2L0 - 5L1 + 5L2 - 2L3

  775. 		"movq temp1, %%mm3				\n\t" // 2H0 - 5H1 + 5H2 - 2H3

  776. //FIXME pxor, psubw, pmax for abs

  777. 		"movq %%mm7, %%mm6				\n\t" // 0

  778. 		"pcmpgtw %%mm0, %%mm6				\n\t"

  779. 		"pxor %%mm6, %%mm0				\n\t"

  780. 		"psubw %%mm6, %%mm0				\n\t" // |2L4 - 5L5 + 5L6 - 2L7|

  781. 		"movq %%mm7, %%mm6				\n\t" // 0

  782. 		"pcmpgtw %%mm1, %%mm6				\n\t"

  783. 		"pxor %%mm6, %%mm1				\n\t"

  784. 		"psubw %%mm6, %%mm1				\n\t" // |2H4 - 5H5 + 5H6 - 2H7|

  785. 

  786. 		"movq %%mm7, %%mm6				\n\t" // 0

  787. 		"pcmpgtw %%mm2, %%mm6				\n\t"

  788. 		"pxor %%mm6, %%mm2				\n\t"

  789. 		"psubw %%mm6, %%mm2				\n\t" // |2L0 - 5L1 + 5L2 - 2L3|

  790. 		"movq %%mm7, %%mm6				\n\t" // 0

  791. 		"pcmpgtw %%mm3, %%mm6				\n\t"

  792. 		"pxor %%mm6, %%mm3				\n\t"

  793. 		"psubw %%mm6, %%mm3				\n\t" // |2H0 - 5H1 + 5H2 - 2H3|

  794. 

  795. #ifdef HAVE_MMX2

  796. 		"pminsw %%mm2, %%mm0				\n\t"

  797. 		"pminsw %%mm3, %%mm1				\n\t"

  798. #else

  799. 		"movq %%mm0, %%mm6				\n\t"

  800. 		"psubusw %%mm2, %%mm6				\n\t"

  801. 		"psubw %%mm6, %%mm0				\n\t"

  802. 		"movq %%mm1, %%mm6				\n\t"

  803. 		"psubusw %%mm3, %%mm6				\n\t"

  804. 		"psubw %%mm6, %%mm1				\n\t"

  805. #endif

  806. 

  807. 		"movq %%mm7, %%mm6				\n\t" // 0

  808. 		"pcmpgtw %%mm4, %%mm6				\n\t" // sign(2L2 - 5L3 + 5L4 - 2L5)

  809. 		"pxor %%mm6, %%mm4				\n\t"

  810. 		"psubw %%mm6, %%mm4				\n\t" // |2L2 - 5L3 + 5L4 - 2L5|

  811. 		"pcmpgtw %%mm5, %%mm7				\n\t" // sign(2H2 - 5H3 + 5H4 - 2H5)

  812. 		"pxor %%mm7, %%mm5				\n\t"

  813. 		"psubw %%mm7, %%mm5				\n\t" // |2H2 - 5H3 + 5H4 - 2H5|

  814. // 100 opcodes

  815. 		"movd %2, %%mm2					\n\t" // QP

  816. //"pcmpeqb %%mm2, %%mm2\n\t"

  817. 		"punpcklwd %%mm2, %%mm2				\n\t"

  818. 		"punpcklwd %%mm2, %%mm2				\n\t"

  819. 		"psllw $3, %%mm2				\n\t" // 8QP

  820. 		"movq %%mm2, %%mm3				\n\t" // 8QP

  821. 		"pcmpgtw %%mm4, %%mm2				\n\t"

  822. 		"pcmpgtw %%mm5, %%mm3				\n\t"

  823. 		"pand %%mm2, %%mm4				\n\t"

  824. 		"pand %%mm3, %%mm5				\n\t"

  825. 

  826. 

  827. 		"psubusw %%mm0, %%mm4				\n\t" // hd

  828. 		"psubusw %%mm1, %%mm5				\n\t" // ld

  829. 

  830. 

  831. 		"movq w05, %%mm2				\n\t" // 5

  832. 		"pmullw %%mm2, %%mm4				\n\t"

  833. 		"pmullw %%mm2, %%mm5				\n\t"

  834. 		"movq w20, %%mm2				\n\t" // 32

  835. 		"paddw %%mm2, %%mm4				\n\t"

  836. 		"paddw %%mm2, %%mm5				\n\t"

  837. 		"psrlw $6, %%mm4				\n\t"

  838. 		"psrlw $6, %%mm5				\n\t"

  839. 

  840. /*

  841. 		"movq w06, %%mm2				\n\t" // 6

  842. 		"paddw %%mm2, %%mm4				\n\t"

  843. 		"paddw %%mm2, %%mm5				\n\t"

  844. 		"movq w1400, %%mm2				\n\t" // 1400h = 5120 = 5/64*2^16

  845. //FIXME if *5/64 is supposed to be /13 then we should use 5041 instead of 5120

  846. 		"pmulhw %%mm2, %%mm4				\n\t" // hd/13

  847. 		"pmulhw %%mm2, %%mm5				\n\t" // ld/13

  848. */

  849. 

  850. 		"movq temp2, %%mm0				\n\t" // L3 - L4

  851. 		"movq temp3, %%mm1				\n\t" // H3 - H4

  852. 

  853. 		"pxor %%mm2, %%mm2				\n\t"

  854. 		"pxor %%mm3, %%mm3				\n\t"

  855. 

  856. 		// FIXME rounding error

  857. 		"psraw $1, %%mm0				\n\t" // (L3 - L4)/2

  858. 		"psraw $1, %%mm1				\n\t" // (H3 - H4)/2

  859. 		"pcmpgtw %%mm0, %%mm2				\n\t" // sign (L3-L4)

  860. 		"pcmpgtw %%mm1, %%mm3				\n\t" // sign (H3-H4)

  861. 		"pxor %%mm2, %%mm0				\n\t"

  862. 		"pxor %%mm3, %%mm1				\n\t"

  863. 		"psubw %%mm2, %%mm0				\n\t" // |L3-L4|

  864. 		"psubw %%mm3, %%mm1				\n\t" // |H3-H4|

  865. //		"psrlw $1, %%mm0				\n\t" // |L3 - L4|/2

  866. //		"psrlw $1, %%mm1				\n\t" // |H3 - H4|/2

  867. 

  868. 		"pxor %%mm6, %%mm2				\n\t"

  869. 		"pxor %%mm7, %%mm3				\n\t"

  870. 		"pand %%mm2, %%mm4				\n\t"

  871. 		"pand %%mm3, %%mm5				\n\t"

  872. 

  873. #ifdef HAVE_MMX2

  874. 		"pminsw %%mm0, %%mm4				\n\t"

  875. 		"pminsw %%mm1, %%mm5				\n\t"

  876. #else

  877. 		"movq %%mm4, %%mm2				\n\t"

  878. 		"psubusw %%mm0, %%mm2				\n\t"

  879. 		"psubw %%mm2, %%mm4				\n\t"

  880. 		"movq %%mm5, %%mm2				\n\t"

  881. 		"psubusw %%mm1, %%mm2				\n\t"

  882. 		"psubw %%mm2, %%mm5				\n\t"

  883. #endif

  884. 		"pxor %%mm6, %%mm4				\n\t"

  885. 		"pxor %%mm7, %%mm5				\n\t"

  886. 		"psubw %%mm6, %%mm4				\n\t"

  887. 		"psubw %%mm7, %%mm5				\n\t"

  888. 		"packsswb %%mm5, %%mm4				\n\t"

  889. 		"movq (%%eax, %1, 2), %%mm0			\n\t"

  890. 		"paddb   %%mm4, %%mm0				\n\t"

  891. 		"movq %%mm0, (%%eax, %1, 2) 			\n\t"

  892. 		"movq (%0, %1, 4), %%mm0			\n\t"

  893. 		"psubb %%mm4, %%mm0				\n\t"

  894. //		"pxor %%mm0, %%mm0 \n\t"

  895. 		"movq %%mm0, (%0, %1, 4) 			\n\t"

  896. 

  897. 		:

  898. 		: "r" (src), "r" (stride), "r" (QP)

  899. 		: "%eax", "%ebx"

  900. 	);

  901. #else

  902. 	const int l1= stride;

  903. 	const int l2= stride + l1;

  904. 	const int l3= stride + l2;

  905. 	const int l4= stride + l3;

  906. 	const int l5= stride + l4;

  907. 	const int l6= stride + l5;

  908. 	const int l7= stride + l6;

  909. 	const int l8= stride + l7;

  910. //	const int l9= stride + l8;

  911. 

  912. 	for(int x=0; x<BLOCK_SIZE; x++)

  913. 	{

  914. 		const int middleEnergy= 5*(src[l5] - src[l4]) + 2*(src[l3] - src[l6]);

  915. 		if(ABS(middleEnergy) < 8*QP)

  916. 		{

  917. 			const int q=(src[l4] - src[l5])/2;

  918. 			const int leftEnergy=  5*(src[l3] - src[l2]) + 2*(src[l1] - src[l4]);

  919. 			const int rightEnergy= 5*(src[l7] - src[l6]) + 2*(src[l5] - src[l8]);

  920. 

  921. 			int d= ABS(middleEnergy) - MIN( ABS(leftEnergy), ABS(rightEnergy) );

  922. 			d= MAX(d, 0);

  923. 

  924. 			d= (5*d + 32) >> 6;

  925. 			d*= SIGN(-middleEnergy);

  926. 

  927. 			if(q>0)

  928. 			{

  929. 				d= d<0 ? 0 : d;

  930. 				d= d>q ? q : d;

  931. 			}

  932. 			else

  933. 			{

  934. 				d= d>0 ? 0 : d;

  935. 				d= d<q ? q : d;

  936. 			}

  937. 

  938.         		src[l4]-= d;

  939. 	        	src[l5]+= d;

  940. 		}

  941. 		src++;

  942. 	}

  943. #endif

  944. }

  945. 

  946. //FIXME?  |255-0| = 1

  947. /**

  948.  * Check if the given 8x8 Block is mostly "flat" and copy the unaliged data into tempBlock.

  949.  */

  950. static inline bool isHorizDCAndCopy2Temp(uint8_t src[], int stride)

  951. {

  952. //	src++;

  953. 	int numEq= 0;

  954. #ifdef HAVE_MMX

  955. asm volatile (

  956. //		"int $3 \n\t"

  957. 		"pushl %1\n\t"

  958. 		"movq b7E, %%mm7				\n\t" // mm7 = 0x7F

  959. 		"movq b7C, %%mm6				\n\t" // mm6 = 0x7D

  960. 		"leal tempBlock, %%eax				\n\t"

  961. 		"pxor %%mm0, %%mm0				\n\t"

  962. 

  963. #define HDC_CHECK_AND_CPY(i) \

  964. 		"movq -4(%1), %%mm2				\n\t"\

  965. 		"psrlq $32, %%mm2				\n\t"\

  966. 		"punpckldq 4(%1), %%mm2				\n\t" /* (%1) */\

  967. 		"movq %%mm2, %%mm1				\n\t"\

  968. 		"psrlq $8, %%mm2				\n\t"\

  969. 		"psubb %%mm1, %%mm2				\n\t"\

  970. 		"paddb %%mm7, %%mm2				\n\t"\

  971. 		"pcmpgtb %%mm6, %%mm2				\n\t"\

  972. 		"paddb %%mm2, %%mm0				\n\t"\

  973. 		"movq %%mm1," #i "(%%eax)			\n\t"

  974. 

  975. 		HDC_CHECK_AND_CPY(0)

  976. 		"addl %2, %1					\n\t"

  977. 		HDC_CHECK_AND_CPY(8)

  978. 		"addl %2, %1					\n\t"

  979. 		HDC_CHECK_AND_CPY(16)

  980. 		"addl %2, %1					\n\t"

  981. 		HDC_CHECK_AND_CPY(24)

  982. 		"addl %2, %1					\n\t"

  983. 		HDC_CHECK_AND_CPY(32)

  984. 		"addl %2, %1					\n\t"

  985. 		HDC_CHECK_AND_CPY(40)

  986. 		"addl %2, %1					\n\t"

  987. 		HDC_CHECK_AND_CPY(48)

  988. 		"addl %2, %1					\n\t"

  989. 		HDC_CHECK_AND_CPY(56)

  990. 

  991. 		"psllq $8, %%mm0				\n\t" // remove dummy value

  992. 		"movq %%mm0, %%mm1				\n\t"

  993. 		"psrlw $8, %%mm0				\n\t"

  994. 		"paddb %%mm1, %%mm0				\n\t"

  995. 		"movq %%mm0, %%mm1				\n\t"

  996. 		"psrlq $16, %%mm0				\n\t"

  997. 		"paddb %%mm1, %%mm0				\n\t"

  998. 		"movq %%mm0, %%mm1				\n\t"

  999. 		"psrlq $32, %%mm0				\n\t"

  1000. 		"paddb %%mm1, %%mm0				\n\t"

  1001. 		"popl %1\n\t"

  1002. 		"movd %%mm0, %0					\n\t"

  1003. 		: "=r" (numEq)

  1004. 		: "r" (src), "r" (stride)

  1005. 		: "%eax"

  1006. 		);

  1007. //	printf("%d\n", numEq);

  1008. 	numEq= (256 - (numEq & 0xFF)) &0xFF;

  1009. #else

  1010. 	for(int y=0; y<BLOCK_SIZE; y++)

  1011. 	{

  1012. 		if(((src[0] - src[1] + 1) & 0xFFFF) < 3) numEq++;

  1013. 		if(((src[1] - src[2] + 1) & 0xFFFF) < 3) numEq++;

  1014. 		if(((src[2] - src[3] + 1) & 0xFFFF) < 3) numEq++;

  1015. 		if(((src[3] - src[4] + 1) & 0xFFFF) < 3) numEq++;

  1016. 		if(((src[4] - src[5] + 1) & 0xFFFF) < 3) numEq++;

  1017. 		if(((src[5] - src[6] + 1) & 0xFFFF) < 3) numEq++;

  1018. 		if(((src[6] - src[7] + 1) & 0xFFFF) < 3) numEq++;

  1019. 		tempBlock[0 + y*TEMP_STRIDE] = src[0];

  1020. 		tempBlock[1 + y*TEMP_STRIDE] = src[1];

  1021. 		tempBlock[2 + y*TEMP_STRIDE] = src[2];

  1022. 		tempBlock[3 + y*TEMP_STRIDE] = src[3];

  1023. 		tempBlock[4 + y*TEMP_STRIDE] = src[4];

  1024. 		tempBlock[5 + y*TEMP_STRIDE] = src[5];

  1025. 		tempBlock[6 + y*TEMP_STRIDE] = src[6];

  1026. 		tempBlock[7 + y*TEMP_STRIDE] = src[7];

  1027. 		src+= stride;

  1028. 	}

  1029. #endif

  1030. /*	if(abs(numEq - asmEq) > 0)

  1031. 	{

  1032. //		printf("\nasm:%d  c:%d\n", asmEq, numEq);

  1033. 		for(int y=0; y<8; y++)

  1034. 		{

  1035. 			for(int x=0; x<8; x++)

  1036. 			{

  1037. 				printf("%d ", src[x + y*stride]);

  1038. 			}

  1039. 			printf("\n");

  1040. 		}

  1041. 	}

  1042. */

  1043. //	printf("%d\n", numEq);

  1044. 	return numEq > hFlatnessThreshold;

  1045. }

  1046. 

  1047. static inline bool isHorizMinMaxOk(uint8_t src[], int stride, int QP)

  1048. {

  1049. #ifdef MMX_FIXME

  1050. FIXME

  1051. 	int isOk;

  1052. 	asm volatile(

  1053. //		"int $3 \n\t"

  1054. 		"movq (%1, %2), %%mm0				\n\t"

  1055. 		"movq (%1, %2, 8), %%mm1			\n\t"

  1056. 		"movq %%mm0, %%mm2				\n\t"

  1057. 		"psubusb %%mm1, %%mm0				\n\t"

  1058. 		"psubusb %%mm2, %%mm1				\n\t"

  1059. 		"por %%mm1, %%mm0				\n\t" // ABS Diff

  1060. 

  1061. 		"movq pQPb, %%mm7				\n\t" // QP,..., QP

  1062. 		"paddusb %%mm7, %%mm7				\n\t" // 2QP ... 2QP

  1063. 		"psubusb %%mm7, %%mm0				\n\t" // Diff <= 2QP -> 0

  1064. 		"pcmpeqd b00, %%mm0				\n\t"

  1065. 		"psrlq $16, %%mm0				\n\t"

  1066. 		"pcmpeqd bFF, %%mm0				\n\t"

  1067. //		"movd %%mm0, (%1, %2, 4)\n\t"

  1068. 		"movd %%mm0, %0					\n\t"

  1069. 		: "=r" (isOk)

  1070. 		: "r" (src), "r" (stride)

  1071. 		);

  1072. 	return isOk;

  1073. #else

  1074. 	if(abs(src[0] - src[7]) > 2*QP) return false;

  1075. 

  1076. 	return true;

  1077. #endif

  1078. }

  1079. 

  1080. static inline void doHorizDefFilterAndCopyBack(uint8_t dst[], int stride, int QP)

  1081. {

  1082. #ifdef HAVE_MMX

  1083. 	asm volatile(

  1084. 		"pushl %0					\n\t"

  1085. 		"pxor %%mm7, %%mm7				\n\t"

  1086. 		"movq bm00001000, %%mm6				\n\t"

  1087. 		"movd %2, %%mm5					\n\t" // QP

  1088. 		"movq %%mm5, %%mm4				\n\t"

  1089. 		"paddusb %%mm5, %%mm5				\n\t" // 2QP

  1090. 		"paddusb %%mm5, %%mm4				\n\t" // 3QP

  1091. 		"psllq $24, %%mm4				\n\t"

  1092. 		"pxor %%mm5, %%mm5				\n\t" // 0

  1093. 		"psubb %%mm4, %%mm5				\n\t" // -QP

  1094. 		"leal tempBlock, %%eax				\n\t"

  1095. 

  1096. //FIXME? "unroll by 2" and mix

  1097. #ifdef HAVE_MMX2

  1098. #define HDF(i)	\

  1099. 		"movq " #i "(%%eax), %%mm0			\n\t"\

  1100. 		"movq %%mm0, %%mm1				\n\t"\

  1101. 		"movq %%mm0, %%mm2				\n\t"\

  1102. 		"psrlq $8, %%mm1				\n\t"\

  1103. 		"psubusb %%mm1, %%mm2				\n\t"\

  1104. 		"psubusb %%mm0, %%mm1				\n\t"\

  1105. 		"por %%mm2, %%mm1				\n\t" /* p´x = |px - p(x+1)| */\

  1106. 		"pcmpeqb %%mm7, %%mm2				\n\t" /* p´x = sgn[px - p(x+1)] */\

  1107. 		"pshufw $0x00, %%mm1, %%mm3			\n\t" /* p´5 = |p1 - p2| */\

  1108. 		"pminub %%mm1, %%mm3				\n\t" /* p´5 = min(|p2-p1|, |p6-p5|)*/\

  1109. 		"psrlq $16, %%mm3				\n\t" /* p´3 = min(|p2-p1|, |p6-p5|)*/\

  1110. 		"psubusb %%mm3, %%mm1			\n\t" /* |p3-p4|-min(|p1-p2|,|p5-p6|) */\

  1111. 		"paddb %%mm5, %%mm1				\n\t"\

  1112. 		"psubusb %%mm5, %%mm1				\n\t"\

  1113. 		"psrlw $2, %%mm1				\n\t"\

  1114. 		"pxor %%mm2, %%mm1				\n\t"\

  1115. 		"psubb %%mm2, %%mm1				\n\t"\

  1116. 		"pand %%mm6, %%mm1				\n\t"\

  1117. 		"psubb %%mm1, %%mm0				\n\t"\

  1118. 		"psllq $8, %%mm1				\n\t"\

  1119. 		"paddb %%mm1, %%mm0				\n\t"\

  1120. 		"movd %%mm0, (%0)				\n\t"\

  1121. 		"psrlq $32, %%mm0				\n\t"\

  1122. 		"movd %%mm0, 4(%0)				\n\t"

  1123. #else

  1124. #define HDF(i)\

  1125. 		"movq " #i "(%%eax), %%mm0			\n\t"\

  1126. 		"movq %%mm0, %%mm1				\n\t"\

  1127. 		"movq %%mm0, %%mm2				\n\t"\

  1128. 		"psrlq $8, %%mm1				\n\t"\

  1129. 		"psubusb %%mm1, %%mm2				\n\t"\

  1130. 		"psubusb %%mm0, %%mm1				\n\t"\

  1131. 		"por %%mm2, %%mm1				\n\t" /* p´x = |px - p(x+1)| */\

  1132. 		"pcmpeqb %%mm7, %%mm2				\n\t" /* p´x = sgn[px - p(x+1)] */\

  1133. 		"movq %%mm1, %%mm3				\n\t"\

  1134. 		"psllq $32, %%mm3				\n\t"\

  1135. 		"movq %%mm3, %%mm4				\n\t"\

  1136. 		"psubusb %%mm1, %%mm4				\n\t"\

  1137. 		"psubb %%mm4, %%mm3				\n\t"\

  1138. 		"psrlq $16, %%mm3				\n\t" /* p´3 = min(|p2-p1|, |p6-p5|)*/\

  1139. 		"psubusb %%mm3, %%mm1			\n\t" /* |p3-p4|-min(|p1-p2|,|p5,ü6|) */\

  1140. 		"paddb %%mm5, %%mm1				\n\t"\

  1141. 		"psubusb %%mm5, %%mm1				\n\t"\

  1142. 		"psrlw $2, %%mm1				\n\t"\

  1143. 		"pxor %%mm2, %%mm1				\n\t"\

  1144. 		"psubb %%mm2, %%mm1				\n\t"\

  1145. 		"pand %%mm6, %%mm1				\n\t"\

  1146. 		"psubb %%mm1, %%mm0				\n\t"\

  1147. 		"psllq $8, %%mm1				\n\t"\

  1148. 		"paddb %%mm1, %%mm0				\n\t"\

  1149. 		"movd %%mm0, (%0)				\n\t"\

  1150. 		"psrlq $32, %%mm0				\n\t"\

  1151. 		"movd %%mm0, 4(%0)				\n\t"

  1152. #endif

  1153. 		HDF(0)

  1154. 		"addl %1, %0					\n\t"

  1155. 		HDF(8)

  1156. 		"addl %1, %0					\n\t"

  1157. 		HDF(16)

  1158. 		"addl %1, %0					\n\t"

  1159. 		HDF(24)

  1160. 		"addl %1, %0					\n\t"

  1161. 		HDF(32)

  1162. 		"addl %1, %0					\n\t"

  1163. 		HDF(40)

  1164. 		"addl %1, %0					\n\t"

  1165. 		HDF(48)

  1166. 		"addl %1, %0					\n\t"

  1167. 		HDF(56)

  1168. 		"popl %0					\n\t"

  1169. 		:

  1170. 		: "r" (dst), "r" (stride), "r" (QP)

  1171. 		: "%eax"

  1172. 	);

  1173. #else

  1174. 	uint8_t *src= tempBlock;

  1175. 

  1176. 	for(int y=0; y<BLOCK_SIZE; y++)

  1177. 	{

  1178. 		dst[0] = src[0];

  1179. 		dst[1] = src[1];

  1180. 		dst[2] = src[2];

  1181. 		dst[3] = src[3];

  1182. 		dst[4] = src[4];

  1183. 		dst[5] = src[5];

  1184. 		dst[6] = src[6];

  1185. 		dst[7] = src[7];

  1186. 

  1187. 		const int middleEnergy= 5*(src[4] - src[5]) + 2*(src[2] - src[5]);

  1188. 		if(ABS(middleEnergy) < 8*QP)

  1189. 		{

  1190. 			const int q=(src[3] - src[4])/2;

  1191. 			const int leftEnergy=  5*(src[2] - src[1]) + 2*(src[0] - src[3]);

  1192. 			const int rightEnergy= 5*(src[6] - src[5]) + 2*(src[4] - src[7]);

  1193. 

  1194. 			int d= ABS(middleEnergy) - MIN( ABS(leftEnergy), ABS(rightEnergy) );

  1195. 			d= MAX(d, 0);

  1196. 

  1197. 			d= (5*d + 32) >> 6;

  1198. 			d*= SIGN(-middleEnergy);

  1199. 

  1200. 			if(q>0)

  1201. 			{

  1202. 				d= d<0 ? 0 : d;

  1203. 				d= d>q ? q : d;

  1204. 			}

  1205. 			else

  1206. 			{

  1207. 				d= d>0 ? 0 : d;

  1208. 				d= d<q ? q : d;

  1209. 			}

  1210. 

  1211.         		dst[3]-= d;

  1212. 	        	dst[4]+= d;

  1213. 		}

  1214. 		dst+= stride;

  1215. 		src+= TEMP_STRIDE;

  1216. 	}

  1217. #endif

  1218. }

  1219. 

  1220. /**

  1221.  * Do a horizontal low pass filter on the 8x8 block

  1222.  * useing the 9-Tap Filter (1,1,2,2,4,2,2,1,1)/16 (C version)

  1223.  * useing approximately the 7-Tap Filter (1,2,3,4,3,2,1)/16 (MMX2/3DNOW version)

  1224.  */

  1225. static inline void doHorizLowPassAndCopyBack(uint8_t dst[], int stride, int QP)

  1226. {

  1227. //return;

  1228. #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)

  1229. 	asm volatile(	//"movv %0 %1 %2\n\t"

  1230. 		"pushl %0\n\t"

  1231. 		"pxor %%mm7, %%mm7					\n\t"

  1232. 		"leal tempBlock, %%eax					\n\t"

  1233. /*

  1234. #define HLP1	"movq (%0), %%mm0					\n\t"\

  1235. 		"movq %%mm0, %%mm1					\n\t"\

  1236. 		"psllq $8, %%mm0					\n\t"\

  1237. 		PAVGB(%%mm1, %%mm0)\

  1238. 		"psrlw $8, %%mm0					\n\t"\

  1239. 		"pxor %%mm1, %%mm1					\n\t"\

  1240. 		"packuswb %%mm1, %%mm0					\n\t"\

  1241. 		"movq %%mm0, %%mm1					\n\t"\

  1242. 		"movq %%mm0, %%mm2					\n\t"\

  1243. 		"psllq $32, %%mm0					\n\t"\

  1244. 		"paddb %%mm0, %%mm1					\n\t"\

  1245. 		"psllq $16, %%mm2					\n\t"\

  1246. 		PAVGB(%%mm2, %%mm0)\

  1247. 		"movq %%mm0, %%mm3					\n\t"\

  1248. 		"pand bm11001100, %%mm0					\n\t"\

  1249. 		"paddusb %%mm0, %%mm3					\n\t"\

  1250. 		"psrlq $8, %%mm3					\n\t"\

  1251. 		PAVGB(%%mm1, %%mm4)\

  1252. 		PAVGB(%%mm3, %%mm2)\

  1253. 		"psrlq $16, %%mm2					\n\t"\

  1254. 		"punpcklbw %%mm2, %%mm2					\n\t"\

  1255. 		"movq %%mm2, (%0)					\n\t"\

  1256. 

  1257. #define HLP2	"movq (%0), %%mm0					\n\t"\

  1258. 		"movq %%mm0, %%mm1					\n\t"\

  1259. 		"psllq $8, %%mm0					\n\t"\

  1260. 		PAVGB(%%mm1, %%mm0)\

  1261. 		"psrlw $8, %%mm0					\n\t"\

  1262. 		"pxor %%mm1, %%mm1					\n\t"\

  1263. 		"packuswb %%mm1, %%mm0					\n\t"\

  1264. 		"movq %%mm0, %%mm2					\n\t"\

  1265. 		"psllq $32, %%mm0					\n\t"\

  1266. 		"psllq $16, %%mm2					\n\t"\

  1267. 		PAVGB(%%mm2, %%mm0)\

  1268. 		"movq %%mm0, %%mm3					\n\t"\

  1269. 		"pand bm11001100, %%mm0					\n\t"\

  1270. 		"paddusb %%mm0, %%mm3					\n\t"\

  1271. 		"psrlq $8, %%mm3					\n\t"\

  1272. 		PAVGB(%%mm3, %%mm2)\

  1273. 		"psrlq $16, %%mm2					\n\t"\

  1274. 		"punpcklbw %%mm2, %%mm2					\n\t"\

  1275. 		"movq %%mm2, (%0)					\n\t"\

  1276. */

  1277. // approximately a 7-Tap Filter with Vector (1,2,3,4,3,2,1)/16

  1278. /*

  1279.  31

  1280.  121

  1281.   121

  1282.    121

  1283.     121

  1284.      121

  1285.       121

  1286.        13

  1287. Implemented	Exact 7-Tap

  1288.  9421		A321

  1289.  36421		64321

  1290.  334321		=

  1291.  1234321	=

  1292.   1234321	=

  1293.    123433	=

  1294.     12463	  12346

  1295.      1249	   123A

  1296. 

  1297. */

  1298. #ifdef HAVE_MMX2

  1299. #define HLP3(i)	"movq " #i "(%%eax), %%mm0				\n\t"\

  1300. 		"movq %%mm0, %%mm1					\n\t"\

  1301. 		"movq %%mm0, %%mm2					\n\t"\

  1302. 		"movq %%mm0, %%mm3					\n\t"\

  1303. 		"movq %%mm0, %%mm4					\n\t"\

  1304. 		"psllq $8, %%mm1					\n\t"\

  1305. 		"psrlq $8, %%mm2					\n\t"\

  1306. 		"pand bm00000001, %%mm3					\n\t"\

  1307. 		"pand bm10000000, %%mm4					\n\t"\

  1308. 		"por %%mm3, %%mm1					\n\t"\

  1309. 		"por %%mm4, %%mm2					\n\t"\

  1310. 		PAVGB(%%mm2, %%mm1)\

  1311. 		PAVGB(%%mm1, %%mm0)\

  1312. \

  1313. 		"pshufw $0xF9, %%mm0, %%mm3				\n\t"\

  1314. 		"pshufw $0x90, %%mm0, %%mm4				\n\t"\

  1315. 		PAVGB(%%mm3, %%mm4)\

  1316. 		PAVGB(%%mm4, %%mm0)\

  1317. 		"movd %%mm0, (%0)					\n\t"\

  1318. 		"psrlq $32, %%mm0					\n\t"\

  1319. 		"movd %%mm0, 4(%0)					\n\t"

  1320. #else

  1321. #define HLP3(i)	"movq " #i "(%%eax), %%mm0				\n\t"\

  1322. 		"movq %%mm0, %%mm1					\n\t"\

  1323. 		"movq %%mm0, %%mm2					\n\t"\

  1324. 		"movq %%mm0, %%mm3					\n\t"\

  1325. 		"movq %%mm0, %%mm4					\n\t"\

  1326. 		"psllq $8, %%mm1					\n\t"\

  1327. 		"psrlq $8, %%mm2					\n\t"\

  1328. 		"pand bm00000001, %%mm3					\n\t"\

  1329. 		"pand bm10000000, %%mm4					\n\t"\

  1330. 		"por %%mm3, %%mm1					\n\t"\

  1331. 		"por %%mm4, %%mm2					\n\t"\

  1332. 		PAVGB(%%mm2, %%mm1)\

  1333. 		PAVGB(%%mm1, %%mm0)\

  1334. \

  1335. 		"movq %%mm0, %%mm3					\n\t"\

  1336. 		"movq %%mm0, %%mm4					\n\t"\

  1337. 		"movq %%mm0, %%mm5					\n\t"\

  1338. 		"psrlq $16, %%mm3					\n\t"\

  1339. 		"psllq $16, %%mm4					\n\t"\

  1340. 		"pand bm11000000, %%mm5					\n\t"\

  1341. 		"por %%mm5, %%mm3					\n\t"\

  1342. 		"movq %%mm0, %%mm5					\n\t"\

  1343. 		"pand bm00000011, %%mm5					\n\t"\

  1344. 		"por %%mm5, %%mm4					\n\t"\

  1345. 		PAVGB(%%mm3, %%mm4)\

  1346. 		PAVGB(%%mm4, %%mm0)\

  1347. 		"movd %%mm0, (%0)					\n\t"\

  1348. 		"psrlq $32, %%mm0					\n\t"\

  1349. 		"movd %%mm0, 4(%0)					\n\t"

  1350. #endif

  1351. 

  1352. #define HLP(i) HLP3(i)

  1353. 

  1354. 		HLP(0)

  1355. 		"addl %1, %0						\n\t"

  1356. 		HLP(8)

  1357. 		"addl %1, %0						\n\t"

  1358. 		HLP(16)

  1359. 		"addl %1, %0						\n\t"

  1360. 		HLP(24)

  1361. 		"addl %1, %0						\n\t"

  1362. 		HLP(32)

  1363. 		"addl %1, %0						\n\t"

  1364. 		HLP(40)

  1365. 		"addl %1, %0						\n\t"

  1366. 		HLP(48)

  1367. 		"addl %1, %0						\n\t"

  1368. 		HLP(56)

  1369. 

  1370. 		"popl %0\n\t"

  1371. 		:

  1372. 		: "r" (dst), "r" (stride)

  1373. 		: "%eax", "%ebx"

  1374. 	);

  1375. 

  1376. #else

  1377. 	uint8_t *temp= tempBlock;

  1378. 	for(int y=0; y<BLOCK_SIZE; y++)

  1379. 	{

  1380. 		const int first= ABS(dst[-1] - dst[0]) < QP ? dst[-1] : dst[0];

  1381. 		const int last= ABS(dst[8] - dst[7]) < QP ? dst[8] : dst[7];

  1382. 

  1383. 		int sums[9];

  1384. 		sums[0] = first + temp[0];

  1385. 		sums[1] = temp[0] + temp[1];

  1386. 		sums[2] = temp[1] + temp[2];

  1387. 		sums[3] = temp[2] + temp[3];

  1388. 		sums[4] = temp[3] + temp[4];

  1389. 		sums[5] = temp[4] + temp[5];

  1390. 		sums[6] = temp[5] + temp[6];

  1391. 		sums[7] = temp[6] + temp[7];

  1392. 		sums[8] = temp[7] + last;

  1393. 

  1394. 		dst[0]= ((sums[0]<<2) + ((first + sums[2])<<1) + sums[4] + 8)>>4;

  1395. 		dst[1]= ((dst[1]<<2) + (first + sums[0] + sums[3]<<1) + sums[5] + 8)>>4;

  1396. 		dst[2]= ((dst[2]<<2) + (first + sums[1] + sums[4]<<1) + sums[6] + 8)>>4;

  1397. 		dst[3]= ((dst[3]<<2) + (sums[2] + sums[5]<<1) + sums[0] + sums[7] + 8)>>4;

  1398. 		dst[4]= ((dst[4]<<2) + (sums[3] + sums[6]<<1) + sums[1] + sums[8] + 8)>>4;

  1399. 		dst[5]= ((dst[5]<<2) + (last + sums[7] + sums[4]<<1) + sums[2] + 8)>>4;

  1400. 		dst[6]= ((last + dst[6]<<2) + (dst[7] + sums[5]<<1) + sums[3] + 8)>>4;

  1401. 		dst[7]= ((sums[8]<<2) + (last + sums[6]<<1) + sums[4] + 8)>>4;

  1402. 

  1403. 		dst+= stride;

  1404. 		temp+= TEMP_STRIDE;

  1405. 	}

  1406. #endif

  1407. }

  1408. 

  1409. 

  1410. static inline void dering(uint8_t src[], int stride, int QP)

  1411. {

  1412. //FIXME

  1413. 

  1414. #ifdef HAVE_MMX2X

  1415. 	asm volatile(

  1416. 		"leal (%0, %1), %%eax				\n\t"

  1417. 		"leal (%%eax, %1, 4), %%ebx			\n\t"

  1418. //	0	1	2	3	4	5	6	7	8	9

  1419. //	%0	eax	eax+%1	eax+2%1	%0+4%1	ebx	ebx+%1	ebx+2%1	%0+8%1	ebx+4%1

  1420. 

  1421. 		"pcmpeq %%mm6, %%mm6				\n\t"

  1422. 		"pxor %%mm7, %%mm7				\n\t"

  1423. 

  1424. #define FIND_MIN_MAX(addr)\

  1425. 		"movq (" #addr "), %%mm0,			\n\t"\

  1426. 		"pminub %%mm0, %%mm6				\n\t"\

  1427. 		"pmaxub %%mm0, %%mm7				\n\t"

  1428. 

  1429. FIND_MIN_MAX(%0)

  1430. FIND_MIN_MAX(%%eax)

  1431. FIND_MIN_MAX(%%eax, %1)

  1432. FIND_MIN_MAX(%%eax, %1, 2)

  1433. FIND_MIN_MAX(%0, %1, 4)

  1434. FIND_MIN_MAX(%%ebx)

  1435. FIND_MIN_MAX(%%ebx, %1)

  1436. FIND_MIN_MAX(%%ebx, %1, 2)

  1437. FIND_MIN_MAX(%0, %1, 8)

  1438. FIND_MIN_MAX(%%ebx, %1, 2)

  1439. 

  1440. 		"movq %%mm6, %%mm4				\n\t"

  1441. 		"psrlq $32, %%mm6				\n\t"

  1442. 		"pminub %%mm4, %%mm6				\n\t"

  1443. 		"movq %%mm6, %%mm4				\n\t"

  1444. 		"psrlq $16, %%mm6				\n\t"

  1445. 		"pminub %%mm4, %%mm6				\n\t"

  1446. 		"movq %%mm6, %%mm4				\n\t"

  1447. 		"psrlq $8, %%mm6				\n\t"

  1448. 		"pminub %%mm4, %%mm6				\n\t" // min of pixels

  1449. 

  1450. 		"movq %%mm7, %%mm4				\n\t"

  1451. 		"psrlq $32, %%mm7				\n\t"

  1452. 		"pmaxub %%mm4, %%mm7				\n\t"

  1453. 		"movq %%mm7, %%mm4				\n\t"

  1454. 		"psrlq $16, %%mm7				\n\t"

  1455. 		"pmaxub %%mm4, %%mm7				\n\t"

  1456. 		"movq %%mm7, %%mm4				\n\t"

  1457. 		"psrlq $8, %%mm7				\n\t"

  1458. 		"pmaxub %%mm4, %%mm7				\n\t" // max of pixels

  1459. 		PAVGB(%%mm6, %%mm7)				      // (max + min)/2

  1460. 

  1461. 

  1462. 		: : "r" (src), "r" (stride), "r" (QP)

  1463. 		: "%eax", "%ebx"

  1464. 	);

  1465. #else

  1466. 

  1467. //FIXME

  1468. #endif

  1469. }

  1470. 

  1471. 

  1472. 

  1473. 

  1474. /**

  1475.  * ...

  1476.  * the mode value is interpreted as a quality value if its negative, its range is then (-1 ... -63)

  1477.  * -63 is best quality -1 is worst

  1478.  */

  1479. extern "C"{

  1480. void  postprocess(unsigned char * src[], int src_stride,

  1481.                  unsigned char * dst[], int dst_stride,

  1482.                  int horizontal_size,   int vertical_size,

  1483.                  QP_STORE_T *QP_store,  int QP_stride,

  1484. 					  int mode)

  1485. {

  1486. 

  1487. 	if(mode<0) mode= getModeForQuality(-mode);

  1488. 

  1489. /*

  1490. 	long long T= rdtsc();

  1491. 	for(int y=vertical_size-1; y>=0 ; y--)

  1492. 		memcpy(dst[0] + y*src_stride, src[0] + y*src_stride,src_stride);

  1493. //	memcpy(dst[0], src[0],src_stride*vertical_size);

  1494. 	printf("%4dk\r", (rdtsc()-T)/1000);

  1495. 

  1496. 	return;

  1497. */

  1498. /*

  1499. 	long long T= rdtsc();

  1500. 	while( (rdtsc() - T)/1000 < 4000);

  1501. 

  1502. 	return;

  1503. */

  1504. 	postProcess(src[0], src_stride, dst[0], dst_stride,

  1505. 		horizontal_size, vertical_size, QP_store, QP_stride, false, mode);

  1506. 

  1507. 	horizontal_size >>= 1;

  1508. 	vertical_size   >>= 1;

  1509. 	src_stride      >>= 1;

  1510. 	dst_stride      >>= 1;

  1511. 

  1512. 	if(1)

  1513. 	{

  1514. 		postProcess(src[1], src_stride, dst[1], dst_stride,

  1515. 			horizontal_size, vertical_size, QP_store, QP_stride, true, mode >>4);

  1516. 		postProcess(src[2], src_stride, dst[2], dst_stride,

  1517. 			horizontal_size, vertical_size, QP_store, QP_stride, true, mode >>4);

  1518. 	}

  1519. 	else

  1520. 	{

  1521. 		memcpy(dst[1], src[1], src_stride*horizontal_size);

  1522. 		memcpy(dst[2], src[2], src_stride*horizontal_size);

  1523. 	}

  1524. }

  1525. /**

  1526.  * gets the mode flags for a given quality (larger values mean slower but better postprocessing)

  1527.  * 0 <= quality < 64

  1528.  */

  1529. int getModeForQuality(int quality){

  1530. 	int modes[6]= {

  1531. 		LUM_V_DEBLOCK,

  1532. 		LUM_V_DEBLOCK | LUM_H_DEBLOCK,

  1533. 		LUM_V_DEBLOCK | LUM_H_DEBLOCK | CHROM_V_DEBLOCK,

  1534. 		LUM_V_DEBLOCK | LUM_H_DEBLOCK | CHROM_V_DEBLOCK | CHROM_H_DEBLOCK,

  1535. 		LUM_V_DEBLOCK | LUM_H_DEBLOCK | CHROM_V_DEBLOCK | CHROM_H_DEBLOCK | LUM_DERING,

  1536. 		LUM_V_DEBLOCK | LUM_H_DEBLOCK | CHROM_V_DEBLOCK | CHROM_H_DEBLOCK | LUM_DERING | CHROM_DERING

  1537. 		};

  1538. 

  1539. 	return modes[ (quality*6) >>6 ];

  1540. }

  1541. 

  1542. } // extern "C"

  1543. 

  1544. /**

  1545.  * Copies a block from src to dst and fixes the blacklevel

  1546.  */

  1547. static inline void blockCopy(uint8_t dst[], int dstStride, uint8_t src[], int srcStride)

  1548. {

  1549. #ifdef HAVE_MMX

  1550. 					asm volatile(

  1551. 						"pushl %0 \n\t"

  1552. 						"pushl %1 \n\t"

  1553. 						"leal (%2,%2), %%eax	\n\t"

  1554. 						"leal (%3,%3), %%ebx	\n\t"

  1555. 						"movq packedYOffset, %%mm2	\n\t"

  1556. 						"movq packedYScale, %%mm3	\n\t"

  1557. 

  1558. #define SIMPLE_CPY					\

  1559. 						"movq (%0), %%mm0	\n\t"\

  1560. 						"movq (%0,%2), %%mm1	\n\t"\

  1561. 						"psubusb %%mm2, %%mm0	\n\t"\

  1562. 						"psubusb %%mm2, %%mm1	\n\t"\

  1563. 						"movq %%mm0, (%1)	\n\t"\

  1564. 						"movq %%mm1, (%1, %3)	\n\t"\

  1565. 

  1566. #define SCALED_CPY					\

  1567. 						"movq (%0), %%mm0	\n\t"\

  1568. 						"movq (%0,%2), %%mm1	\n\t"\

  1569. 						"psubusb %%mm2, %%mm0	\n\t"\

  1570. 						"psubusb %%mm2, %%mm1	\n\t"\

  1571. 						"pxor %%mm4, %%mm4	\n\t"\

  1572. 						"pxor %%mm5, %%mm5	\n\t"\

  1573. 						"punpcklbw %%mm0, %%mm4 \n\t"\

  1574. 						"punpckhbw %%mm0, %%mm5 \n\t"\

  1575. 						"pmulhuw %%mm3, %%mm4	\n\t"\

  1576. 						"pmulhuw %%mm3, %%mm5	\n\t"\

  1577. 						"packuswb %%mm5, %%mm4	\n\t"\

  1578. 						"movq %%mm4, (%1)	\n\t"\

  1579. 						"pxor %%mm4, %%mm4	\n\t"\

  1580. 						"pxor %%mm5, %%mm5	\n\t"\

  1581. 						"punpcklbw %%mm1, %%mm4 \n\t"\

  1582. 						"punpckhbw %%mm1, %%mm5 \n\t"\

  1583. 						"pmulhuw %%mm3, %%mm4	\n\t"\

  1584. 						"pmulhuw %%mm3, %%mm5	\n\t"\

  1585. 						"packuswb %%mm5, %%mm4	\n\t"\

  1586. 						"movq %%mm4, (%1, %3)	\n\t"\

  1587. 

  1588. 

  1589. #define CPY SCALED_CPY

  1590. //#define CPY SIMPLE_CPY

  1591. //				"prefetchnta 8(%0)\n\t"

  1592. CPY

  1593. 						"addl %%eax, %0		\n\t"

  1594. 						"addl %%ebx, %1		\n\t"

  1595. CPY

  1596. 						"addl %%eax, %0		\n\t"

  1597. 						"addl %%ebx, %1		\n\t"

  1598. CPY

  1599. 						"addl %%eax, %0		\n\t"

  1600. 						"addl %%ebx, %1		\n\t"

  1601. CPY

  1602. 						"popl %1 \n\t"

  1603. 						"popl %0 \n\t"

  1604. 						: : "r" (src),

  1605. 						"r" (dst),

  1606. 						"r" (srcStride),

  1607. 						"r" (dstStride)

  1608. 						: "%eax", "%ebx"

  1609. 					);

  1610. #else

  1611. 				for(int i=0; i<BLOCK_SIZE; i++) // last 10x8 Block is copied allready so +2

  1612. 					memcpy(	&(dst[dstStride*i]),

  1613. 						&(src[srcStride*i]), BLOCK_SIZE);

  1614. #endif

  1615. }

  1616. 

  1617. 

  1618. /**

  1619.  * Filters array of bytes (Y or U or V values)

  1620.  */

  1621. void postProcess(uint8_t src[], int srcStride, uint8_t dst[], int dstStride, int width, int height,

  1622. 	QP_STORE_T QPs[], int QPStride, bool isColor, int mode)

  1623. {

  1624. 

  1625. #ifdef TIMEING

  1626. 	long long T0, T1, memcpyTime=0, vertTime=0, horizTime=0, sumTime, diffTime=0;

  1627. 	sumTime= rdtsc();

  1628. #endif

  1629. 

  1630. 	/* we need 64bit here otherwise we´ll going to have a problem

  1631. 	   after watching a black picture for 5 hours*/

  1632. 	static uint64_t *yHistogram= NULL;

  1633. 	if(!yHistogram)

  1634. 	{

  1635. 		yHistogram= new uint64_t[256];

  1636. 		for(int i=0; i<256; i++) yHistogram[i]= width*height/64/256;

  1637. 	}

  1638. 

  1639. 	int black=0, white=255; // blackest black and whitest white in the picture

  1640. 	if(!isColor)

  1641. 	{

  1642. 		uint64_t sum= 0;

  1643. 		for(int i=0; i<256; i++)

  1644. 			sum+= yHistogram[i];

  1645. 

  1646. 		uint64_t maxClipped= (uint64_t)(sum * maxClippedThreshold);

  1647. 

  1648. 		uint64_t clipped= sum;

  1649. 		for(black=255; black>0; black--)

  1650. 		{

  1651. 			if(clipped < maxClipped) break;

  1652. 			clipped-= yHistogram[black];

  1653. 		}

  1654. 

  1655. 		clipped= sum;

  1656. 		for(white=0; white<256; white++)

  1657. 		{

  1658. 			if(clipped < maxClipped) break;

  1659. 			clipped-= yHistogram[white];

  1660. 		}

  1661. 

  1662. 		// we cant handle negative correctures

  1663. 		packedYOffset= MAX(black - minAllowedY, 0);

  1664. 		packedYOffset|= packedYOffset<<32;

  1665. 		packedYOffset|= packedYOffset<<16;

  1666. 		packedYOffset|= packedYOffset<<8;

  1667. 

  1668. 		double scale= (double)(maxAllowedY - minAllowedY) / (double)(white-black);

  1669. 

  1670. 		packedYScale= uint16_t(scale*256.0 + 0.5);

  1671. 		packedYScale|= packedYScale<<32;

  1672. 		packedYScale|= packedYScale<<16;

  1673. 	}

  1674. 	else

  1675. 	{

  1676. 		packedYScale= 0x0100010001000100LL;

  1677. 		packedYOffset= 0;

  1678. 	}

  1679. 

  1680. 	for(int x=0; x<width; x+=BLOCK_SIZE)

  1681. 		blockCopy(dst + x, dstStride, src + x, srcStride);

  1682. 

  1683. 	for(int y=0; y<height; y+=BLOCK_SIZE)

  1684. 	{

  1685. 		//1% speedup if these are here instead of the inner loop

  1686. 		uint8_t *srcBlock= &(src[y*srcStride]);

  1687. 		uint8_t *dstBlock= &(dst[y*dstStride]);

  1688. 		uint8_t *vertSrcBlock= &(srcBlock[srcStride*3]); // Blocks are 10x8 -> *3 to start

  1689. 		uint8_t *vertBlock= &(dstBlock[dstStride*3]);

  1690. 

  1691. 		// finish 1 block before the next otherwise we´ll might have a problem

  1692. 		// with the L1 Cache of the P4 ... or only a few blocks at a time or soemthing

  1693. 		for(int x=0; x<width; x+=BLOCK_SIZE)

  1694. 		{

  1695. 			int QP= isColor ?

  1696. 				QPs[(y>>3)*QPStride + (x>>3)]:

  1697. 				(QPs[(y>>4)*QPStride + (x>>4)] * (packedYScale &0xFFFF))>>8;

  1698. #ifdef HAVE_MMX

  1699. 		asm volatile(

  1700. 			"movd %0, %%mm7					\n\t"

  1701. 			"packuswb %%mm7, %%mm7				\n\t" // 0, 0, 0, QP, 0, 0, 0, QP

  1702. 			"packuswb %%mm7, %%mm7				\n\t" // 0,QP, 0, QP, 0,QP, 0, QP

  1703. 			"packuswb %%mm7, %%mm7				\n\t" // QP,..., QP

  1704. 			"movq %%mm7, pQPb				\n\t"

  1705. 			: : "r" (QP)

  1706. 		);

  1707. #endif

  1708. 

  1709. 

  1710. 			const int stride= dstStride;

  1711. 			if(y + 12 < height)

  1712. 			{

  1713. #ifdef MORE_TIMEING

  1714. 				T0= rdtsc();

  1715. #endif

  1716. 

  1717. #ifdef HAVE_MMX2

  1718. 				prefetchnta(vertSrcBlock + (((x>>3)&3) + 2)*srcStride + 32);

  1719. 				prefetchnta(vertSrcBlock + (((x>>3)&3) + 6)*srcStride + 32);

  1720. 				prefetcht0(vertBlock + (((x>>3)&3) + 2)*dstStride + 32);

  1721. 				prefetcht0(vertBlock + (((x>>3)&3) + 6)*dstStride + 32);

  1722. #elif defined(HAVE_3DNOW)

  1723. //FIXME check if this is faster on an 3dnow chip or if its faster without the prefetch or ...

  1724. /*				prefetch(vertSrcBlock + (((x>>3)&3) + 2)*srcStride + 32);

  1725. 				prefetch(vertSrcBlock + (((x>>3)&3) + 6)*srcStride + 32);

  1726. 				prefetchw(vertBlock + (((x>>3)&3) + 2)*dstStride + 32);

  1727. 				prefetchw(vertBlock + (((x>>3)&3) + 6)*dstStride + 32);

  1728. */

  1729. #endif

  1730. 				if(!isColor) yHistogram[ srcBlock[0] ]++;

  1731. 

  1732. 				blockCopy(vertBlock + dstStride*2, dstStride,

  1733. 					vertSrcBlock + srcStride*2, srcStride);

  1734. 

  1735. 

  1736. #ifdef MORE_TIMEING

  1737. 				T1= rdtsc();

  1738. 				memcpyTime+= T1-T0;

  1739. 				T0=T1;

  1740. #endif

  1741. 				if(mode & V_DEBLOCK)

  1742. 				{

  1743. 					if(mode & RK_FILTER)

  1744. 						vertRKFilter(vertBlock, stride, QP);

  1745. 					else if(0)

  1746. 						vertX1Filter(vertBlock, stride, QP);

  1747. 					else

  1748. 					{

  1749. 						if( isVertDC(vertBlock, stride))

  1750. 						{

  1751. 							if(isVertMinMaxOk(vertBlock, stride, QP))

  1752. 								doVertLowPass(vertBlock, stride, QP);

  1753. 						}

  1754. 						else

  1755. 							doVertDefFilter(vertBlock, stride, QP);

  1756. 					}

  1757. 				}

  1758. #ifdef MORE_TIMEING

  1759. 				T1= rdtsc();

  1760. 				vertTime+= T1-T0;

  1761. 				T0=T1;

  1762. #endif

  1763. 			}

  1764. 			else

  1765. 			{

  1766. 				for(int i=2; i<BLOCK_SIZE/2+1; i++) // last 10x8 Block is copied allready so +2

  1767. 					memcpy(	&(vertBlock[dstStride*i]),

  1768. 						&(vertSrcBlock[srcStride*i]), BLOCK_SIZE);

  1769. 

  1770. 			}

  1771. 

  1772. 			if(x - 8 >= 0 && x<width)

  1773. 			{

  1774. #ifdef MORE_TIMEING

  1775. 				T0= rdtsc();

  1776. #endif

  1777. 				if(mode & H_DEBLOCK)

  1778. 				{

  1779. 					if( isHorizDCAndCopy2Temp(dstBlock-4, stride))

  1780. 					{

  1781. 						if(isHorizMinMaxOk(tempBlock, TEMP_STRIDE, QP))

  1782. 							doHorizLowPassAndCopyBack(dstBlock-4, stride, QP);

  1783. 					}

  1784. 					else

  1785. 						doHorizDefFilterAndCopyBack(dstBlock-4, stride, QP);

  1786. 				}

  1787. #ifdef MORE_TIMEING

  1788. 				T1= rdtsc();

  1789. 				horizTime+= T1-T0;

  1790. 				T0=T1;

  1791. #endif

  1792. 				dering(dstBlock - 9 - stride, stride, QP);

  1793. 			}

  1794. 			else if(y!=0)

  1795. 				dering(dstBlock - stride*9 + width-9, stride, QP);

  1796. 			//FIXME dering filter will not be applied to last block (bottom right)

  1797. 

  1798. 

  1799. 			dstBlock+=8;

  1800. 			srcBlock+=8;

  1801. 			vertBlock+=8;

  1802. 			vertSrcBlock+=8;

  1803. 		}

  1804. 	}

  1805. #ifdef HAVE_3DNOW

  1806. 	asm volatile("femms");

  1807. #elif defined (HAVE_MMX)

  1808. 	asm volatile("emms");

  1809. #endif

  1810. 

  1811. #ifdef TIMEING

  1812. 	// FIXME diff is mostly the time spent for rdtsc (should subtract that but ...)

  1813. 	sumTime= rdtsc() - sumTime;

  1814. 	if(!isColor)

  1815. 		printf("cpy:%4dk, vert:%4dk, horiz:%4dk, sum:%4dk, diff:%4dk, color: %d/%d    \r",

  1816. 			int(memcpyTime/1000), int(vertTime/1000), int(horizTime/1000),

  1817. 			int(sumTime/1000), int((sumTime-memcpyTime-vertTime-horizTime)/1000)

  1818. 			, black, white);

  1819. #endif

  1820. }

  1821.