falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1651 Commits
32 Branches
771MB
Tree: a33c7159e1
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a33c7159e1'
${ noResults }
FFmpeg/libavcodec/i386/simple_idct_mmx.c

1311 lines
52KB
Raw Blame History 

  1. /*

  2.  * Simple IDCT MMX

  3.  *

  4.  * Copyright (c) 2001, 2002 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * This library is free software; you can redistribute it and/or

  7.  * modify it under the terms of the GNU Lesser General Public

  8.  * License as published by the Free Software Foundation; either

  9.  * version 2 of the License, or (at your option) any later version.

  10.  *

  11.  * This library is distributed in the hope that it will be useful,

  12.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  13.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  14.  * Lesser General Public License for more details.

  15.  *

  16.  * You should have received a copy of the GNU Lesser General Public

  17.  * License along with this library; if not, write to the Free Software

  18.  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

  19.  */

  20. #include "../dsputil.h"

  21. 

  22. /*

  23. 23170.475006

  24. 22725.260826

  25. 21406.727617

  26. 19265.545870

  27. 16384.000000

  28. 12872.826198

  29. 8866.956905

  30. 4520.335430

  31. */

  32. #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  33. #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  34. #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  35. #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  36. #if 0

  37. #define C4 16384 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  38. #else

  39. #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5

  40. #endif

  41. #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  42. #define C6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  43. #define C7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  44. 

  45. #define ROW_SHIFT 11

  46. #define COL_SHIFT 20 // 6

  47. 

  48. static const uint64_t __attribute__((aligned(8))) wm1010= 0xFFFF0000FFFF0000ULL;

  49. static const uint64_t __attribute__((aligned(8))) d40000= 0x0000000000040000ULL;

  50. static int16_t __attribute__((aligned(8))) temp[64];

  51. static int16_t __attribute__((aligned(8))) coeffs[]= {

  52. 	1<<(ROW_SHIFT-1), 0, 1<<(ROW_SHIFT-1), 0,

  53. //	1<<(COL_SHIFT-1), 0, 1<<(COL_SHIFT-1), 0,

  54. //	0, 1<<(COL_SHIFT-1-16), 0, 1<<(COL_SHIFT-1-16),

  55. 	1<<(ROW_SHIFT-1), 1, 1<<(ROW_SHIFT-1), 0,

  56. 	// the 1 = ((1<<(COL_SHIFT-1))/C4)<<ROW_SHIFT :)

  57. //	0, 0, 0, 0,

  58. //	0, 0, 0, 0,

  59. 

  60.  C4,  C4,  C4,  C4,

  61.  C4, -C4,  C4, -C4,

  62.  

  63.  C2,  C6,  C2,  C6,

  64.  C6, -C2,  C6, -C2,

  65.  

  66.  C1,  C3,  C1,  C3,

  67.  C5,  C7,  C5,  C7,

  68.  

  69.  C3, -C7,  C3, -C7,

  70. -C1, -C5, -C1, -C5,

  71.  

  72.  C5, -C1,  C5, -C1,

  73.  C7,  C3,  C7,  C3,

  74.  

  75.  C7, -C5,  C7, -C5,

  76.  C3, -C1,  C3, -C1

  77. };

  78. 

  79. #if 0

  80. static void unused_var_killer(){

  81. 	int a= wm1010 + d40000;

  82. 	temp[0]=a;

  83. }

  84. 

  85. static void inline idctCol (int16_t * col, int16_t *input)

  86. {

  87. #undef C0

  88. #undef C1

  89. #undef C2

  90. #undef C3

  91. #undef C4

  92. #undef C5

  93. #undef C6

  94. #undef C7

  95. 	int a0, a1, a2, a3, b0, b1, b2, b3;

  96. 	const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  97. 	const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  98. 	const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  99. 	const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  100. 	const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  101. 	const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  102. 	const int C6 = 8867; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  103. 	const int C7 = 4520; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  104. /*

  105. 	if( !(col[8*1] | col[8*2] |col[8*3] |col[8*4] |col[8*5] |col[8*6] | col[8*7])) {

  106. 		col[8*0] = col[8*1] = col[8*2] = col[8*3] = col[8*4] =

  107. 			col[8*5] = col[8*6] = col[8*7] = col[8*0]<<3;

  108. 		return;

  109. 	}*/

  110. 

  111. col[8*0] = input[8*0 + 0];

  112. col[8*1] = input[8*2 + 0];

  113. col[8*2] = input[8*0 + 1];

  114. col[8*3] = input[8*2 + 1];

  115. col[8*4] = input[8*4 + 0];

  116. col[8*5] = input[8*6 + 0];

  117. col[8*6] = input[8*4 + 1];

  118. col[8*7] = input[8*6 + 1];

  119. 

  120. 	a0 = C4*col[8*0] + C2*col[8*2] + C4*col[8*4] + C6*col[8*6] + (1<<(COL_SHIFT-1));

  121. 	a1 = C4*col[8*0] + C6*col[8*2] - C4*col[8*4] - C2*col[8*6] + (1<<(COL_SHIFT-1));

  122. 	a2 = C4*col[8*0] - C6*col[8*2] - C4*col[8*4] + C2*col[8*6] + (1<<(COL_SHIFT-1));

  123. 	a3 = C4*col[8*0] - C2*col[8*2] + C4*col[8*4] - C6*col[8*6] + (1<<(COL_SHIFT-1));

  124. 

  125. 	b0 = C1*col[8*1] + C3*col[8*3] + C5*col[8*5] + C7*col[8*7];

  126. 	b1 = C3*col[8*1] - C7*col[8*3] - C1*col[8*5] - C5*col[8*7];

  127. 	b2 = C5*col[8*1] - C1*col[8*3] + C7*col[8*5] + C3*col[8*7];

  128. 	b3 = C7*col[8*1] - C5*col[8*3] + C3*col[8*5] - C1*col[8*7];

  129. 

  130. 	col[8*0] = (a0 + b0) >> COL_SHIFT;

  131. 	col[8*1] = (a1 + b1) >> COL_SHIFT;

  132. 	col[8*2] = (a2 + b2) >> COL_SHIFT;

  133. 	col[8*3] = (a3 + b3) >> COL_SHIFT;

  134. 	col[8*4] = (a3 - b3) >> COL_SHIFT;

  135. 	col[8*5] = (a2 - b2) >> COL_SHIFT;

  136. 	col[8*6] = (a1 - b1) >> COL_SHIFT;

  137. 	col[8*7] = (a0 - b0) >> COL_SHIFT;

  138. }

  139. 

  140. static void inline idctRow (int16_t * output, int16_t * input)

  141. {

  142. 	int16_t row[8];

  143. 

  144. 	int a0, a1, a2, a3, b0, b1, b2, b3;

  145. 	const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  146. 	const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  147. 	const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  148. 	const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  149. 	const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  150. 	const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  151. 	const int C6 = 8867; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  152. 	const int C7 = 4520; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5

  153. 

  154. row[0] = input[0];

  155. row[2] = input[1];

  156. row[4] = input[4];

  157. row[6] = input[5];

  158. row[1] = input[8];

  159. row[3] = input[9];

  160. row[5] = input[12];

  161. row[7] = input[13];

  162. 

  163. 	if( !(row[1] | row[2] |row[3] |row[4] |row[5] |row[6] | row[7]) ) {

  164. 		row[0] = row[1] = row[2] = row[3] = row[4] =

  165. 			row[5] = row[6] = row[7] = row[0]<<3;

  166. 	output[0] = row[0];

  167. 	output[2] = row[1];

  168. 	output[4] = row[2];

  169. 	output[6] = row[3];

  170. 	output[8] = row[4];

  171. 	output[10] = row[5];

  172. 	output[12] = row[6];

  173. 	output[14] = row[7];

  174. 		return;

  175. 	}

  176. 

  177. 	a0 = C4*row[0] + C2*row[2] + C4*row[4] + C6*row[6] + (1<<(ROW_SHIFT-1));

  178. 	a1 = C4*row[0] + C6*row[2] - C4*row[4] - C2*row[6] + (1<<(ROW_SHIFT-1));

  179. 	a2 = C4*row[0] - C6*row[2] - C4*row[4] + C2*row[6] + (1<<(ROW_SHIFT-1));

  180. 	a3 = C4*row[0] - C2*row[2] + C4*row[4] - C6*row[6] + (1<<(ROW_SHIFT-1));

  181. 

  182. 	b0 = C1*row[1] + C3*row[3] + C5*row[5] + C7*row[7];

  183. 	b1 = C3*row[1] - C7*row[3] - C1*row[5] - C5*row[7];

  184. 	b2 = C5*row[1] - C1*row[3] + C7*row[5] + C3*row[7];

  185. 	b3 = C7*row[1] - C5*row[3] + C3*row[5] - C1*row[7];

  186. 

  187. 	row[0] = (a0 + b0) >> ROW_SHIFT;

  188. 	row[1] = (a1 + b1) >> ROW_SHIFT;

  189. 	row[2] = (a2 + b2) >> ROW_SHIFT;

  190. 	row[3] = (a3 + b3) >> ROW_SHIFT;

  191. 	row[4] = (a3 - b3) >> ROW_SHIFT;

  192. 	row[5] = (a2 - b2) >> ROW_SHIFT;

  193. 	row[6] = (a1 - b1) >> ROW_SHIFT;

  194. 	row[7] = (a0 - b0) >> ROW_SHIFT;

  195. 

  196. 	output[0] = row[0];

  197. 	output[2] = row[1];

  198. 	output[4] = row[2];

  199. 	output[6] = row[3];

  200. 	output[8] = row[4];

  201. 	output[10] = row[5];

  202. 	output[12] = row[6];

  203. 	output[14] = row[7];

  204. }

  205. #endif

  206. 

  207. static inline void idct(int16_t *block)

  208. {

  209. 	asm volatile(

  210. #if 0 //Alternative, simpler variant

  211. 

  212. #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  213. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  214. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  215. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  216. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  217. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  218. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  219. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  220. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  221. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  222. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  223. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  224. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  225. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  226. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  227. 	#rounder ", %%mm4			\n\t"\

  228. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  229. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  230. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  231. 	"movq 56(%2), %%mm5			\n\t" /* C7	C5	C7	C5 */\

  232. 	"pmaddwd %%mm3, %%mm5			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  233. 	#rounder ", %%mm0			\n\t"\

  234. 	"paddd %%mm0, %%mm1			\n\t" /* A1		a1 */\

  235. 	"paddd %%mm0, %%mm0			\n\t" \

  236. 	"psubd %%mm1, %%mm0			\n\t" /* A2		a2 */\

  237. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  238. 	"paddd %%mm5, %%mm7			\n\t" /* B0		b0 */\

  239. 	"movq 72(%2), %%mm5			\n\t" /* -C5	-C1	-C5	-C1 */\

  240. 	"pmaddwd %%mm3, %%mm5			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  241. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  242. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  243. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  244. 	"paddd %%mm2, %%mm5			\n\t" /* B1		b1 */\

  245. 	"psrad $" #shift ", %%mm7		\n\t"\

  246. 	"psrad $" #shift ", %%mm4		\n\t"\

  247. 	"movq %%mm1, %%mm2			\n\t" /* A1		a1 */\

  248. 	"paddd %%mm5, %%mm1			\n\t" /* A1+B1		a1+b1 */\

  249. 	"psubd %%mm5, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  250. 	"psrad $" #shift ", %%mm1		\n\t"\

  251. 	"psrad $" #shift ", %%mm2		\n\t"\

  252. 	"packssdw %%mm1, %%mm7			\n\t" /* A1+B1	a1+b1	A0+B0	a0+b0 */\

  253. 	"packssdw %%mm4, %%mm2			\n\t" /* A0-B0	a0-b0	A1-B1	a1-b1 */\

  254. 	"movq %%mm7, " #dst "			\n\t"\

  255. 	"movq " #src1 ", %%mm1			\n\t" /* R3	R1	r3	r1 */\

  256. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  257. 	"movq %%mm2, 24+" #dst "		\n\t"\

  258. 	"pmaddwd %%mm1, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  259. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  260. 	"pmaddwd 96(%2), %%mm1			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  261. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  262. 	"movq %%mm0, %%mm2			\n\t" /* A2		a2 */\

  263. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  264. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  265. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  266. 	"psubd %%mm4, %%mm0			\n\t" /* a2-B2		a2-b2 */\

  267. 	"psrad $" #shift ", %%mm2		\n\t"\

  268. 	"psrad $" #shift ", %%mm0		\n\t"\

  269. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  270. 	"paddd %%mm1, %%mm3			\n\t" /* B3		b3 */\

  271. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  272. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  273. 	"psrad $" #shift ", %%mm6		\n\t"\

  274. 	"packssdw %%mm6, %%mm2			\n\t" /* A3+B3	a3+b3	A2+B2	a2+b2 */\

  275. 	"movq %%mm2, 8+" #dst "			\n\t"\

  276. 	"psrad $" #shift ", %%mm4		\n\t"\

  277. 	"packssdw %%mm0, %%mm4			\n\t" /* A2-B2	a2-b2	A3-B3	a3-b3 */\

  278. 	"movq %%mm4, 16+" #dst "		\n\t"\

  279. 

  280. #define COL_IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  281. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  282. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  283. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  284. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  285. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  286. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  287. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  288. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  289. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  290. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  291. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  292. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  293. 	#rounder ", %%mm4			\n\t"\

  294. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  295. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  296. 	#rounder ", %%mm0			\n\t"\

  297. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  298. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  299. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  300. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  301. 	"paddd %%mm1, %%mm0			\n\t" /* A1		a1 */\

  302. 	"psubd %%mm1, %%mm5			\n\t" /* A2		a2 */\

  303. 	"movq 56(%2), %%mm1			\n\t" /* C7	C5	C7	C5 */\

  304. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  305. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  306. 	"paddd %%mm1, %%mm7			\n\t" /* B0		b0 */\

  307. 	"movq 72(%2), %%mm1			\n\t" /* -C5	-C1	-C5	-C1 */\

  308. 	"pmaddwd %%mm3, %%mm1			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  309. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  310. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  311. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  312. 	"paddd %%mm2, %%mm1			\n\t" /* B1		b1 */\

  313. 	"psrad $" #shift ", %%mm7		\n\t"\

  314. 	"psrad $" #shift ", %%mm4		\n\t"\

  315. 	"movq %%mm0, %%mm2			\n\t" /* A1		a1 */\

  316. 	"paddd %%mm1, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  317. 	"psubd %%mm1, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  318. 	"psrad $" #shift ", %%mm0		\n\t"\

  319. 	"psrad $" #shift ", %%mm2		\n\t"\

  320. 	"packssdw %%mm7, %%mm7			\n\t" /* A0+B0	a0+b0 */\

  321. 	"movd %%mm7, " #dst "			\n\t"\

  322. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  323. 	"movd %%mm0, 16+" #dst "		\n\t"\

  324. 	"packssdw %%mm2, %%mm2			\n\t" /* A1-B1	a1-b1 */\

  325. 	"movd %%mm2, 96+" #dst "		\n\t"\

  326. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  327. 	"movd %%mm4, 112+" #dst "		\n\t"\

  328. 	"movq " #src1 ", %%mm0			\n\t" /* R3	R1	r3	r1 */\

  329. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  330. 	"pmaddwd %%mm0, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  331. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  332. 	"pmaddwd 96(%2), %%mm0			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  333. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  334. 	"movq %%mm5, %%mm2			\n\t" /* A2		a2 */\

  335. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  336. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  337. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  338. 	"psubd %%mm4, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  339. 	"psrad $" #shift ", %%mm2		\n\t"\

  340. 	"psrad $" #shift ", %%mm5		\n\t"\

  341. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  342. 	"paddd %%mm0, %%mm3			\n\t" /* B3		b3 */\

  343. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  344. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  345. 	"psrad $" #shift ", %%mm6		\n\t"\

  346. 	"psrad $" #shift ", %%mm4		\n\t"\

  347. 	"packssdw %%mm2, %%mm2			\n\t" /* A2+B2	a2+b2 */\

  348. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  349. 	"movd %%mm2, 32+" #dst "		\n\t"\

  350. 	"packssdw %%mm4, %%mm4			\n\t" /* A3-B3	a3-b3 */\

  351. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  352. 	"movd %%mm6, 48+" #dst "		\n\t"\

  353. 	"movd %%mm4, 64+" #dst "		\n\t"\

  354. 	"movd %%mm5, 80+" #dst "		\n\t"\

  355. 

  356. 	

  357. #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  358. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  359. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  360. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  361. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  362. 	"movq "MANGLE(wm1010)", %%mm4		\n\t"\

  363. 	"pand %%mm0, %%mm4			\n\t"\

  364. 	"por %%mm1, %%mm4			\n\t"\

  365. 	"por %%mm2, %%mm4			\n\t"\

  366. 	"por %%mm3, %%mm4			\n\t"\

  367. 	"packssdw %%mm4,%%mm4			\n\t"\

  368. 	"movd %%mm4, %%eax			\n\t"\

  369. 	"orl %%eax, %%eax			\n\t"\

  370. 	"jz 1f					\n\t"\

  371. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  372. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  373. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  374. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  375. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  376. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  377. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  378. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  379. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  380. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  381. 	#rounder ", %%mm4			\n\t"\

  382. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  383. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  384. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  385. 	"movq 56(%2), %%mm5			\n\t" /* C7	C5	C7	C5 */\

  386. 	"pmaddwd %%mm3, %%mm5			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  387. 	#rounder ", %%mm0			\n\t"\

  388. 	"paddd %%mm0, %%mm1			\n\t" /* A1		a1 */\

  389. 	"paddd %%mm0, %%mm0			\n\t" \

  390. 	"psubd %%mm1, %%mm0			\n\t" /* A2		a2 */\

  391. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  392. 	"paddd %%mm5, %%mm7			\n\t" /* B0		b0 */\

  393. 	"movq 72(%2), %%mm5			\n\t" /* -C5	-C1	-C5	-C1 */\

  394. 	"pmaddwd %%mm3, %%mm5			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  395. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  396. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  397. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  398. 	"paddd %%mm2, %%mm5			\n\t" /* B1		b1 */\

  399. 	"psrad $" #shift ", %%mm7		\n\t"\

  400. 	"psrad $" #shift ", %%mm4		\n\t"\

  401. 	"movq %%mm1, %%mm2			\n\t" /* A1		a1 */\

  402. 	"paddd %%mm5, %%mm1			\n\t" /* A1+B1		a1+b1 */\

  403. 	"psubd %%mm5, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  404. 	"psrad $" #shift ", %%mm1		\n\t"\

  405. 	"psrad $" #shift ", %%mm2		\n\t"\

  406. 	"packssdw %%mm1, %%mm7			\n\t" /* A1+B1	a1+b1	A0+B0	a0+b0 */\

  407. 	"packssdw %%mm4, %%mm2			\n\t" /* A0-B0	a0-b0	A1-B1	a1-b1 */\

  408. 	"movq %%mm7, " #dst "			\n\t"\

  409. 	"movq " #src1 ", %%mm1			\n\t" /* R3	R1	r3	r1 */\

  410. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  411. 	"movq %%mm2, 24+" #dst "		\n\t"\

  412. 	"pmaddwd %%mm1, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  413. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  414. 	"pmaddwd 96(%2), %%mm1			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  415. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  416. 	"movq %%mm0, %%mm2			\n\t" /* A2		a2 */\

  417. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  418. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  419. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  420. 	"psubd %%mm4, %%mm0			\n\t" /* a2-B2		a2-b2 */\

  421. 	"psrad $" #shift ", %%mm2		\n\t"\

  422. 	"psrad $" #shift ", %%mm0		\n\t"\

  423. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  424. 	"paddd %%mm1, %%mm3			\n\t" /* B3		b3 */\

  425. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  426. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  427. 	"psrad $" #shift ", %%mm6		\n\t"\

  428. 	"packssdw %%mm6, %%mm2			\n\t" /* A3+B3	a3+b3	A2+B2	a2+b2 */\

  429. 	"movq %%mm2, 8+" #dst "			\n\t"\

  430. 	"psrad $" #shift ", %%mm4		\n\t"\

  431. 	"packssdw %%mm0, %%mm4			\n\t" /* A2-B2	a2-b2	A3-B3	a3-b3 */\

  432. 	"movq %%mm4, 16+" #dst "		\n\t"\

  433. 	"jmp 2f					\n\t"\

  434. 	"1:					\n\t"\

  435. 	"pslld $16, %%mm0			\n\t"\

  436. 	"#paddd "MANGLE(d40000)", %%mm0		\n\t"\

  437. 	"psrad $13, %%mm0			\n\t"\

  438. 	"packssdw %%mm0, %%mm0			\n\t"\

  439. 	"movq %%mm0, " #dst "			\n\t"\

  440. 	"movq %%mm0, 8+" #dst "			\n\t"\

  441. 	"movq %%mm0, 16+" #dst "		\n\t"\

  442. 	"movq %%mm0, 24+" #dst "		\n\t"\

  443. 	"2:					\n\t"

  444. 

  445. 

  446. //IDCT(      src0,   src4,   src1,   src5,    dst,    rounder, shift)

  447. ROW_IDCT(    (%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)

  448. /*ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1), paddd (%2), 11)

  449. ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1), paddd (%2), 11)

  450. ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1), paddd (%2), 11)*/

  451. 

  452. DC_COND_ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)

  453. DC_COND_ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)

  454. DC_COND_ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)

  455. 

  456. 

  457. //IDCT(      src0,   src4,   src1,    src5,    dst, rounder, shift)

  458. COL_IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  459. COL_IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  460. COL_IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  461. COL_IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  462. 

  463. #else

  464. 

  465. #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  466. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  467. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  468. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  469. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  470. 	"movq "MANGLE(wm1010)", %%mm4		\n\t"\

  471. 	"pand %%mm0, %%mm4			\n\t"\

  472. 	"por %%mm1, %%mm4			\n\t"\

  473. 	"por %%mm2, %%mm4			\n\t"\

  474. 	"por %%mm3, %%mm4			\n\t"\

  475. 	"packssdw %%mm4,%%mm4			\n\t"\

  476. 	"movd %%mm4, %%eax			\n\t"\

  477. 	"orl %%eax, %%eax			\n\t"\

  478. 	"jz 1f					\n\t"\

  479. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  480. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  481. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  482. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  483. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  484. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  485. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  486. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  487. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  488. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  489. 	#rounder ", %%mm4			\n\t"\

  490. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  491. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  492. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  493. 	"movq 56(%2), %%mm5			\n\t" /* C7	C5	C7	C5 */\

  494. 	"pmaddwd %%mm3, %%mm5			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  495. 	#rounder ", %%mm0			\n\t"\

  496. 	"paddd %%mm0, %%mm1			\n\t" /* A1		a1 */\

  497. 	"paddd %%mm0, %%mm0			\n\t" \

  498. 	"psubd %%mm1, %%mm0			\n\t" /* A2		a2 */\

  499. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  500. 	"paddd %%mm5, %%mm7			\n\t" /* B0		b0 */\

  501. 	"movq 72(%2), %%mm5			\n\t" /* -C5	-C1	-C5	-C1 */\

  502. 	"pmaddwd %%mm3, %%mm5			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  503. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  504. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  505. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  506. 	"paddd %%mm2, %%mm5			\n\t" /* B1		b1 */\

  507. 	"psrad $" #shift ", %%mm7		\n\t"\

  508. 	"psrad $" #shift ", %%mm4		\n\t"\

  509. 	"movq %%mm1, %%mm2			\n\t" /* A1		a1 */\

  510. 	"paddd %%mm5, %%mm1			\n\t" /* A1+B1		a1+b1 */\

  511. 	"psubd %%mm5, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  512. 	"psrad $" #shift ", %%mm1		\n\t"\

  513. 	"psrad $" #shift ", %%mm2		\n\t"\

  514. 	"packssdw %%mm1, %%mm7			\n\t" /* A1+B1	a1+b1	A0+B0	a0+b0 */\

  515. 	"packssdw %%mm4, %%mm2			\n\t" /* A0-B0	a0-b0	A1-B1	a1-b1 */\

  516. 	"movq %%mm7, " #dst "			\n\t"\

  517. 	"movq " #src1 ", %%mm1			\n\t" /* R3	R1	r3	r1 */\

  518. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  519. 	"movq %%mm2, 24+" #dst "		\n\t"\

  520. 	"pmaddwd %%mm1, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  521. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  522. 	"pmaddwd 96(%2), %%mm1			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  523. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  524. 	"movq %%mm0, %%mm2			\n\t" /* A2		a2 */\

  525. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  526. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  527. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  528. 	"psubd %%mm4, %%mm0			\n\t" /* a2-B2		a2-b2 */\

  529. 	"psrad $" #shift ", %%mm2		\n\t"\

  530. 	"psrad $" #shift ", %%mm0		\n\t"\

  531. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  532. 	"paddd %%mm1, %%mm3			\n\t" /* B3		b3 */\

  533. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  534. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  535. 	"psrad $" #shift ", %%mm6		\n\t"\

  536. 	"packssdw %%mm6, %%mm2			\n\t" /* A3+B3	a3+b3	A2+B2	a2+b2 */\

  537. 	"movq %%mm2, 8+" #dst "			\n\t"\

  538. 	"psrad $" #shift ", %%mm4		\n\t"\

  539. 	"packssdw %%mm0, %%mm4			\n\t" /* A2-B2	a2-b2	A3-B3	a3-b3 */\

  540. 	"movq %%mm4, 16+" #dst "		\n\t"\

  541. 	"jmp 2f					\n\t"\

  542. 	"1:					\n\t"\

  543. 	"pslld $16, %%mm0			\n\t"\

  544. 	"paddd "MANGLE(d40000)", %%mm0		\n\t"\

  545. 	"psrad $13, %%mm0			\n\t"\

  546. 	"packssdw %%mm0, %%mm0			\n\t"\

  547. 	"movq %%mm0, " #dst "			\n\t"\

  548. 	"movq %%mm0, 8+" #dst "			\n\t"\

  549. 	"movq %%mm0, 16+" #dst "		\n\t"\

  550. 	"movq %%mm0, 24+" #dst "		\n\t"\

  551. 	"2:					\n\t"

  552. 

  553. #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \

  554. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  555. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  556. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  557. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  558. 	"movq %%mm0, %%mm4			\n\t"\

  559. 	"por %%mm1, %%mm4			\n\t"\

  560. 	"por %%mm2, %%mm4			\n\t"\

  561. 	"por %%mm3, %%mm4			\n\t"\

  562. 	"packssdw %%mm4,%%mm4			\n\t"\

  563. 	"movd %%mm4, %%eax			\n\t"\

  564. 	"orl %%eax, %%eax			\n\t"\

  565. 	"jz " #bt "				\n\t"\

  566. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  567. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  568. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  569. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  570. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  571. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  572. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  573. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  574. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  575. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  576. 	#rounder ", %%mm4			\n\t"\

  577. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  578. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  579. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  580. 	"movq 56(%2), %%mm5			\n\t" /* C7	C5	C7	C5 */\

  581. 	"pmaddwd %%mm3, %%mm5			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  582. 	#rounder ", %%mm0			\n\t"\

  583. 	"paddd %%mm0, %%mm1			\n\t" /* A1		a1 */\

  584. 	"paddd %%mm0, %%mm0			\n\t" \

  585. 	"psubd %%mm1, %%mm0			\n\t" /* A2		a2 */\

  586. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  587. 	"paddd %%mm5, %%mm7			\n\t" /* B0		b0 */\

  588. 	"movq 72(%2), %%mm5			\n\t" /* -C5	-C1	-C5	-C1 */\

  589. 	"pmaddwd %%mm3, %%mm5			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  590. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  591. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  592. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  593. 	"paddd %%mm2, %%mm5			\n\t" /* B1		b1 */\

  594. 	"psrad $" #shift ", %%mm7		\n\t"\

  595. 	"psrad $" #shift ", %%mm4		\n\t"\

  596. 	"movq %%mm1, %%mm2			\n\t" /* A1		a1 */\

  597. 	"paddd %%mm5, %%mm1			\n\t" /* A1+B1		a1+b1 */\

  598. 	"psubd %%mm5, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  599. 	"psrad $" #shift ", %%mm1		\n\t"\

  600. 	"psrad $" #shift ", %%mm2		\n\t"\

  601. 	"packssdw %%mm1, %%mm7			\n\t" /* A1+B1	a1+b1	A0+B0	a0+b0 */\

  602. 	"packssdw %%mm4, %%mm2			\n\t" /* A0-B0	a0-b0	A1-B1	a1-b1 */\

  603. 	"movq %%mm7, " #dst "			\n\t"\

  604. 	"movq " #src1 ", %%mm1			\n\t" /* R3	R1	r3	r1 */\

  605. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  606. 	"movq %%mm2, 24+" #dst "		\n\t"\

  607. 	"pmaddwd %%mm1, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  608. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  609. 	"pmaddwd 96(%2), %%mm1			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  610. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  611. 	"movq %%mm0, %%mm2			\n\t" /* A2		a2 */\

  612. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  613. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  614. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  615. 	"psubd %%mm4, %%mm0			\n\t" /* a2-B2		a2-b2 */\

  616. 	"psrad $" #shift ", %%mm2		\n\t"\

  617. 	"psrad $" #shift ", %%mm0		\n\t"\

  618. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  619. 	"paddd %%mm1, %%mm3			\n\t" /* B3		b3 */\

  620. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  621. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  622. 	"psrad $" #shift ", %%mm6		\n\t"\

  623. 	"packssdw %%mm6, %%mm2			\n\t" /* A3+B3	a3+b3	A2+B2	a2+b2 */\

  624. 	"movq %%mm2, 8+" #dst "			\n\t"\

  625. 	"psrad $" #shift ", %%mm4		\n\t"\

  626. 	"packssdw %%mm0, %%mm4			\n\t" /* A2-B2	a2-b2	A3-B3	a3-b3 */\

  627. 	"movq %%mm4, 16+" #dst "		\n\t"\

  628. 

  629. #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  630. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  631. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  632. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  633. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  634. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  635. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  636. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  637. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  638. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  639. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  640. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  641. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  642. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  643. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  644. 	#rounder ", %%mm4			\n\t"\

  645. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  646. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  647. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  648. 	"movq 56(%2), %%mm5			\n\t" /* C7	C5	C7	C5 */\

  649. 	"pmaddwd %%mm3, %%mm5			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  650. 	#rounder ", %%mm0			\n\t"\

  651. 	"paddd %%mm0, %%mm1			\n\t" /* A1		a1 */\

  652. 	"paddd %%mm0, %%mm0			\n\t" \

  653. 	"psubd %%mm1, %%mm0			\n\t" /* A2		a2 */\

  654. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  655. 	"paddd %%mm5, %%mm7			\n\t" /* B0		b0 */\

  656. 	"movq 72(%2), %%mm5			\n\t" /* -C5	-C1	-C5	-C1 */\

  657. 	"pmaddwd %%mm3, %%mm5			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  658. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  659. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  660. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  661. 	"paddd %%mm2, %%mm5			\n\t" /* B1		b1 */\

  662. 	"psrad $" #shift ", %%mm7		\n\t"\

  663. 	"psrad $" #shift ", %%mm4		\n\t"\

  664. 	"movq %%mm1, %%mm2			\n\t" /* A1		a1 */\

  665. 	"paddd %%mm5, %%mm1			\n\t" /* A1+B1		a1+b1 */\

  666. 	"psubd %%mm5, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  667. 	"psrad $" #shift ", %%mm1		\n\t"\

  668. 	"psrad $" #shift ", %%mm2		\n\t"\

  669. 	"packssdw %%mm1, %%mm7			\n\t" /* A1+B1	a1+b1	A0+B0	a0+b0 */\

  670. 	"packssdw %%mm4, %%mm2			\n\t" /* A0-B0	a0-b0	A1-B1	a1-b1 */\

  671. 	"movq %%mm7, " #dst "			\n\t"\

  672. 	"movq " #src1 ", %%mm1			\n\t" /* R3	R1	r3	r1 */\

  673. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  674. 	"movq %%mm2, 24+" #dst "		\n\t"\

  675. 	"pmaddwd %%mm1, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  676. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  677. 	"pmaddwd 96(%2), %%mm1			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  678. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  679. 	"movq %%mm0, %%mm2			\n\t" /* A2		a2 */\

  680. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  681. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  682. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  683. 	"psubd %%mm4, %%mm0			\n\t" /* a2-B2		a2-b2 */\

  684. 	"psrad $" #shift ", %%mm2		\n\t"\

  685. 	"psrad $" #shift ", %%mm0		\n\t"\

  686. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  687. 	"paddd %%mm1, %%mm3			\n\t" /* B3		b3 */\

  688. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  689. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  690. 	"psrad $" #shift ", %%mm6		\n\t"\

  691. 	"packssdw %%mm6, %%mm2			\n\t" /* A3+B3	a3+b3	A2+B2	a2+b2 */\

  692. 	"movq %%mm2, 8+" #dst "			\n\t"\

  693. 	"psrad $" #shift ", %%mm4		\n\t"\

  694. 	"packssdw %%mm0, %%mm4			\n\t" /* A2-B2	a2-b2	A3-B3	a3-b3 */\

  695. 	"movq %%mm4, 16+" #dst "		\n\t"\

  696. 

  697. //IDCT(         src0,   src4,   src1,   src5,    dst,   rounder, shift)

  698. DC_COND_IDCT(  0(%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)

  699. Z_COND_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)

  700. Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)

  701. Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)

  702. 

  703. #undef IDCT

  704. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  705. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  706. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  707. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  708. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  709. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  710. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  711. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  712. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  713. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  714. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  715. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  716. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  717. 	#rounder ", %%mm4			\n\t"\

  718. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  719. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  720. 	#rounder ", %%mm0			\n\t"\

  721. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  722. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  723. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  724. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  725. 	"paddd %%mm1, %%mm0			\n\t" /* A1		a1 */\

  726. 	"psubd %%mm1, %%mm5			\n\t" /* A2		a2 */\

  727. 	"movq 56(%2), %%mm1			\n\t" /* C7	C5	C7	C5 */\

  728. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  729. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  730. 	"paddd %%mm1, %%mm7			\n\t" /* B0		b0 */\

  731. 	"movq 72(%2), %%mm1			\n\t" /* -C5	-C1	-C5	-C1 */\

  732. 	"pmaddwd %%mm3, %%mm1			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  733. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  734. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  735. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  736. 	"paddd %%mm2, %%mm1			\n\t" /* B1		b1 */\

  737. 	"psrad $" #shift ", %%mm7		\n\t"\

  738. 	"psrad $" #shift ", %%mm4		\n\t"\

  739. 	"movq %%mm0, %%mm2			\n\t" /* A1		a1 */\

  740. 	"paddd %%mm1, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  741. 	"psubd %%mm1, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  742. 	"psrad $" #shift ", %%mm0		\n\t"\

  743. 	"psrad $" #shift ", %%mm2		\n\t"\

  744. 	"packssdw %%mm7, %%mm7			\n\t" /* A0+B0	a0+b0 */\

  745. 	"movd %%mm7, " #dst "			\n\t"\

  746. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  747. 	"movd %%mm0, 16+" #dst "		\n\t"\

  748. 	"packssdw %%mm2, %%mm2			\n\t" /* A1-B1	a1-b1 */\

  749. 	"movd %%mm2, 96+" #dst "		\n\t"\

  750. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  751. 	"movd %%mm4, 112+" #dst "		\n\t"\

  752. 	"movq " #src1 ", %%mm0			\n\t" /* R3	R1	r3	r1 */\

  753. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  754. 	"pmaddwd %%mm0, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  755. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  756. 	"pmaddwd 96(%2), %%mm0			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  757. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  758. 	"movq %%mm5, %%mm2			\n\t" /* A2		a2 */\

  759. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  760. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  761. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  762. 	"psubd %%mm4, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  763. 	"psrad $" #shift ", %%mm2		\n\t"\

  764. 	"psrad $" #shift ", %%mm5		\n\t"\

  765. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  766. 	"paddd %%mm0, %%mm3			\n\t" /* B3		b3 */\

  767. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  768. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  769. 	"psrad $" #shift ", %%mm6		\n\t"\

  770. 	"psrad $" #shift ", %%mm4		\n\t"\

  771. 	"packssdw %%mm2, %%mm2			\n\t" /* A2+B2	a2+b2 */\

  772. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  773. 	"movd %%mm2, 32+" #dst "		\n\t"\

  774. 	"packssdw %%mm4, %%mm4			\n\t" /* A3-B3	a3-b3 */\

  775. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  776. 	"movd %%mm6, 48+" #dst "		\n\t"\

  777. 	"movd %%mm4, 64+" #dst "		\n\t"\

  778. 	"movd %%mm5, 80+" #dst "		\n\t"

  779. 

  780. 

  781. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  782. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  783. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  784. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  785. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  786. 	"jmp 9f					\n\t"

  787. 

  788. 	"#.balign 16				\n\t"\

  789. 	"4:					\n\t"

  790. Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)

  791. Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)

  792. 

  793. #undef IDCT

  794. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  795. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  796. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  797. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  798. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  799. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  800. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  801. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  802. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  803. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  804. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  805. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  806. 	#rounder ", %%mm4			\n\t"\

  807. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  808. 	#rounder ", %%mm0			\n\t"\

  809. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  810. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  811. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  812. 	"paddd %%mm1, %%mm0			\n\t" /* A1		a1 */\

  813. 	"psubd %%mm1, %%mm5			\n\t" /* A2		a2 */\

  814. 	"movq 56(%2), %%mm1			\n\t" /* C7	C5	C7	C5 */\

  815. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  816. 	"movq 72(%2), %%mm7			\n\t" /* -C5	-C1	-C5	-C1 */\

  817. 	"pmaddwd %%mm3, %%mm7			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  818. 	"paddd %%mm4, %%mm1			\n\t" /* A0+B0		a0+b0 */\

  819. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  820. 	"psubd %%mm1, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  821. 	"psrad $" #shift ", %%mm1		\n\t"\

  822. 	"psrad $" #shift ", %%mm4		\n\t"\

  823. 	"movq %%mm0, %%mm2			\n\t" /* A1		a1 */\

  824. 	"paddd %%mm7, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  825. 	"psubd %%mm7, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  826. 	"psrad $" #shift ", %%mm0		\n\t"\

  827. 	"psrad $" #shift ", %%mm2		\n\t"\

  828. 	"packssdw %%mm1, %%mm1			\n\t" /* A0+B0	a0+b0 */\

  829. 	"movd %%mm1, " #dst "			\n\t"\

  830. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  831. 	"movd %%mm0, 16+" #dst "		\n\t"\

  832. 	"packssdw %%mm2, %%mm2			\n\t" /* A1-B1	a1-b1 */\

  833. 	"movd %%mm2, 96+" #dst "		\n\t"\

  834. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  835. 	"movd %%mm4, 112+" #dst "		\n\t"\

  836. 	"movq 88(%2), %%mm1			\n\t" /* C3	C7	C3 	C7 */\

  837. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  838. 	"movq %%mm5, %%mm2			\n\t" /* A2		a2 */\

  839. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  840. 	"paddd %%mm1, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  841. 	"psubd %%mm1, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  842. 	"psrad $" #shift ", %%mm2		\n\t"\

  843. 	"psrad $" #shift ", %%mm5		\n\t"\

  844. 	"movq %%mm6, %%mm1			\n\t" /* A3		a3 */\

  845. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  846. 	"psubd %%mm3, %%mm1			\n\t" /* a3-B3		a3-b3 */\

  847. 	"psrad $" #shift ", %%mm6		\n\t"\

  848. 	"psrad $" #shift ", %%mm1		\n\t"\

  849. 	"packssdw %%mm2, %%mm2			\n\t" /* A2+B2	a2+b2 */\

  850. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  851. 	"movd %%mm2, 32+" #dst "		\n\t"\

  852. 	"packssdw %%mm1, %%mm1			\n\t" /* A3-B3	a3-b3 */\

  853. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  854. 	"movd %%mm6, 48+" #dst "		\n\t"\

  855. 	"movd %%mm1, 64+" #dst "		\n\t"\

  856. 	"movd %%mm5, 80+" #dst "		\n\t"	

  857. 

  858. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  859. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  860. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  861. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  862. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  863. 	"jmp 9f					\n\t"

  864. 

  865. 	"#.balign 16				\n\t"\

  866. 	"6:					\n\t"

  867. Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)

  868. 

  869. #undef IDCT

  870. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  871. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  872. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  873. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  874. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  875. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  876. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  877. 	#rounder ", %%mm4			\n\t"\

  878. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  879. 	#rounder ", %%mm0			\n\t"\

  880. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  881. 	"movq 56(%2), %%mm1			\n\t" /* C7	C5	C7	C5 */\

  882. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  883. 	"movq 72(%2), %%mm7			\n\t" /* -C5	-C1	-C5	-C1 */\

  884. 	"pmaddwd %%mm3, %%mm7			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  885. 	"paddd %%mm4, %%mm1			\n\t" /* A0+B0		a0+b0 */\

  886. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  887. 	"psubd %%mm1, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  888. 	"psrad $" #shift ", %%mm1		\n\t"\

  889. 	"psrad $" #shift ", %%mm4		\n\t"\

  890. 	"movq %%mm0, %%mm2			\n\t" /* A1		a1 */\

  891. 	"paddd %%mm7, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  892. 	"psubd %%mm7, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  893. 	"psrad $" #shift ", %%mm0		\n\t"\

  894. 	"psrad $" #shift ", %%mm2		\n\t"\

  895. 	"packssdw %%mm1, %%mm1			\n\t" /* A0+B0	a0+b0 */\

  896. 	"movd %%mm1, " #dst "			\n\t"\

  897. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  898. 	"movd %%mm0, 16+" #dst "		\n\t"\

  899. 	"packssdw %%mm2, %%mm2			\n\t" /* A1-B1	a1-b1 */\

  900. 	"movd %%mm2, 96+" #dst "		\n\t"\

  901. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  902. 	"movd %%mm4, 112+" #dst "		\n\t"\

  903. 	"movq 88(%2), %%mm1			\n\t" /* C3	C7	C3 	C7 */\

  904. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  905. 	"movq %%mm5, %%mm2			\n\t" /* A2		a2 */\

  906. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  907. 	"paddd %%mm1, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  908. 	"psubd %%mm1, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  909. 	"psrad $" #shift ", %%mm2		\n\t"\

  910. 	"psrad $" #shift ", %%mm5		\n\t"\

  911. 	"movq %%mm6, %%mm1			\n\t" /* A3		a3 */\

  912. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  913. 	"psubd %%mm3, %%mm1			\n\t" /* a3-B3		a3-b3 */\

  914. 	"psrad $" #shift ", %%mm6		\n\t"\

  915. 	"psrad $" #shift ", %%mm1		\n\t"\

  916. 	"packssdw %%mm2, %%mm2			\n\t" /* A2+B2	a2+b2 */\

  917. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  918. 	"movd %%mm2, 32+" #dst "		\n\t"\

  919. 	"packssdw %%mm1, %%mm1			\n\t" /* A3-B3	a3-b3 */\

  920. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  921. 	"movd %%mm6, 48+" #dst "		\n\t"\

  922. 	"movd %%mm1, 64+" #dst "		\n\t"\

  923. 	"movd %%mm5, 80+" #dst "		\n\t"	

  924. 

  925. 

  926. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  927. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  928. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  929. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  930. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  931. 	"jmp 9f					\n\t"

  932. 

  933. 	"#.balign 16				\n\t"\

  934. 	"2:					\n\t"

  935. Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)

  936. 

  937. #undef IDCT

  938. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  939. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  940. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  941. 	"movq " #src5 ", %%mm3			\n\t" /* R7	R5	r7	r5 */\

  942. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  943. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  944. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  945. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  946. 	#rounder ", %%mm4			\n\t"\

  947. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  948. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  949. 	#rounder ", %%mm0			\n\t"\

  950. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  951. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  952. 	"movq 56(%2), %%mm1			\n\t" /* C7	C5	C7	C5 */\

  953. 	"pmaddwd %%mm3, %%mm1			\n\t" /* C7R7+C5R5	C7r7+C5r5 */\

  954. 	"pmaddwd 64(%2), %%mm2			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  955. 	"paddd %%mm1, %%mm7			\n\t" /* B0		b0 */\

  956. 	"movq 72(%2), %%mm1			\n\t" /* -C5	-C1	-C5	-C1 */\

  957. 	"pmaddwd %%mm3, %%mm1			\n\t" /* -C5R7-C1R5	-C5r7-C1r5 */\

  958. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  959. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  960. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  961. 	"paddd %%mm2, %%mm1			\n\t" /* B1		b1 */\

  962. 	"psrad $" #shift ", %%mm7		\n\t"\

  963. 	"psrad $" #shift ", %%mm4		\n\t"\

  964. 	"movq %%mm0, %%mm2			\n\t" /* A1		a1 */\

  965. 	"paddd %%mm1, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  966. 	"psubd %%mm1, %%mm2			\n\t" /* A1-B1		a1-b1 */\

  967. 	"psrad $" #shift ", %%mm0		\n\t"\

  968. 	"psrad $" #shift ", %%mm2		\n\t"\

  969. 	"packssdw %%mm7, %%mm7			\n\t" /* A0+B0	a0+b0 */\

  970. 	"movd %%mm7, " #dst "			\n\t"\

  971. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  972. 	"movd %%mm0, 16+" #dst "		\n\t"\

  973. 	"packssdw %%mm2, %%mm2			\n\t" /* A1-B1	a1-b1 */\

  974. 	"movd %%mm2, 96+" #dst "		\n\t"\

  975. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  976. 	"movd %%mm4, 112+" #dst "		\n\t"\

  977. 	"movq " #src1 ", %%mm0			\n\t" /* R3	R1	r3	r1 */\

  978. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  979. 	"pmaddwd %%mm0, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  980. 	"movq 88(%2), %%mm7			\n\t" /* C3	C7	C3 	C7 */\

  981. 	"pmaddwd 96(%2), %%mm0			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  982. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C3R7+C7R5	C3r7+C7r5 */\

  983. 	"movq %%mm5, %%mm2			\n\t" /* A2		a2 */\

  984. 	"pmaddwd 104(%2), %%mm3			\n\t" /* -C1R7+C3R5	-C1r7+C3r5 */\

  985. 	"paddd %%mm7, %%mm4			\n\t" /* B2		b2 */\

  986. 	"paddd %%mm4, %%mm2			\n\t" /* A2+B2		a2+b2 */\

  987. 	"psubd %%mm4, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  988. 	"psrad $" #shift ", %%mm2		\n\t"\

  989. 	"psrad $" #shift ", %%mm5		\n\t"\

  990. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  991. 	"paddd %%mm0, %%mm3			\n\t" /* B3		b3 */\

  992. 	"paddd %%mm3, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  993. 	"psubd %%mm3, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  994. 	"psrad $" #shift ", %%mm6		\n\t"\

  995. 	"psrad $" #shift ", %%mm4		\n\t"\

  996. 	"packssdw %%mm2, %%mm2			\n\t" /* A2+B2	a2+b2 */\

  997. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  998. 	"movd %%mm2, 32+" #dst "		\n\t"\

  999. 	"packssdw %%mm4, %%mm4			\n\t" /* A3-B3	a3-b3 */\

  1000. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  1001. 	"movd %%mm6, 48+" #dst "		\n\t"\

  1002. 	"movd %%mm4, 64+" #dst "		\n\t"\

  1003. 	"movd %%mm5, 80+" #dst "		\n\t"

  1004. 

  1005. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  1006. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  1007. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  1008. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  1009. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  1010. 	"jmp 9f					\n\t"

  1011. 

  1012. 	"#.balign 16				\n\t"\

  1013. 	"3:					\n\t"

  1014. #undef IDCT

  1015. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  1016. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  1017. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  1018. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  1019. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1020. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  1021. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1022. 	#rounder ", %%mm4			\n\t"\

  1023. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1024. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  1025. 	#rounder ", %%mm0			\n\t"\

  1026. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  1027. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1028. 	"movq 64(%2), %%mm3			\n\t"\

  1029. 	"pmaddwd %%mm2, %%mm3			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  1030. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  1031. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  1032. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  1033. 	"psrad $" #shift ", %%mm7		\n\t"\

  1034. 	"psrad $" #shift ", %%mm4		\n\t"\

  1035. 	"movq %%mm0, %%mm1			\n\t" /* A1		a1 */\

  1036. 	"paddd %%mm3, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  1037. 	"psubd %%mm3, %%mm1			\n\t" /* A1-B1		a1-b1 */\

  1038. 	"psrad $" #shift ", %%mm0		\n\t"\

  1039. 	"psrad $" #shift ", %%mm1		\n\t"\

  1040. 	"packssdw %%mm7, %%mm7			\n\t" /* A0+B0	a0+b0 */\

  1041. 	"movd %%mm7, " #dst "			\n\t"\

  1042. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  1043. 	"movd %%mm0, 16+" #dst "		\n\t"\

  1044. 	"packssdw %%mm1, %%mm1			\n\t" /* A1-B1	a1-b1 */\

  1045. 	"movd %%mm1, 96+" #dst "		\n\t"\

  1046. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  1047. 	"movd %%mm4, 112+" #dst "		\n\t"\

  1048. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  1049. 	"pmaddwd %%mm2, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  1050. 	"pmaddwd 96(%2), %%mm2			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  1051. 	"movq %%mm5, %%mm1			\n\t" /* A2		a2 */\

  1052. 	"paddd %%mm4, %%mm1			\n\t" /* A2+B2		a2+b2 */\

  1053. 	"psubd %%mm4, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  1054. 	"psrad $" #shift ", %%mm1		\n\t"\

  1055. 	"psrad $" #shift ", %%mm5		\n\t"\

  1056. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  1057. 	"paddd %%mm2, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  1058. 	"psubd %%mm2, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  1059. 	"psrad $" #shift ", %%mm6		\n\t"\

  1060. 	"psrad $" #shift ", %%mm4		\n\t"\

  1061. 	"packssdw %%mm1, %%mm1			\n\t" /* A2+B2	a2+b2 */\

  1062. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  1063. 	"movd %%mm1, 32+" #dst "		\n\t"\

  1064. 	"packssdw %%mm4, %%mm4			\n\t" /* A3-B3	a3-b3 */\

  1065. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  1066. 	"movd %%mm6, 48+" #dst "		\n\t"\

  1067. 	"movd %%mm4, 64+" #dst "		\n\t"\

  1068. 	"movd %%mm5, 80+" #dst "		\n\t"

  1069. 

  1070. 

  1071. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  1072. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  1073. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  1074. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  1075. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  1076. 	"jmp 9f					\n\t"

  1077. 

  1078. 	"#.balign 16				\n\t"\

  1079. 	"5:					\n\t"

  1080. #undef IDCT

  1081. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  1082. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  1083. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  1084. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  1085. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1086. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  1087. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1088. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  1089. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  1090. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  1091. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  1092. 	#rounder ", %%mm4			\n\t"\

  1093. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1094. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  1095. 	#rounder ", %%mm0			\n\t"\

  1096. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  1097. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1098. 	"paddd %%mm1, %%mm0			\n\t" /* A1		a1 */\

  1099. 	"psubd %%mm1, %%mm5			\n\t" /* A2		a2 */\

  1100. 	"movq 8+" #src0 ", %%mm2		\n\t" /* R4	R0	r4	r0 */\

  1101. 	"movq 8+" #src4 ", %%mm3		\n\t" /* R6	R2	r6	r2 */\

  1102. 	"movq 16(%2), %%mm1			\n\t" /* C4	C4	C4	C4 */\

  1103. 	"pmaddwd %%mm2, %%mm1			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1104. 	"movq 24(%2), %%mm7			\n\t" /* -C4	C4	-C4	C4 */\

  1105. 	"pmaddwd %%mm7, %%mm2			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1106. 	"movq 32(%2), %%mm7			\n\t" /* C6	C2	C6	C2 */\

  1107. 	"pmaddwd %%mm3, %%mm7			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  1108. 	"pmaddwd 40(%2), %%mm3			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  1109. 	#rounder ", %%mm1			\n\t"\

  1110. 	"paddd %%mm1, %%mm7			\n\t" /* A0		a0 */\

  1111. 	"paddd %%mm1, %%mm1			\n\t" /* 2C0		2c0 */\

  1112. 	#rounder ", %%mm2			\n\t"\

  1113. 	"psubd %%mm7, %%mm1			\n\t" /* A3		a3 */\

  1114. 	"paddd %%mm2, %%mm3			\n\t" /* A1		a1 */\

  1115. 	"paddd %%mm2, %%mm2			\n\t" /* 2C1		2c1 */\

  1116. 	"psubd %%mm3, %%mm2			\n\t" /* A2		a2 */\

  1117. 	"psrad $" #shift ", %%mm4		\n\t"\

  1118. 	"psrad $" #shift ", %%mm7		\n\t"\

  1119. 	"psrad $" #shift ", %%mm3		\n\t"\

  1120. 	"packssdw %%mm7, %%mm4			\n\t" /* A0	a0 */\

  1121. 	"movq %%mm4, " #dst "			\n\t"\

  1122. 	"psrad $" #shift ", %%mm0		\n\t"\

  1123. 	"packssdw %%mm3, %%mm0			\n\t" /* A1	a1 */\

  1124. 	"movq %%mm0, 16+" #dst "		\n\t"\

  1125. 	"movq %%mm0, 96+" #dst "		\n\t"\

  1126. 	"movq %%mm4, 112+" #dst "		\n\t"\

  1127. 	"psrad $" #shift ", %%mm5		\n\t"\

  1128. 	"psrad $" #shift ", %%mm6		\n\t"\

  1129. 	"psrad $" #shift ", %%mm2		\n\t"\

  1130. 	"packssdw %%mm2, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  1131. 	"movq %%mm5, 32+" #dst "		\n\t"\

  1132. 	"psrad $" #shift ", %%mm1		\n\t"\

  1133. 	"packssdw %%mm1, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  1134. 	"movq %%mm6, 48+" #dst "		\n\t"\

  1135. 	"movq %%mm6, 64+" #dst "		\n\t"\

  1136. 	"movq %%mm5, 80+" #dst "		\n\t"	

  1137. 	

  1138. 

  1139. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  1140. IDCT(    0(%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  1141. //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  1142. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  1143. //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  1144. 	"jmp 9f					\n\t"

  1145. 

  1146. 

  1147. 	"#.balign 16				\n\t"\

  1148. 	"1:					\n\t"

  1149. #undef IDCT

  1150. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  1151. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  1152. 	"movq " #src4 ", %%mm1			\n\t" /* R6	R2	r6	r2 */\

  1153. 	"movq " #src1 ", %%mm2			\n\t" /* R3	R1	r3	r1 */\

  1154. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  1155. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1156. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  1157. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1158. 	"movq 32(%2), %%mm5			\n\t" /* C6	C2	C6	C2 */\

  1159. 	"pmaddwd %%mm1, %%mm5			\n\t" /* C6R6+C2R2	C6r6+C2r2 */\

  1160. 	"movq 40(%2), %%mm6			\n\t" /* -C2	C6	-C2	C6 */\

  1161. 	"pmaddwd %%mm6, %%mm1			\n\t" /* -C2R6+C6R2	-C2r6+C6r2 */\

  1162. 	#rounder ", %%mm4			\n\t"\

  1163. 	"movq %%mm4, %%mm6			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1164. 	"movq 48(%2), %%mm7			\n\t" /* C3	C1	C3	C1 */\

  1165. 	#rounder ", %%mm0			\n\t"\

  1166. 	"pmaddwd %%mm2, %%mm7			\n\t" /* C3R3+C1R1	C3r3+C1r1 */\

  1167. 	"paddd %%mm5, %%mm4			\n\t" /* A0		a0 */\

  1168. 	"psubd %%mm5, %%mm6			\n\t" /* A3		a3 */\

  1169. 	"movq %%mm0, %%mm5			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1170. 	"paddd %%mm1, %%mm0			\n\t" /* A1		a1 */\

  1171. 	"psubd %%mm1, %%mm5			\n\t" /* A2		a2 */\

  1172. 	"movq 64(%2), %%mm1			\n\t"\

  1173. 	"pmaddwd %%mm2, %%mm1			\n\t" /* -C7R3+C3R1	-C7r3+C3r1 */\

  1174. 	"paddd %%mm4, %%mm7			\n\t" /* A0+B0		a0+b0 */\

  1175. 	"paddd %%mm4, %%mm4			\n\t" /* 2A0		2a0 */\

  1176. 	"psubd %%mm7, %%mm4			\n\t" /* A0-B0		a0-b0 */\

  1177. 	"psrad $" #shift ", %%mm7		\n\t"\

  1178. 	"psrad $" #shift ", %%mm4		\n\t"\

  1179. 	"movq %%mm0, %%mm3			\n\t" /* A1		a1 */\

  1180. 	"paddd %%mm1, %%mm0			\n\t" /* A1+B1		a1+b1 */\

  1181. 	"psubd %%mm1, %%mm3			\n\t" /* A1-B1		a1-b1 */\

  1182. 	"psrad $" #shift ", %%mm0		\n\t"\

  1183. 	"psrad $" #shift ", %%mm3		\n\t"\

  1184. 	"packssdw %%mm7, %%mm7			\n\t" /* A0+B0	a0+b0 */\

  1185. 	"movd %%mm7, " #dst "			\n\t"\

  1186. 	"packssdw %%mm0, %%mm0			\n\t" /* A1+B1	a1+b1 */\

  1187. 	"movd %%mm0, 16+" #dst "		\n\t"\

  1188. 	"packssdw %%mm3, %%mm3			\n\t" /* A1-B1	a1-b1 */\

  1189. 	"movd %%mm3, 96+" #dst "		\n\t"\

  1190. 	"packssdw %%mm4, %%mm4			\n\t" /* A0-B0	a0-b0 */\

  1191. 	"movd %%mm4, 112+" #dst "		\n\t"\

  1192. 	"movq 80(%2), %%mm4			\n\t" /* -C1	C5	-C1 	C5 */\

  1193. 	"pmaddwd %%mm2, %%mm4			\n\t" /* -C1R3+C5R1	-C1r3+C5r1 */\

  1194. 	"pmaddwd 96(%2), %%mm2			\n\t" /* -C5R3+C7R1	-C5r3+C7r1 */\

  1195. 	"movq %%mm5, %%mm3			\n\t" /* A2		a2 */\

  1196. 	"paddd %%mm4, %%mm3			\n\t" /* A2+B2		a2+b2 */\

  1197. 	"psubd %%mm4, %%mm5			\n\t" /* a2-B2		a2-b2 */\

  1198. 	"psrad $" #shift ", %%mm3		\n\t"\

  1199. 	"psrad $" #shift ", %%mm5		\n\t"\

  1200. 	"movq %%mm6, %%mm4			\n\t" /* A3		a3 */\

  1201. 	"paddd %%mm2, %%mm6			\n\t" /* A3+B3		a3+b3 */\

  1202. 	"psubd %%mm2, %%mm4			\n\t" /* a3-B3		a3-b3 */\

  1203. 	"psrad $" #shift ", %%mm6		\n\t"\

  1204. 	"packssdw %%mm3, %%mm3			\n\t" /* A2+B2	a2+b2 */\

  1205. 	"movd %%mm3, 32+" #dst "		\n\t"\

  1206. 	"psrad $" #shift ", %%mm4		\n\t"\

  1207. 	"packssdw %%mm6, %%mm6			\n\t" /* A3+B3	a3+b3 */\

  1208. 	"movd %%mm6, 48+" #dst "		\n\t"\

  1209. 	"packssdw %%mm4, %%mm4			\n\t" /* A3-B3	a3-b3 */\

  1210. 	"packssdw %%mm5, %%mm5			\n\t" /* A2-B2	a2-b2 */\

  1211. 	"movd %%mm4, 64+" #dst "		\n\t"\

  1212. 	"movd %%mm5, 80+" #dst "		\n\t"

  1213. 	

  1214. 

  1215. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  1216. IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  1217. IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  1218. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  1219. IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  1220. 	"jmp 9f					\n\t"

  1221. 

  1222. 

  1223. 	"#.balign 16				\n\t"

  1224. 	"7:					\n\t"

  1225. #undef IDCT

  1226. #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \

  1227. 	"movq " #src0 ", %%mm0			\n\t" /* R4	R0	r4	r0 */\

  1228. 	"movq 16(%2), %%mm4			\n\t" /* C4	C4	C4	C4 */\

  1229. 	"pmaddwd %%mm0, %%mm4			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1230. 	"movq 24(%2), %%mm5			\n\t" /* -C4	C4	-C4	C4 */\

  1231. 	"pmaddwd %%mm5, %%mm0			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1232. 	#rounder ", %%mm4			\n\t"\

  1233. 	#rounder ", %%mm0			\n\t"\

  1234. 	"psrad $" #shift ", %%mm4		\n\t"\

  1235. 	"psrad $" #shift ", %%mm0		\n\t"\

  1236. 	"movq 8+" #src0 ", %%mm2		\n\t" /* R4	R0	r4	r0 */\

  1237. 	"movq 16(%2), %%mm1			\n\t" /* C4	C4	C4	C4 */\

  1238. 	"pmaddwd %%mm2, %%mm1			\n\t" /* C4R4+C4R0	C4r4+C4r0 */\

  1239. 	"movq 24(%2), %%mm7			\n\t" /* -C4	C4	-C4	C4 */\

  1240. 	"pmaddwd %%mm7, %%mm2			\n\t" /* -C4R4+C4R0	-C4r4+C4r0 */\

  1241. 	"movq 32(%2), %%mm7			\n\t" /* C6	C2	C6	C2 */\

  1242. 	#rounder ", %%mm1			\n\t"\

  1243. 	#rounder ", %%mm2			\n\t"\

  1244. 	"psrad $" #shift ", %%mm1		\n\t"\

  1245. 	"packssdw %%mm1, %%mm4			\n\t" /* A0	a0 */\

  1246. 	"movq %%mm4, " #dst "			\n\t"\

  1247. 	"psrad $" #shift ", %%mm2		\n\t"\

  1248. 	"packssdw %%mm2, %%mm0			\n\t" /* A1	a1 */\

  1249. 	"movq %%mm0, 16+" #dst "		\n\t"\

  1250. 	"movq %%mm0, 96+" #dst "		\n\t"\

  1251. 	"movq %%mm4, 112+" #dst "		\n\t"\

  1252. 	"movq %%mm0, 32+" #dst "		\n\t"\

  1253. 	"movq %%mm4, 48+" #dst "		\n\t"\

  1254. 	"movq %%mm4, 64+" #dst "		\n\t"\

  1255. 	"movq %%mm0, 80+" #dst "		\n\t"	

  1256. 

  1257. //IDCT(  src0,   src4,   src1,    src5,    dst, rounder, shift)

  1258. IDCT(   0(%1), 64(%1), 32(%1),  96(%1),  0(%0),/nop, 20)

  1259. //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0),/nop, 20)

  1260. IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0),/nop, 20)

  1261. //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)

  1262. 

  1263. 

  1264. #endif

  1265. 

  1266. /*

  1267. Input

  1268.  00 40 04 44 20 60 24 64

  1269.  10 30 14 34 50 70 54 74

  1270.  01 41 03 43 21 61 23 63

  1271.  11 31 13 33 51 71 53 73

  1272.  02 42 06 46 22 62 26 66

  1273.  12 32 16 36 52 72 56 76

  1274.  05 45 07 47 25 65 27 67

  1275.  15 35 17 37 55 75 57 77

  1276.   

  1277. Temp

  1278.  00 04 10 14 20 24 30 34

  1279.  40 44 50 54 60 64 70 74

  1280.  01 03 11 13 21 23 31 33

  1281.  41 43 51 53 61 63 71 73

  1282.  02 06 12 16 22 26 32 36

  1283.  42 46 52 56 62 66 72 76

  1284.  05 07 15 17 25 27 35 37

  1285.  45 47 55 57 65 67 75 77

  1286. */

  1287. 

  1288. "9: \n\t"

  1289. 		:: "r" (block), "r" (temp), "r" (coeffs)

  1290. 		: "%eax"

  1291. 	);

  1292. }

  1293. 

  1294. void ff_simple_idct_mmx(int16_t *block)

  1295. {

  1296.     idct(block);

  1297. }

  1298. 

  1299. //FIXME merge add/put into the idct

  1300. 

  1301. void ff_simple_idct_put_mmx(UINT8 *dest, int line_size, DCTELEM *block)

  1302. {

  1303.     idct(block);

  1304.     put_pixels_clamped_mmx(block, dest, line_size);

  1305. }

  1306. void ff_simple_idct_add_mmx(UINT8 *dest, int line_size, DCTELEM *block)

  1307. {

  1308.     idct(block);

  1309.     add_pixels_clamped_mmx(block, dest, line_size);

  1310. }