You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

468 lines
16KB

  1. /*
  2. * MMX optimized motion estimation
  3. * Copyright (c) 2001 Fabrice Bellard
  4. * Copyright (c) 2002-2004 Michael Niedermayer
  5. *
  6. * mostly by Michael Niedermayer <michaelni@gmx.at>
  7. *
  8. * This file is part of Libav.
  9. *
  10. * Libav is free software; you can redistribute it and/or
  11. * modify it under the terms of the GNU Lesser General Public
  12. * License as published by the Free Software Foundation; either
  13. * version 2.1 of the License, or (at your option) any later version.
  14. *
  15. * Libav is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * Lesser General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU Lesser General Public
  21. * License along with Libav; if not, write to the Free Software
  22. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23. */
  24. #include "libavutil/x86_cpu.h"
  25. #include "libavcodec/dsputil.h"
  26. #include "dsputil_mmx.h"
  27. #if HAVE_INLINE_ASM
  28. DECLARE_ASM_CONST(8, uint64_t, round_tab)[3]={
  29. 0x0000000000000000ULL,
  30. 0x0001000100010001ULL,
  31. 0x0002000200020002ULL,
  32. };
  33. DECLARE_ASM_CONST(8, uint64_t, bone)= 0x0101010101010101LL;
  34. static inline void sad8_1_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  35. {
  36. x86_reg len= -(stride*h);
  37. __asm__ volatile(
  38. ".p2align 4 \n\t"
  39. "1: \n\t"
  40. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  41. "movq (%2, %%"REG_a"), %%mm2 \n\t"
  42. "movq (%2, %%"REG_a"), %%mm4 \n\t"
  43. "add %3, %%"REG_a" \n\t"
  44. "psubusb %%mm0, %%mm2 \n\t"
  45. "psubusb %%mm4, %%mm0 \n\t"
  46. "movq (%1, %%"REG_a"), %%mm1 \n\t"
  47. "movq (%2, %%"REG_a"), %%mm3 \n\t"
  48. "movq (%2, %%"REG_a"), %%mm5 \n\t"
  49. "psubusb %%mm1, %%mm3 \n\t"
  50. "psubusb %%mm5, %%mm1 \n\t"
  51. "por %%mm2, %%mm0 \n\t"
  52. "por %%mm1, %%mm3 \n\t"
  53. "movq %%mm0, %%mm1 \n\t"
  54. "movq %%mm3, %%mm2 \n\t"
  55. "punpcklbw %%mm7, %%mm0 \n\t"
  56. "punpckhbw %%mm7, %%mm1 \n\t"
  57. "punpcklbw %%mm7, %%mm3 \n\t"
  58. "punpckhbw %%mm7, %%mm2 \n\t"
  59. "paddw %%mm1, %%mm0 \n\t"
  60. "paddw %%mm3, %%mm2 \n\t"
  61. "paddw %%mm2, %%mm0 \n\t"
  62. "paddw %%mm0, %%mm6 \n\t"
  63. "add %3, %%"REG_a" \n\t"
  64. " js 1b \n\t"
  65. : "+a" (len)
  66. : "r" (blk1 - len), "r" (blk2 - len), "r" ((x86_reg)stride)
  67. );
  68. }
  69. static inline void sad8_1_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  70. {
  71. __asm__ volatile(
  72. ".p2align 4 \n\t"
  73. "1: \n\t"
  74. "movq (%1), %%mm0 \n\t"
  75. "movq (%1, %3), %%mm1 \n\t"
  76. "psadbw (%2), %%mm0 \n\t"
  77. "psadbw (%2, %3), %%mm1 \n\t"
  78. "paddw %%mm0, %%mm6 \n\t"
  79. "paddw %%mm1, %%mm6 \n\t"
  80. "lea (%1,%3,2), %1 \n\t"
  81. "lea (%2,%3,2), %2 \n\t"
  82. "sub $2, %0 \n\t"
  83. " jg 1b \n\t"
  84. : "+r" (h), "+r" (blk1), "+r" (blk2)
  85. : "r" ((x86_reg)stride)
  86. );
  87. }
  88. static int sad16_sse2(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)
  89. {
  90. int ret;
  91. __asm__ volatile(
  92. "pxor %%xmm2, %%xmm2 \n\t"
  93. ".p2align 4 \n\t"
  94. "1: \n\t"
  95. "movdqu (%1), %%xmm0 \n\t"
  96. "movdqu (%1, %4), %%xmm1 \n\t"
  97. "psadbw (%2), %%xmm0 \n\t"
  98. "psadbw (%2, %4), %%xmm1 \n\t"
  99. "paddw %%xmm0, %%xmm2 \n\t"
  100. "paddw %%xmm1, %%xmm2 \n\t"
  101. "lea (%1,%4,2), %1 \n\t"
  102. "lea (%2,%4,2), %2 \n\t"
  103. "sub $2, %0 \n\t"
  104. " jg 1b \n\t"
  105. "movhlps %%xmm2, %%xmm0 \n\t"
  106. "paddw %%xmm0, %%xmm2 \n\t"
  107. "movd %%xmm2, %3 \n\t"
  108. : "+r" (h), "+r" (blk1), "+r" (blk2), "=r"(ret)
  109. : "r" ((x86_reg)stride)
  110. );
  111. return ret;
  112. }
  113. static inline void sad8_x2a_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  114. {
  115. __asm__ volatile(
  116. ".p2align 4 \n\t"
  117. "1: \n\t"
  118. "movq (%1), %%mm0 \n\t"
  119. "movq (%1, %3), %%mm1 \n\t"
  120. "pavgb 1(%1), %%mm0 \n\t"
  121. "pavgb 1(%1, %3), %%mm1 \n\t"
  122. "psadbw (%2), %%mm0 \n\t"
  123. "psadbw (%2, %3), %%mm1 \n\t"
  124. "paddw %%mm0, %%mm6 \n\t"
  125. "paddw %%mm1, %%mm6 \n\t"
  126. "lea (%1,%3,2), %1 \n\t"
  127. "lea (%2,%3,2), %2 \n\t"
  128. "sub $2, %0 \n\t"
  129. " jg 1b \n\t"
  130. : "+r" (h), "+r" (blk1), "+r" (blk2)
  131. : "r" ((x86_reg)stride)
  132. );
  133. }
  134. static inline void sad8_y2a_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  135. {
  136. __asm__ volatile(
  137. "movq (%1), %%mm0 \n\t"
  138. "add %3, %1 \n\t"
  139. ".p2align 4 \n\t"
  140. "1: \n\t"
  141. "movq (%1), %%mm1 \n\t"
  142. "movq (%1, %3), %%mm2 \n\t"
  143. "pavgb %%mm1, %%mm0 \n\t"
  144. "pavgb %%mm2, %%mm1 \n\t"
  145. "psadbw (%2), %%mm0 \n\t"
  146. "psadbw (%2, %3), %%mm1 \n\t"
  147. "paddw %%mm0, %%mm6 \n\t"
  148. "paddw %%mm1, %%mm6 \n\t"
  149. "movq %%mm2, %%mm0 \n\t"
  150. "lea (%1,%3,2), %1 \n\t"
  151. "lea (%2,%3,2), %2 \n\t"
  152. "sub $2, %0 \n\t"
  153. " jg 1b \n\t"
  154. : "+r" (h), "+r" (blk1), "+r" (blk2)
  155. : "r" ((x86_reg)stride)
  156. );
  157. }
  158. static inline void sad8_4_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  159. {
  160. __asm__ volatile(
  161. "movq "MANGLE(bone)", %%mm5 \n\t"
  162. "movq (%1), %%mm0 \n\t"
  163. "pavgb 1(%1), %%mm0 \n\t"
  164. "add %3, %1 \n\t"
  165. ".p2align 4 \n\t"
  166. "1: \n\t"
  167. "movq (%1), %%mm1 \n\t"
  168. "movq (%1,%3), %%mm2 \n\t"
  169. "pavgb 1(%1), %%mm1 \n\t"
  170. "pavgb 1(%1,%3), %%mm2 \n\t"
  171. "psubusb %%mm5, %%mm1 \n\t"
  172. "pavgb %%mm1, %%mm0 \n\t"
  173. "pavgb %%mm2, %%mm1 \n\t"
  174. "psadbw (%2), %%mm0 \n\t"
  175. "psadbw (%2,%3), %%mm1 \n\t"
  176. "paddw %%mm0, %%mm6 \n\t"
  177. "paddw %%mm1, %%mm6 \n\t"
  178. "movq %%mm2, %%mm0 \n\t"
  179. "lea (%1,%3,2), %1 \n\t"
  180. "lea (%2,%3,2), %2 \n\t"
  181. "sub $2, %0 \n\t"
  182. " jg 1b \n\t"
  183. : "+r" (h), "+r" (blk1), "+r" (blk2)
  184. : "r" ((x86_reg)stride)
  185. );
  186. }
  187. static inline void sad8_2_mmx(uint8_t *blk1a, uint8_t *blk1b, uint8_t *blk2, int stride, int h)
  188. {
  189. x86_reg len= -(stride*h);
  190. __asm__ volatile(
  191. ".p2align 4 \n\t"
  192. "1: \n\t"
  193. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  194. "movq (%2, %%"REG_a"), %%mm1 \n\t"
  195. "movq (%1, %%"REG_a"), %%mm2 \n\t"
  196. "movq (%2, %%"REG_a"), %%mm3 \n\t"
  197. "punpcklbw %%mm7, %%mm0 \n\t"
  198. "punpcklbw %%mm7, %%mm1 \n\t"
  199. "punpckhbw %%mm7, %%mm2 \n\t"
  200. "punpckhbw %%mm7, %%mm3 \n\t"
  201. "paddw %%mm0, %%mm1 \n\t"
  202. "paddw %%mm2, %%mm3 \n\t"
  203. "movq (%3, %%"REG_a"), %%mm4 \n\t"
  204. "movq (%3, %%"REG_a"), %%mm2 \n\t"
  205. "paddw %%mm5, %%mm1 \n\t"
  206. "paddw %%mm5, %%mm3 \n\t"
  207. "psrlw $1, %%mm1 \n\t"
  208. "psrlw $1, %%mm3 \n\t"
  209. "packuswb %%mm3, %%mm1 \n\t"
  210. "psubusb %%mm1, %%mm4 \n\t"
  211. "psubusb %%mm2, %%mm1 \n\t"
  212. "por %%mm4, %%mm1 \n\t"
  213. "movq %%mm1, %%mm0 \n\t"
  214. "punpcklbw %%mm7, %%mm0 \n\t"
  215. "punpckhbw %%mm7, %%mm1 \n\t"
  216. "paddw %%mm1, %%mm0 \n\t"
  217. "paddw %%mm0, %%mm6 \n\t"
  218. "add %4, %%"REG_a" \n\t"
  219. " js 1b \n\t"
  220. : "+a" (len)
  221. : "r" (blk1a - len), "r" (blk1b -len), "r" (blk2 - len), "r" ((x86_reg)stride)
  222. );
  223. }
  224. static inline void sad8_4_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  225. {
  226. x86_reg len= -(stride*h);
  227. __asm__ volatile(
  228. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  229. "movq 1(%1, %%"REG_a"), %%mm2 \n\t"
  230. "movq %%mm0, %%mm1 \n\t"
  231. "movq %%mm2, %%mm3 \n\t"
  232. "punpcklbw %%mm7, %%mm0 \n\t"
  233. "punpckhbw %%mm7, %%mm1 \n\t"
  234. "punpcklbw %%mm7, %%mm2 \n\t"
  235. "punpckhbw %%mm7, %%mm3 \n\t"
  236. "paddw %%mm2, %%mm0 \n\t"
  237. "paddw %%mm3, %%mm1 \n\t"
  238. ".p2align 4 \n\t"
  239. "1: \n\t"
  240. "movq (%2, %%"REG_a"), %%mm2 \n\t"
  241. "movq 1(%2, %%"REG_a"), %%mm4 \n\t"
  242. "movq %%mm2, %%mm3 \n\t"
  243. "movq %%mm4, %%mm5 \n\t"
  244. "punpcklbw %%mm7, %%mm2 \n\t"
  245. "punpckhbw %%mm7, %%mm3 \n\t"
  246. "punpcklbw %%mm7, %%mm4 \n\t"
  247. "punpckhbw %%mm7, %%mm5 \n\t"
  248. "paddw %%mm4, %%mm2 \n\t"
  249. "paddw %%mm5, %%mm3 \n\t"
  250. "movq 16+"MANGLE(round_tab)", %%mm5 \n\t"
  251. "paddw %%mm2, %%mm0 \n\t"
  252. "paddw %%mm3, %%mm1 \n\t"
  253. "paddw %%mm5, %%mm0 \n\t"
  254. "paddw %%mm5, %%mm1 \n\t"
  255. "movq (%3, %%"REG_a"), %%mm4 \n\t"
  256. "movq (%3, %%"REG_a"), %%mm5 \n\t"
  257. "psrlw $2, %%mm0 \n\t"
  258. "psrlw $2, %%mm1 \n\t"
  259. "packuswb %%mm1, %%mm0 \n\t"
  260. "psubusb %%mm0, %%mm4 \n\t"
  261. "psubusb %%mm5, %%mm0 \n\t"
  262. "por %%mm4, %%mm0 \n\t"
  263. "movq %%mm0, %%mm4 \n\t"
  264. "punpcklbw %%mm7, %%mm0 \n\t"
  265. "punpckhbw %%mm7, %%mm4 \n\t"
  266. "paddw %%mm0, %%mm6 \n\t"
  267. "paddw %%mm4, %%mm6 \n\t"
  268. "movq %%mm2, %%mm0 \n\t"
  269. "movq %%mm3, %%mm1 \n\t"
  270. "add %4, %%"REG_a" \n\t"
  271. " js 1b \n\t"
  272. : "+a" (len)
  273. : "r" (blk1 - len), "r" (blk1 -len + stride), "r" (blk2 - len), "r" ((x86_reg)stride)
  274. );
  275. }
  276. static inline int sum_mmx(void)
  277. {
  278. int ret;
  279. __asm__ volatile(
  280. "movq %%mm6, %%mm0 \n\t"
  281. "psrlq $32, %%mm6 \n\t"
  282. "paddw %%mm0, %%mm6 \n\t"
  283. "movq %%mm6, %%mm0 \n\t"
  284. "psrlq $16, %%mm6 \n\t"
  285. "paddw %%mm0, %%mm6 \n\t"
  286. "movd %%mm6, %0 \n\t"
  287. : "=r" (ret)
  288. );
  289. return ret&0xFFFF;
  290. }
  291. static inline int sum_mmx2(void)
  292. {
  293. int ret;
  294. __asm__ volatile(
  295. "movd %%mm6, %0 \n\t"
  296. : "=r" (ret)
  297. );
  298. return ret;
  299. }
  300. static inline void sad8_x2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  301. {
  302. sad8_2_mmx(blk1, blk1+1, blk2, stride, h);
  303. }
  304. static inline void sad8_y2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  305. {
  306. sad8_2_mmx(blk1, blk1+stride, blk2, stride, h);
  307. }
  308. #define PIX_SAD(suf)\
  309. static int sad8_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  310. {\
  311. assert(h==8);\
  312. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  313. "pxor %%mm6, %%mm6 \n\t":);\
  314. \
  315. sad8_1_ ## suf(blk1, blk2, stride, 8);\
  316. \
  317. return sum_ ## suf();\
  318. }\
  319. static int sad8_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  320. {\
  321. assert(h==8);\
  322. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  323. "pxor %%mm6, %%mm6 \n\t"\
  324. "movq %0, %%mm5 \n\t"\
  325. :: "m"(round_tab[1]) \
  326. );\
  327. \
  328. sad8_x2a_ ## suf(blk1, blk2, stride, 8);\
  329. \
  330. return sum_ ## suf();\
  331. }\
  332. \
  333. static int sad8_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  334. {\
  335. assert(h==8);\
  336. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  337. "pxor %%mm6, %%mm6 \n\t"\
  338. "movq %0, %%mm5 \n\t"\
  339. :: "m"(round_tab[1]) \
  340. );\
  341. \
  342. sad8_y2a_ ## suf(blk1, blk2, stride, 8);\
  343. \
  344. return sum_ ## suf();\
  345. }\
  346. \
  347. static int sad8_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  348. {\
  349. assert(h==8);\
  350. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  351. "pxor %%mm6, %%mm6 \n\t"\
  352. ::);\
  353. \
  354. sad8_4_ ## suf(blk1, blk2, stride, 8);\
  355. \
  356. return sum_ ## suf();\
  357. }\
  358. \
  359. static int sad16_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  360. {\
  361. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  362. "pxor %%mm6, %%mm6 \n\t":);\
  363. \
  364. sad8_1_ ## suf(blk1 , blk2 , stride, h);\
  365. sad8_1_ ## suf(blk1+8, blk2+8, stride, h);\
  366. \
  367. return sum_ ## suf();\
  368. }\
  369. static int sad16_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  370. {\
  371. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  372. "pxor %%mm6, %%mm6 \n\t"\
  373. "movq %0, %%mm5 \n\t"\
  374. :: "m"(round_tab[1]) \
  375. );\
  376. \
  377. sad8_x2a_ ## suf(blk1 , blk2 , stride, h);\
  378. sad8_x2a_ ## suf(blk1+8, blk2+8, stride, h);\
  379. \
  380. return sum_ ## suf();\
  381. }\
  382. static int sad16_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  383. {\
  384. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  385. "pxor %%mm6, %%mm6 \n\t"\
  386. "movq %0, %%mm5 \n\t"\
  387. :: "m"(round_tab[1]) \
  388. );\
  389. \
  390. sad8_y2a_ ## suf(blk1 , blk2 , stride, h);\
  391. sad8_y2a_ ## suf(blk1+8, blk2+8, stride, h);\
  392. \
  393. return sum_ ## suf();\
  394. }\
  395. static int sad16_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  396. {\
  397. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  398. "pxor %%mm6, %%mm6 \n\t"\
  399. ::);\
  400. \
  401. sad8_4_ ## suf(blk1 , blk2 , stride, h);\
  402. sad8_4_ ## suf(blk1+8, blk2+8, stride, h);\
  403. \
  404. return sum_ ## suf();\
  405. }\
  406. PIX_SAD(mmx)
  407. PIX_SAD(mmx2)
  408. #endif /* HAVE_INLINE_ASM */
  409. void ff_dsputil_init_pix_mmx(DSPContext* c, AVCodecContext *avctx)
  410. {
  411. #if HAVE_INLINE_ASM
  412. int mm_flags = av_get_cpu_flags();
  413. if (mm_flags & AV_CPU_FLAG_MMX) {
  414. c->pix_abs[0][0] = sad16_mmx;
  415. c->pix_abs[0][1] = sad16_x2_mmx;
  416. c->pix_abs[0][2] = sad16_y2_mmx;
  417. c->pix_abs[0][3] = sad16_xy2_mmx;
  418. c->pix_abs[1][0] = sad8_mmx;
  419. c->pix_abs[1][1] = sad8_x2_mmx;
  420. c->pix_abs[1][2] = sad8_y2_mmx;
  421. c->pix_abs[1][3] = sad8_xy2_mmx;
  422. c->sad[0]= sad16_mmx;
  423. c->sad[1]= sad8_mmx;
  424. }
  425. if (mm_flags & AV_CPU_FLAG_MMXEXT) {
  426. c->pix_abs[0][0] = sad16_mmx2;
  427. c->pix_abs[1][0] = sad8_mmx2;
  428. c->sad[0]= sad16_mmx2;
  429. c->sad[1]= sad8_mmx2;
  430. if(!(avctx->flags & CODEC_FLAG_BITEXACT)){
  431. c->pix_abs[0][1] = sad16_x2_mmx2;
  432. c->pix_abs[0][2] = sad16_y2_mmx2;
  433. c->pix_abs[0][3] = sad16_xy2_mmx2;
  434. c->pix_abs[1][1] = sad8_x2_mmx2;
  435. c->pix_abs[1][2] = sad8_y2_mmx2;
  436. c->pix_abs[1][3] = sad8_xy2_mmx2;
  437. }
  438. }
  439. if ((mm_flags & AV_CPU_FLAG_SSE2) && !(mm_flags & AV_CPU_FLAG_3DNOW) && avctx->codec_id != AV_CODEC_ID_SNOW) {
  440. c->sad[0]= sad16_sse2;
  441. }
  442. #endif /* HAVE_INLINE_ASM */
  443. }