You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

474 lines
16KB

  1. /*
  2. * MMX optimized motion estimation
  3. * Copyright (c) 2001 Fabrice Bellard
  4. * Copyright (c) 2002-2004 Michael Niedermayer
  5. *
  6. * mostly by Michael Niedermayer <michaelni@gmx.at>
  7. *
  8. * This file is part of FFmpeg.
  9. *
  10. * FFmpeg is free software; you can redistribute it and/or
  11. * modify it under the terms of the GNU Lesser General Public
  12. * License as published by the Free Software Foundation; either
  13. * version 2.1 of the License, or (at your option) any later version.
  14. *
  15. * FFmpeg is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * Lesser General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU Lesser General Public
  21. * License along with FFmpeg; if not, write to the Free Software
  22. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23. */
  24. #include "libavutil/avassert.h"
  25. #include "libavutil/mem.h"
  26. #include "libavutil/x86/asm.h"
  27. #include "libavcodec/dsputil.h"
  28. #include "dsputil_mmx.h"
  29. #if HAVE_INLINE_ASM
  30. DECLARE_ASM_CONST(8, uint64_t, round_tab)[3]={
  31. 0x0000000000000000ULL,
  32. 0x0001000100010001ULL,
  33. 0x0002000200020002ULL,
  34. };
  35. DECLARE_ASM_CONST(8, uint64_t, bone)= 0x0101010101010101LL;
  36. static inline void sad8_1_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  37. {
  38. x86_reg len= -(x86_reg)stride*h;
  39. __asm__ volatile(
  40. ".p2align 4 \n\t"
  41. "1: \n\t"
  42. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  43. "movq (%2, %%"REG_a"), %%mm2 \n\t"
  44. "movq (%2, %%"REG_a"), %%mm4 \n\t"
  45. "add %3, %%"REG_a" \n\t"
  46. "psubusb %%mm0, %%mm2 \n\t"
  47. "psubusb %%mm4, %%mm0 \n\t"
  48. "movq (%1, %%"REG_a"), %%mm1 \n\t"
  49. "movq (%2, %%"REG_a"), %%mm3 \n\t"
  50. "movq (%2, %%"REG_a"), %%mm5 \n\t"
  51. "psubusb %%mm1, %%mm3 \n\t"
  52. "psubusb %%mm5, %%mm1 \n\t"
  53. "por %%mm2, %%mm0 \n\t"
  54. "por %%mm1, %%mm3 \n\t"
  55. "movq %%mm0, %%mm1 \n\t"
  56. "movq %%mm3, %%mm2 \n\t"
  57. "punpcklbw %%mm7, %%mm0 \n\t"
  58. "punpckhbw %%mm7, %%mm1 \n\t"
  59. "punpcklbw %%mm7, %%mm3 \n\t"
  60. "punpckhbw %%mm7, %%mm2 \n\t"
  61. "paddw %%mm1, %%mm0 \n\t"
  62. "paddw %%mm3, %%mm2 \n\t"
  63. "paddw %%mm2, %%mm0 \n\t"
  64. "paddw %%mm0, %%mm6 \n\t"
  65. "add %3, %%"REG_a" \n\t"
  66. " js 1b \n\t"
  67. : "+a" (len)
  68. : "r" (blk1 - len), "r" (blk2 - len), "r" ((x86_reg)stride)
  69. );
  70. }
  71. static inline void sad8_1_mmxext(uint8_t *blk1, uint8_t *blk2,
  72. int stride, int h)
  73. {
  74. __asm__ volatile(
  75. ".p2align 4 \n\t"
  76. "1: \n\t"
  77. "movq (%1), %%mm0 \n\t"
  78. "movq (%1, %3), %%mm1 \n\t"
  79. "psadbw (%2), %%mm0 \n\t"
  80. "psadbw (%2, %3), %%mm1 \n\t"
  81. "paddw %%mm0, %%mm6 \n\t"
  82. "paddw %%mm1, %%mm6 \n\t"
  83. "lea (%1,%3,2), %1 \n\t"
  84. "lea (%2,%3,2), %2 \n\t"
  85. "sub $2, %0 \n\t"
  86. " jg 1b \n\t"
  87. : "+r" (h), "+r" (blk1), "+r" (blk2)
  88. : "r" ((x86_reg)stride)
  89. );
  90. }
  91. static int sad16_sse2(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)
  92. {
  93. int ret;
  94. __asm__ volatile(
  95. "pxor %%xmm2, %%xmm2 \n\t"
  96. ".p2align 4 \n\t"
  97. "1: \n\t"
  98. "movdqu (%1), %%xmm0 \n\t"
  99. "movdqu (%1, %4), %%xmm1 \n\t"
  100. "psadbw (%2), %%xmm0 \n\t"
  101. "psadbw (%2, %4), %%xmm1 \n\t"
  102. "paddw %%xmm0, %%xmm2 \n\t"
  103. "paddw %%xmm1, %%xmm2 \n\t"
  104. "lea (%1,%4,2), %1 \n\t"
  105. "lea (%2,%4,2), %2 \n\t"
  106. "sub $2, %0 \n\t"
  107. " jg 1b \n\t"
  108. "movhlps %%xmm2, %%xmm0 \n\t"
  109. "paddw %%xmm0, %%xmm2 \n\t"
  110. "movd %%xmm2, %3 \n\t"
  111. : "+r" (h), "+r" (blk1), "+r" (blk2), "=r"(ret)
  112. : "r" ((x86_reg)stride)
  113. );
  114. return ret;
  115. }
  116. static inline void sad8_x2a_mmxext(uint8_t *blk1, uint8_t *blk2,
  117. int stride, int h)
  118. {
  119. __asm__ volatile(
  120. ".p2align 4 \n\t"
  121. "1: \n\t"
  122. "movq (%1), %%mm0 \n\t"
  123. "movq (%1, %3), %%mm1 \n\t"
  124. "pavgb 1(%1), %%mm0 \n\t"
  125. "pavgb 1(%1, %3), %%mm1 \n\t"
  126. "psadbw (%2), %%mm0 \n\t"
  127. "psadbw (%2, %3), %%mm1 \n\t"
  128. "paddw %%mm0, %%mm6 \n\t"
  129. "paddw %%mm1, %%mm6 \n\t"
  130. "lea (%1,%3,2), %1 \n\t"
  131. "lea (%2,%3,2), %2 \n\t"
  132. "sub $2, %0 \n\t"
  133. " jg 1b \n\t"
  134. : "+r" (h), "+r" (blk1), "+r" (blk2)
  135. : "r" ((x86_reg)stride)
  136. );
  137. }
  138. static inline void sad8_y2a_mmxext(uint8_t *blk1, uint8_t *blk2,
  139. int stride, int h)
  140. {
  141. __asm__ volatile(
  142. "movq (%1), %%mm0 \n\t"
  143. "add %3, %1 \n\t"
  144. ".p2align 4 \n\t"
  145. "1: \n\t"
  146. "movq (%1), %%mm1 \n\t"
  147. "movq (%1, %3), %%mm2 \n\t"
  148. "pavgb %%mm1, %%mm0 \n\t"
  149. "pavgb %%mm2, %%mm1 \n\t"
  150. "psadbw (%2), %%mm0 \n\t"
  151. "psadbw (%2, %3), %%mm1 \n\t"
  152. "paddw %%mm0, %%mm6 \n\t"
  153. "paddw %%mm1, %%mm6 \n\t"
  154. "movq %%mm2, %%mm0 \n\t"
  155. "lea (%1,%3,2), %1 \n\t"
  156. "lea (%2,%3,2), %2 \n\t"
  157. "sub $2, %0 \n\t"
  158. " jg 1b \n\t"
  159. : "+r" (h), "+r" (blk1), "+r" (blk2)
  160. : "r" ((x86_reg)stride)
  161. );
  162. }
  163. static inline void sad8_4_mmxext(uint8_t *blk1, uint8_t *blk2,
  164. int stride, int h)
  165. {
  166. __asm__ volatile(
  167. "movq "MANGLE(bone)", %%mm5 \n\t"
  168. "movq (%1), %%mm0 \n\t"
  169. "pavgb 1(%1), %%mm0 \n\t"
  170. "add %3, %1 \n\t"
  171. ".p2align 4 \n\t"
  172. "1: \n\t"
  173. "movq (%1), %%mm1 \n\t"
  174. "movq (%1,%3), %%mm2 \n\t"
  175. "pavgb 1(%1), %%mm1 \n\t"
  176. "pavgb 1(%1,%3), %%mm2 \n\t"
  177. "psubusb %%mm5, %%mm1 \n\t"
  178. "pavgb %%mm1, %%mm0 \n\t"
  179. "pavgb %%mm2, %%mm1 \n\t"
  180. "psadbw (%2), %%mm0 \n\t"
  181. "psadbw (%2,%3), %%mm1 \n\t"
  182. "paddw %%mm0, %%mm6 \n\t"
  183. "paddw %%mm1, %%mm6 \n\t"
  184. "movq %%mm2, %%mm0 \n\t"
  185. "lea (%1,%3,2), %1 \n\t"
  186. "lea (%2,%3,2), %2 \n\t"
  187. "sub $2, %0 \n\t"
  188. " jg 1b \n\t"
  189. : "+r" (h), "+r" (blk1), "+r" (blk2)
  190. : "r" ((x86_reg)stride)
  191. );
  192. }
  193. static inline void sad8_2_mmx(uint8_t *blk1a, uint8_t *blk1b, uint8_t *blk2, int stride, int h)
  194. {
  195. x86_reg len= -(x86_reg)stride*h;
  196. __asm__ volatile(
  197. ".p2align 4 \n\t"
  198. "1: \n\t"
  199. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  200. "movq (%2, %%"REG_a"), %%mm1 \n\t"
  201. "movq (%1, %%"REG_a"), %%mm2 \n\t"
  202. "movq (%2, %%"REG_a"), %%mm3 \n\t"
  203. "punpcklbw %%mm7, %%mm0 \n\t"
  204. "punpcklbw %%mm7, %%mm1 \n\t"
  205. "punpckhbw %%mm7, %%mm2 \n\t"
  206. "punpckhbw %%mm7, %%mm3 \n\t"
  207. "paddw %%mm0, %%mm1 \n\t"
  208. "paddw %%mm2, %%mm3 \n\t"
  209. "movq (%3, %%"REG_a"), %%mm4 \n\t"
  210. "movq (%3, %%"REG_a"), %%mm2 \n\t"
  211. "paddw %%mm5, %%mm1 \n\t"
  212. "paddw %%mm5, %%mm3 \n\t"
  213. "psrlw $1, %%mm1 \n\t"
  214. "psrlw $1, %%mm3 \n\t"
  215. "packuswb %%mm3, %%mm1 \n\t"
  216. "psubusb %%mm1, %%mm4 \n\t"
  217. "psubusb %%mm2, %%mm1 \n\t"
  218. "por %%mm4, %%mm1 \n\t"
  219. "movq %%mm1, %%mm0 \n\t"
  220. "punpcklbw %%mm7, %%mm0 \n\t"
  221. "punpckhbw %%mm7, %%mm1 \n\t"
  222. "paddw %%mm1, %%mm0 \n\t"
  223. "paddw %%mm0, %%mm6 \n\t"
  224. "add %4, %%"REG_a" \n\t"
  225. " js 1b \n\t"
  226. : "+a" (len)
  227. : "r" (blk1a - len), "r" (blk1b -len), "r" (blk2 - len), "r" ((x86_reg)stride)
  228. );
  229. }
  230. static inline void sad8_4_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  231. {
  232. x86_reg len= -(x86_reg)stride*h;
  233. __asm__ volatile(
  234. "movq (%1, %%"REG_a"), %%mm0 \n\t"
  235. "movq 1(%1, %%"REG_a"), %%mm2 \n\t"
  236. "movq %%mm0, %%mm1 \n\t"
  237. "movq %%mm2, %%mm3 \n\t"
  238. "punpcklbw %%mm7, %%mm0 \n\t"
  239. "punpckhbw %%mm7, %%mm1 \n\t"
  240. "punpcklbw %%mm7, %%mm2 \n\t"
  241. "punpckhbw %%mm7, %%mm3 \n\t"
  242. "paddw %%mm2, %%mm0 \n\t"
  243. "paddw %%mm3, %%mm1 \n\t"
  244. ".p2align 4 \n\t"
  245. "1: \n\t"
  246. "movq (%2, %%"REG_a"), %%mm2 \n\t"
  247. "movq 1(%2, %%"REG_a"), %%mm4 \n\t"
  248. "movq %%mm2, %%mm3 \n\t"
  249. "movq %%mm4, %%mm5 \n\t"
  250. "punpcklbw %%mm7, %%mm2 \n\t"
  251. "punpckhbw %%mm7, %%mm3 \n\t"
  252. "punpcklbw %%mm7, %%mm4 \n\t"
  253. "punpckhbw %%mm7, %%mm5 \n\t"
  254. "paddw %%mm4, %%mm2 \n\t"
  255. "paddw %%mm5, %%mm3 \n\t"
  256. "movq 16+"MANGLE(round_tab)", %%mm5 \n\t"
  257. "paddw %%mm2, %%mm0 \n\t"
  258. "paddw %%mm3, %%mm1 \n\t"
  259. "paddw %%mm5, %%mm0 \n\t"
  260. "paddw %%mm5, %%mm1 \n\t"
  261. "movq (%3, %%"REG_a"), %%mm4 \n\t"
  262. "movq (%3, %%"REG_a"), %%mm5 \n\t"
  263. "psrlw $2, %%mm0 \n\t"
  264. "psrlw $2, %%mm1 \n\t"
  265. "packuswb %%mm1, %%mm0 \n\t"
  266. "psubusb %%mm0, %%mm4 \n\t"
  267. "psubusb %%mm5, %%mm0 \n\t"
  268. "por %%mm4, %%mm0 \n\t"
  269. "movq %%mm0, %%mm4 \n\t"
  270. "punpcklbw %%mm7, %%mm0 \n\t"
  271. "punpckhbw %%mm7, %%mm4 \n\t"
  272. "paddw %%mm0, %%mm6 \n\t"
  273. "paddw %%mm4, %%mm6 \n\t"
  274. "movq %%mm2, %%mm0 \n\t"
  275. "movq %%mm3, %%mm1 \n\t"
  276. "add %4, %%"REG_a" \n\t"
  277. " js 1b \n\t"
  278. : "+a" (len)
  279. : "r" (blk1 - len), "r" (blk1 -len + stride), "r" (blk2 - len), "r" ((x86_reg)stride)
  280. );
  281. }
  282. static inline int sum_mmx(void)
  283. {
  284. int ret;
  285. __asm__ volatile(
  286. "movq %%mm6, %%mm0 \n\t"
  287. "psrlq $32, %%mm6 \n\t"
  288. "paddw %%mm0, %%mm6 \n\t"
  289. "movq %%mm6, %%mm0 \n\t"
  290. "psrlq $16, %%mm6 \n\t"
  291. "paddw %%mm0, %%mm6 \n\t"
  292. "movd %%mm6, %0 \n\t"
  293. : "=r" (ret)
  294. );
  295. return ret&0xFFFF;
  296. }
  297. static inline int sum_mmxext(void)
  298. {
  299. int ret;
  300. __asm__ volatile(
  301. "movd %%mm6, %0 \n\t"
  302. : "=r" (ret)
  303. );
  304. return ret;
  305. }
  306. static inline void sad8_x2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  307. {
  308. sad8_2_mmx(blk1, blk1+1, blk2, stride, h);
  309. }
  310. static inline void sad8_y2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
  311. {
  312. sad8_2_mmx(blk1, blk1+stride, blk2, stride, h);
  313. }
  314. #define PIX_SAD(suf)\
  315. static int sad8_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  316. {\
  317. av_assert2(h==8);\
  318. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  319. "pxor %%mm6, %%mm6 \n\t":);\
  320. \
  321. sad8_1_ ## suf(blk1, blk2, stride, 8);\
  322. \
  323. return sum_ ## suf();\
  324. }\
  325. static int sad8_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  326. {\
  327. av_assert2(h==8);\
  328. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  329. "pxor %%mm6, %%mm6 \n\t"\
  330. "movq %0, %%mm5 \n\t"\
  331. :: "m"(round_tab[1]) \
  332. );\
  333. \
  334. sad8_x2a_ ## suf(blk1, blk2, stride, 8);\
  335. \
  336. return sum_ ## suf();\
  337. }\
  338. \
  339. static int sad8_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  340. {\
  341. av_assert2(h==8);\
  342. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  343. "pxor %%mm6, %%mm6 \n\t"\
  344. "movq %0, %%mm5 \n\t"\
  345. :: "m"(round_tab[1]) \
  346. );\
  347. \
  348. sad8_y2a_ ## suf(blk1, blk2, stride, 8);\
  349. \
  350. return sum_ ## suf();\
  351. }\
  352. \
  353. static int sad8_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  354. {\
  355. av_assert2(h==8);\
  356. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  357. "pxor %%mm6, %%mm6 \n\t"\
  358. ::);\
  359. \
  360. sad8_4_ ## suf(blk1, blk2, stride, 8);\
  361. \
  362. return sum_ ## suf();\
  363. }\
  364. \
  365. static int sad16_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  366. {\
  367. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  368. "pxor %%mm6, %%mm6 \n\t":);\
  369. \
  370. sad8_1_ ## suf(blk1 , blk2 , stride, h);\
  371. sad8_1_ ## suf(blk1+8, blk2+8, stride, h);\
  372. \
  373. return sum_ ## suf();\
  374. }\
  375. static int sad16_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  376. {\
  377. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  378. "pxor %%mm6, %%mm6 \n\t"\
  379. "movq %0, %%mm5 \n\t"\
  380. :: "m"(round_tab[1]) \
  381. );\
  382. \
  383. sad8_x2a_ ## suf(blk1 , blk2 , stride, h);\
  384. sad8_x2a_ ## suf(blk1+8, blk2+8, stride, h);\
  385. \
  386. return sum_ ## suf();\
  387. }\
  388. static int sad16_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  389. {\
  390. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  391. "pxor %%mm6, %%mm6 \n\t"\
  392. "movq %0, %%mm5 \n\t"\
  393. :: "m"(round_tab[1]) \
  394. );\
  395. \
  396. sad8_y2a_ ## suf(blk1 , blk2 , stride, h);\
  397. sad8_y2a_ ## suf(blk1+8, blk2+8, stride, h);\
  398. \
  399. return sum_ ## suf();\
  400. }\
  401. static int sad16_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
  402. {\
  403. __asm__ volatile("pxor %%mm7, %%mm7 \n\t"\
  404. "pxor %%mm6, %%mm6 \n\t"\
  405. ::);\
  406. \
  407. sad8_4_ ## suf(blk1 , blk2 , stride, h);\
  408. sad8_4_ ## suf(blk1+8, blk2+8, stride, h);\
  409. \
  410. return sum_ ## suf();\
  411. }\
  412. PIX_SAD(mmx)
  413. PIX_SAD(mmxext)
  414. #endif /* HAVE_INLINE_ASM */
  415. void ff_dsputil_init_pix_mmx(DSPContext* c, AVCodecContext *avctx)
  416. {
  417. #if HAVE_INLINE_ASM
  418. int mm_flags = av_get_cpu_flags();
  419. if (mm_flags & AV_CPU_FLAG_MMX) {
  420. c->pix_abs[0][0] = sad16_mmx;
  421. c->pix_abs[0][1] = sad16_x2_mmx;
  422. c->pix_abs[0][2] = sad16_y2_mmx;
  423. c->pix_abs[0][3] = sad16_xy2_mmx;
  424. c->pix_abs[1][0] = sad8_mmx;
  425. c->pix_abs[1][1] = sad8_x2_mmx;
  426. c->pix_abs[1][2] = sad8_y2_mmx;
  427. c->pix_abs[1][3] = sad8_xy2_mmx;
  428. c->sad[0]= sad16_mmx;
  429. c->sad[1]= sad8_mmx;
  430. }
  431. if (mm_flags & AV_CPU_FLAG_MMXEXT) {
  432. c->pix_abs[0][0] = sad16_mmxext;
  433. c->pix_abs[1][0] = sad8_mmxext;
  434. c->sad[0] = sad16_mmxext;
  435. c->sad[1] = sad8_mmxext;
  436. if(!(avctx->flags & CODEC_FLAG_BITEXACT)){
  437. c->pix_abs[0][1] = sad16_x2_mmxext;
  438. c->pix_abs[0][2] = sad16_y2_mmxext;
  439. c->pix_abs[0][3] = sad16_xy2_mmxext;
  440. c->pix_abs[1][1] = sad8_x2_mmxext;
  441. c->pix_abs[1][2] = sad8_y2_mmxext;
  442. c->pix_abs[1][3] = sad8_xy2_mmxext;
  443. }
  444. }
  445. if ((mm_flags & AV_CPU_FLAG_SSE2) && !(mm_flags & AV_CPU_FLAG_3DNOW) && avctx->codec_id != AV_CODEC_ID_SNOW) {
  446. c->sad[0]= sad16_sse2;
  447. }
  448. #endif /* HAVE_INLINE_ASM */
  449. }