You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

383 lines
17KB

  1. /*
  2. * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
  3. *
  4. * This file is part of Libav.
  5. *
  6. * Libav is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2.1 of the License, or (at your option) any later version.
  10. *
  11. * Libav is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with Libav; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19. */
  20. #include "libavutil/cpu.h"
  21. #include "libavutil/x86_cpu.h"
  22. #include "libavcodec/h264dsp.h"
  23. #include "dsputil_mmx.h"
  24. DECLARE_ALIGNED(8, static const uint64_t, ff_pb_3_1 ) = 0x0103010301030103ULL;
  25. /***********************************/
  26. /* IDCT */
  27. void ff_h264_idct_add_mmx (uint8_t *dst, int16_t *block, int stride);
  28. void ff_h264_idct8_add_mmx (uint8_t *dst, int16_t *block, int stride);
  29. void ff_h264_idct8_add_sse2 (uint8_t *dst, int16_t *block, int stride);
  30. void ff_h264_idct_dc_add_mmx2 (uint8_t *dst, int16_t *block, int stride);
  31. void ff_h264_idct8_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride);
  32. void ff_h264_idct_add16_mmx (uint8_t *dst, const int *block_offset,
  33. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  34. void ff_h264_idct8_add4_mmx (uint8_t *dst, const int *block_offset,
  35. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  36. void ff_h264_idct_add16_mmx2 (uint8_t *dst, const int *block_offset,
  37. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  38. void ff_h264_idct_add16intra_mmx (uint8_t *dst, const int *block_offset,
  39. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  40. void ff_h264_idct_add16intra_mmx2(uint8_t *dst, const int *block_offset,
  41. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  42. void ff_h264_idct8_add4_mmx2 (uint8_t *dst, const int *block_offset,
  43. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  44. void ff_h264_idct8_add4_sse2 (uint8_t *dst, const int *block_offset,
  45. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  46. void ff_h264_idct_add8_mmx (uint8_t **dest, const int *block_offset,
  47. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  48. void ff_h264_idct_add8_mmx2 (uint8_t **dest, const int *block_offset,
  49. DCTELEM *block, int stride, const uint8_t nnzc[6*8]);
  50. void ff_h264_idct_add16_sse2 (uint8_t *dst, const int *block_offset, DCTELEM *block,
  51. int stride, const uint8_t nnzc[6*8]);
  52. void ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset, DCTELEM *block,
  53. int stride, const uint8_t nnzc[6*8]);
  54. void ff_h264_idct_add8_sse2 (uint8_t **dest, const int *block_offset, DCTELEM *block,
  55. int stride, const uint8_t nnzc[6*8]);
  56. void ff_h264_luma_dc_dequant_idct_mmx (DCTELEM *output, DCTELEM *input, int qmul);
  57. void ff_h264_luma_dc_dequant_idct_sse2(DCTELEM *output, DCTELEM *input, int qmul);
  58. /***********************************/
  59. /* deblocking */
  60. #define h264_loop_filter_strength_iteration_mmx2(bS, nz, ref, mv, bidir, edges, step, mask_mv, dir, d_idx, mask_dir) \
  61. do { \
  62. x86_reg b_idx; \
  63. mask_mv <<= 3; \
  64. for( b_idx=0; b_idx<edges; b_idx+=step ) { \
  65. if (!mask_dir) \
  66. __asm__ volatile( \
  67. "pxor %%mm0, %%mm0 \n\t" \
  68. :: \
  69. ); \
  70. if(!(mask_mv & b_idx)) { \
  71. if(bidir) { \
  72. __asm__ volatile( \
  73. "movd %a3(%0,%2), %%mm2 \n" \
  74. "punpckldq %a4(%0,%2), %%mm2 \n" /* { ref0[bn], ref1[bn] } */ \
  75. "pshufw $0x44, 12(%0,%2), %%mm0 \n" /* { ref0[b], ref0[b] } */ \
  76. "pshufw $0x44, 52(%0,%2), %%mm1 \n" /* { ref1[b], ref1[b] } */ \
  77. "pshufw $0x4E, %%mm2, %%mm3 \n" \
  78. "psubb %%mm2, %%mm0 \n" /* { ref0[b]!=ref0[bn], ref0[b]!=ref1[bn] } */ \
  79. "psubb %%mm3, %%mm1 \n" /* { ref1[b]!=ref1[bn], ref1[b]!=ref0[bn] } */ \
  80. \
  81. "por %%mm1, %%mm0 \n" \
  82. "movq %a5(%1,%2,4), %%mm1 \n" \
  83. "movq %a6(%1,%2,4), %%mm2 \n" \
  84. "movq %%mm1, %%mm3 \n" \
  85. "movq %%mm2, %%mm4 \n" \
  86. "psubw 48(%1,%2,4), %%mm1 \n" \
  87. "psubw 56(%1,%2,4), %%mm2 \n" \
  88. "psubw 208(%1,%2,4), %%mm3 \n" \
  89. "psubw 216(%1,%2,4), %%mm4 \n" \
  90. "packsswb %%mm2, %%mm1 \n" \
  91. "packsswb %%mm4, %%mm3 \n" \
  92. "paddb %%mm6, %%mm1 \n" \
  93. "paddb %%mm6, %%mm3 \n" \
  94. "psubusb %%mm5, %%mm1 \n" /* abs(mv[b] - mv[bn]) >= limit */ \
  95. "psubusb %%mm5, %%mm3 \n" \
  96. "packsswb %%mm3, %%mm1 \n" \
  97. \
  98. "por %%mm1, %%mm0 \n" \
  99. "movq %a7(%1,%2,4), %%mm1 \n" \
  100. "movq %a8(%1,%2,4), %%mm2 \n" \
  101. "movq %%mm1, %%mm3 \n" \
  102. "movq %%mm2, %%mm4 \n" \
  103. "psubw 48(%1,%2,4), %%mm1 \n" \
  104. "psubw 56(%1,%2,4), %%mm2 \n" \
  105. "psubw 208(%1,%2,4), %%mm3 \n" \
  106. "psubw 216(%1,%2,4), %%mm4 \n" \
  107. "packsswb %%mm2, %%mm1 \n" \
  108. "packsswb %%mm4, %%mm3 \n" \
  109. "paddb %%mm6, %%mm1 \n" \
  110. "paddb %%mm6, %%mm3 \n" \
  111. "psubusb %%mm5, %%mm1 \n" /* abs(mv[b] - mv[bn]) >= limit */ \
  112. "psubusb %%mm5, %%mm3 \n" \
  113. "packsswb %%mm3, %%mm1 \n" \
  114. \
  115. "pshufw $0x4E, %%mm1, %%mm1 \n" \
  116. "por %%mm1, %%mm0 \n" \
  117. "pshufw $0x4E, %%mm0, %%mm1 \n" \
  118. "pminub %%mm1, %%mm0 \n" \
  119. ::"r"(ref), \
  120. "r"(mv), \
  121. "r"(b_idx), \
  122. "i"(d_idx+12), \
  123. "i"(d_idx+52), \
  124. "i"(d_idx*4+48), \
  125. "i"(d_idx*4+56), \
  126. "i"(d_idx*4+208), \
  127. "i"(d_idx*4+216) \
  128. ); \
  129. } else { \
  130. __asm__ volatile( \
  131. "movd 12(%0,%2), %%mm0 \n" \
  132. "psubb %a3(%0,%2), %%mm0 \n" /* ref[b] != ref[bn] */ \
  133. "movq 48(%1,%2,4), %%mm1 \n" \
  134. "movq 56(%1,%2,4), %%mm2 \n" \
  135. "psubw %a4(%1,%2,4), %%mm1 \n" \
  136. "psubw %a5(%1,%2,4), %%mm2 \n" \
  137. "packsswb %%mm2, %%mm1 \n" \
  138. "paddb %%mm6, %%mm1 \n" \
  139. "psubusb %%mm5, %%mm1 \n" /* abs(mv[b] - mv[bn]) >= limit */ \
  140. "packsswb %%mm1, %%mm1 \n" \
  141. "por %%mm1, %%mm0 \n" \
  142. ::"r"(ref), \
  143. "r"(mv), \
  144. "r"(b_idx), \
  145. "i"(d_idx+12), \
  146. "i"(d_idx*4+48), \
  147. "i"(d_idx*4+56) \
  148. ); \
  149. } \
  150. } \
  151. __asm__ volatile( \
  152. "movd 12(%0,%1), %%mm1 \n" \
  153. "por %a2(%0,%1), %%mm1 \n" /* nnz[b] || nnz[bn] */ \
  154. ::"r"(nnz), \
  155. "r"(b_idx), \
  156. "i"(d_idx+12) \
  157. ); \
  158. __asm__ volatile( \
  159. "pminub %%mm7, %%mm1 \n" \
  160. "pminub %%mm7, %%mm0 \n" \
  161. "psllw $1, %%mm1 \n" \
  162. "pxor %%mm2, %%mm2 \n" \
  163. "pmaxub %%mm0, %%mm1 \n" \
  164. "punpcklbw %%mm2, %%mm1 \n" \
  165. "movq %%mm1, %a1(%0,%2) \n" \
  166. ::"r"(bS), \
  167. "i"(32*dir), \
  168. "r"(b_idx) \
  169. :"memory" \
  170. ); \
  171. } \
  172. } while (0)
  173. static void h264_loop_filter_strength_mmx2( int16_t bS[2][4][4], uint8_t nnz[40], int8_t ref[2][40], int16_t mv[2][40][2],
  174. int bidir, int edges, int step, int mask_mv0, int mask_mv1, int field ) {
  175. __asm__ volatile(
  176. "movq %0, %%mm7 \n"
  177. "movq %1, %%mm6 \n"
  178. ::"m"(ff_pb_1), "m"(ff_pb_3)
  179. );
  180. if(field)
  181. __asm__ volatile(
  182. "movq %0, %%mm6 \n"
  183. ::"m"(ff_pb_3_1)
  184. );
  185. __asm__ volatile(
  186. "movq %%mm6, %%mm5 \n"
  187. "paddb %%mm5, %%mm5 \n"
  188. :);
  189. // could do a special case for dir==0 && edges==1, but it only reduces the
  190. // average filter time by 1.2%
  191. step <<= 3;
  192. edges <<= 3;
  193. h264_loop_filter_strength_iteration_mmx2(bS, nnz, ref, mv, bidir, edges, step, mask_mv1, 1, -8, 0);
  194. h264_loop_filter_strength_iteration_mmx2(bS, nnz, ref, mv, bidir, 32, 8, mask_mv0, 0, -1, -1);
  195. __asm__ volatile(
  196. "movq (%0), %%mm0 \n\t"
  197. "movq 8(%0), %%mm1 \n\t"
  198. "movq 16(%0), %%mm2 \n\t"
  199. "movq 24(%0), %%mm3 \n\t"
  200. TRANSPOSE4(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4)
  201. "movq %%mm0, (%0) \n\t"
  202. "movq %%mm3, 8(%0) \n\t"
  203. "movq %%mm4, 16(%0) \n\t"
  204. "movq %%mm2, 24(%0) \n\t"
  205. ::"r"(bS[0])
  206. :"memory"
  207. );
  208. }
  209. #define LF_FUNC(DIR, TYPE, OPT) \
  210. void ff_x264_deblock_ ## DIR ## _ ## TYPE ## _ ## OPT (uint8_t *pix, int stride, \
  211. int alpha, int beta, int8_t *tc0);
  212. #define LF_IFUNC(DIR, TYPE, OPT) \
  213. void ff_x264_deblock_ ## DIR ## _ ## TYPE ## _ ## OPT (uint8_t *pix, int stride, \
  214. int alpha, int beta);
  215. LF_FUNC (h, chroma, mmxext)
  216. LF_IFUNC(h, chroma_intra, mmxext)
  217. LF_FUNC (v, chroma, mmxext)
  218. LF_IFUNC(v, chroma_intra, mmxext)
  219. LF_FUNC (h, luma, mmxext)
  220. LF_IFUNC(h, luma_intra, mmxext)
  221. #if HAVE_YASM && ARCH_X86_32
  222. LF_FUNC (v8, luma, mmxext)
  223. static void ff_x264_deblock_v_luma_mmxext(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
  224. {
  225. if((tc0[0] & tc0[1]) >= 0)
  226. ff_x264_deblock_v8_luma_mmxext(pix+0, stride, alpha, beta, tc0);
  227. if((tc0[2] & tc0[3]) >= 0)
  228. ff_x264_deblock_v8_luma_mmxext(pix+8, stride, alpha, beta, tc0+2);
  229. }
  230. LF_IFUNC(v8, luma_intra, mmxext)
  231. static void ff_x264_deblock_v_luma_intra_mmxext(uint8_t *pix, int stride, int alpha, int beta)
  232. {
  233. ff_x264_deblock_v8_luma_intra_mmxext(pix+0, stride, alpha, beta);
  234. ff_x264_deblock_v8_luma_intra_mmxext(pix+8, stride, alpha, beta);
  235. }
  236. #endif
  237. LF_FUNC (h, luma, sse2)
  238. LF_IFUNC(h, luma_intra, sse2)
  239. LF_FUNC (v, luma, sse2)
  240. LF_IFUNC(v, luma_intra, sse2)
  241. /***********************************/
  242. /* weighted prediction */
  243. #define H264_WEIGHT(W, H, OPT) \
  244. void ff_h264_weight_ ## W ## x ## H ## _ ## OPT(uint8_t *dst, \
  245. int stride, int log2_denom, int weight, int offset);
  246. #define H264_BIWEIGHT(W, H, OPT) \
  247. void ff_h264_biweight_ ## W ## x ## H ## _ ## OPT(uint8_t *dst, \
  248. uint8_t *src, int stride, int log2_denom, int weightd, \
  249. int weights, int offset);
  250. #define H264_BIWEIGHT_MMX(W,H) \
  251. H264_WEIGHT (W, H, mmx2) \
  252. H264_BIWEIGHT(W, H, mmx2)
  253. #define H264_BIWEIGHT_MMX_SSE(W,H) \
  254. H264_BIWEIGHT_MMX(W, H) \
  255. H264_WEIGHT (W, H, sse2) \
  256. H264_BIWEIGHT (W, H, sse2) \
  257. H264_BIWEIGHT (W, H, ssse3)
  258. H264_BIWEIGHT_MMX_SSE(16, 16)
  259. H264_BIWEIGHT_MMX_SSE(16, 8)
  260. H264_BIWEIGHT_MMX_SSE( 8, 16)
  261. H264_BIWEIGHT_MMX_SSE( 8, 8)
  262. H264_BIWEIGHT_MMX_SSE( 8, 4)
  263. H264_BIWEIGHT_MMX ( 4, 8)
  264. H264_BIWEIGHT_MMX ( 4, 4)
  265. H264_BIWEIGHT_MMX ( 4, 2)
  266. void ff_h264dsp_init_x86(H264DSPContext *c)
  267. {
  268. int mm_flags = av_get_cpu_flags();
  269. if (mm_flags & AV_CPU_FLAG_MMX2) {
  270. c->h264_loop_filter_strength= h264_loop_filter_strength_mmx2;
  271. }
  272. #if HAVE_YASM
  273. if (mm_flags & AV_CPU_FLAG_MMX) {
  274. c->h264_idct_dc_add=
  275. c->h264_idct_add= ff_h264_idct_add_mmx;
  276. c->h264_idct8_dc_add=
  277. c->h264_idct8_add= ff_h264_idct8_add_mmx;
  278. c->h264_idct_add16 = ff_h264_idct_add16_mmx;
  279. c->h264_idct8_add4 = ff_h264_idct8_add4_mmx;
  280. c->h264_idct_add8 = ff_h264_idct_add8_mmx;
  281. c->h264_idct_add16intra= ff_h264_idct_add16intra_mmx;
  282. c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_mmx;
  283. if (mm_flags & AV_CPU_FLAG_MMX2) {
  284. c->h264_idct_dc_add= ff_h264_idct_dc_add_mmx2;
  285. c->h264_idct8_dc_add= ff_h264_idct8_dc_add_mmx2;
  286. c->h264_idct_add16 = ff_h264_idct_add16_mmx2;
  287. c->h264_idct8_add4 = ff_h264_idct8_add4_mmx2;
  288. c->h264_idct_add8 = ff_h264_idct_add8_mmx2;
  289. c->h264_idct_add16intra= ff_h264_idct_add16intra_mmx2;
  290. c->h264_v_loop_filter_chroma= ff_x264_deblock_v_chroma_mmxext;
  291. c->h264_h_loop_filter_chroma= ff_x264_deblock_h_chroma_mmxext;
  292. c->h264_v_loop_filter_chroma_intra= ff_x264_deblock_v_chroma_intra_mmxext;
  293. c->h264_h_loop_filter_chroma_intra= ff_x264_deblock_h_chroma_intra_mmxext;
  294. #if ARCH_X86_32
  295. c->h264_v_loop_filter_luma= ff_x264_deblock_v_luma_mmxext;
  296. c->h264_h_loop_filter_luma= ff_x264_deblock_h_luma_mmxext;
  297. c->h264_v_loop_filter_luma_intra = ff_x264_deblock_v_luma_intra_mmxext;
  298. c->h264_h_loop_filter_luma_intra = ff_x264_deblock_h_luma_intra_mmxext;
  299. #endif
  300. c->weight_h264_pixels_tab[0]= ff_h264_weight_16x16_mmx2;
  301. c->weight_h264_pixels_tab[1]= ff_h264_weight_16x8_mmx2;
  302. c->weight_h264_pixels_tab[2]= ff_h264_weight_8x16_mmx2;
  303. c->weight_h264_pixels_tab[3]= ff_h264_weight_8x8_mmx2;
  304. c->weight_h264_pixels_tab[4]= ff_h264_weight_8x4_mmx2;
  305. c->weight_h264_pixels_tab[5]= ff_h264_weight_4x8_mmx2;
  306. c->weight_h264_pixels_tab[6]= ff_h264_weight_4x4_mmx2;
  307. c->weight_h264_pixels_tab[7]= ff_h264_weight_4x2_mmx2;
  308. c->biweight_h264_pixels_tab[0]= ff_h264_biweight_16x16_mmx2;
  309. c->biweight_h264_pixels_tab[1]= ff_h264_biweight_16x8_mmx2;
  310. c->biweight_h264_pixels_tab[2]= ff_h264_biweight_8x16_mmx2;
  311. c->biweight_h264_pixels_tab[3]= ff_h264_biweight_8x8_mmx2;
  312. c->biweight_h264_pixels_tab[4]= ff_h264_biweight_8x4_mmx2;
  313. c->biweight_h264_pixels_tab[5]= ff_h264_biweight_4x8_mmx2;
  314. c->biweight_h264_pixels_tab[6]= ff_h264_biweight_4x4_mmx2;
  315. c->biweight_h264_pixels_tab[7]= ff_h264_biweight_4x2_mmx2;
  316. if (mm_flags&AV_CPU_FLAG_SSE2) {
  317. c->h264_idct8_add = ff_h264_idct8_add_sse2;
  318. c->h264_idct8_add4= ff_h264_idct8_add4_sse2;
  319. c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_sse2;
  320. c->weight_h264_pixels_tab[0]= ff_h264_weight_16x16_sse2;
  321. c->weight_h264_pixels_tab[1]= ff_h264_weight_16x8_sse2;
  322. c->weight_h264_pixels_tab[2]= ff_h264_weight_8x16_sse2;
  323. c->weight_h264_pixels_tab[3]= ff_h264_weight_8x8_sse2;
  324. c->weight_h264_pixels_tab[4]= ff_h264_weight_8x4_sse2;
  325. c->biweight_h264_pixels_tab[0]= ff_h264_biweight_16x16_sse2;
  326. c->biweight_h264_pixels_tab[1]= ff_h264_biweight_16x8_sse2;
  327. c->biweight_h264_pixels_tab[2]= ff_h264_biweight_8x16_sse2;
  328. c->biweight_h264_pixels_tab[3]= ff_h264_biweight_8x8_sse2;
  329. c->biweight_h264_pixels_tab[4]= ff_h264_biweight_8x4_sse2;
  330. #if HAVE_ALIGNED_STACK
  331. c->h264_v_loop_filter_luma = ff_x264_deblock_v_luma_sse2;
  332. c->h264_h_loop_filter_luma = ff_x264_deblock_h_luma_sse2;
  333. c->h264_v_loop_filter_luma_intra = ff_x264_deblock_v_luma_intra_sse2;
  334. c->h264_h_loop_filter_luma_intra = ff_x264_deblock_h_luma_intra_sse2;
  335. #endif
  336. c->h264_idct_add16 = ff_h264_idct_add16_sse2;
  337. c->h264_idct_add8 = ff_h264_idct_add8_sse2;
  338. c->h264_idct_add16intra = ff_h264_idct_add16intra_sse2;
  339. }
  340. if (mm_flags&AV_CPU_FLAG_SSSE3) {
  341. c->biweight_h264_pixels_tab[0]= ff_h264_biweight_16x16_ssse3;
  342. c->biweight_h264_pixels_tab[1]= ff_h264_biweight_16x8_ssse3;
  343. c->biweight_h264_pixels_tab[2]= ff_h264_biweight_8x16_ssse3;
  344. c->biweight_h264_pixels_tab[3]= ff_h264_biweight_8x8_ssse3;
  345. c->biweight_h264_pixels_tab[4]= ff_h264_biweight_8x4_ssse3;
  346. }
  347. }
  348. }
  349. #endif
  350. }