mdct15: add inverse transform postrotation SIMD

2.5ms frames:
Before   (c):  2638 decicycles in postrotate, 2097040 runs,    112 skips
After (sse3):  1467 decicycles in postrotate, 2097083 runs,     69 skips
After (avx2):  1244 decicycles in postrotate, 2097085 runs,     67 skips

5ms frames:
Before   (c):  4987 decicycles in postrotate, 1048371 runs,    205 skips
After (sse3):  2644 decicycles in postrotate, 1048509 runs,     67 skips
After (avx2):  2031 decicycles in postrotate, 1048523 runs,     53 skips

10ms frames:
Before   (c):  9153 decicycles in postrotate,  523575 runs,    713 skips
After (sse3):  5110 decicycles in postrotate,  523726 runs,    562 skips
After (avx2):  3738 decicycles in postrotate,  524223 runs,     65 skips

20ms frames:
Before   (c): 17857 decicycles in postrotate,  261866 runs,    278 skips
After (sse3): 10041 decicycles in postrotate,  261746 runs,    398 skips
After (avx2):  7050 decicycles in postrotate,  262116 runs,     28 skips

Improves total decoding performance for real world content by 9% with avx2.

Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>

libavcodec/mdct15.c

@@ -65,11 +65,11 @@ static inline int init_pfa_reindex_tabs(MDCT15Context *s)
     const int inv_1 = l_ptwo << ((4 - b_ptwo) & 3); /* (2^b_ptwo)^-1 mod 15 */
     const int inv_2 = 0xeeeeeeef & ((1U << b_ptwo) - 1); /* 15^-1 mod 2^b_ptwo */
 
-    s->pfa_prereindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_prereindex));
+    s->pfa_prereindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_prereindex));
     if (!s->pfa_prereindex)
         return 1;
 
-    s->pfa_postreindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_postreindex));
+    s->pfa_postreindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_postreindex));
     if (!s->pfa_postreindex)
         return 1;
 
@@ -225,15 +225,22 @@ static void imdct15_half(MDCT15Context *s, float *dst, const float *src,
     for (i = 0; i < 15; i++)
         s->ptwo_fft.fft_calc(&s->ptwo_fft, s->tmp + l_ptwo*i);
 
+    /* Reindex again, apply twiddles and output */
+    s->postreindex(z, s->tmp, s->twiddle_exptab, s->pfa_postreindex, len8);
+}
+
+static void postrotate_c(FFTComplex *out, FFTComplex *in, FFTComplex *exp,
+                         int *lut, ptrdiff_t len8)
+{
+    int i;
+
     /* Reindex again, apply twiddles and output */
     for (i = 0; i < len8; i++) {
         const int i0 = len8 + i, i1 = len8 - i - 1;
-        const int s0 = s->pfa_postreindex[i0], s1 = s->pfa_postreindex[i1];
+        const int s0 = lut[i0], s1 = lut[i1];
 
-        CMUL(z[i1].re, z[i0].im, s->tmp[s1].im, s->tmp[s1].re,
-             s->twiddle_exptab[i1].im, s->twiddle_exptab[i1].re);
-        CMUL(z[i0].re, z[i1].im, s->tmp[s0].im, s->tmp[s0].re,
-             s->twiddle_exptab[i0].im, s->twiddle_exptab[i0].re);
+        CMUL(out[i1].re, out[i0].im, in[s1].im, in[s1].re, exp[i1].im, exp[i1].re);
+        CMUL(out[i0].re, out[i1].im, in[s0].im, in[s0].re, exp[i0].im, exp[i0].re);
     }
 }
 
@@ -253,13 +260,14 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale)
     if (!s)
         return AVERROR(ENOMEM);
 
-    s->fft_n      = N - 1;
-    s->len4       = len2 / 2;
-    s->len2       = len2;
-    s->inverse    = inverse;
-    s->fft15      = fft15_c;
-    s->mdct       = mdct15;
-    s->imdct_half = imdct15_half;
+    s->fft_n       = N - 1;
+    s->len4        = len2 / 2;
+    s->len2        = len2;
+    s->inverse     = inverse;
+    s->fft15       = fft15_c;
+    s->mdct        = mdct15;
+    s->imdct_half  = imdct15_half;
+    s->postreindex = postrotate_c;
 
     if (ff_fft_init(&s->ptwo_fft, N - 1, s->inverse) < 0)
         goto fail;
@@ -271,7 +279,7 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale)
     if (!s->tmp)
         goto fail;
 
-    s->twiddle_exptab  = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab));
+    s->twiddle_exptab = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab));
     if (!s->twiddle_exptab)
         goto fail;
 

libavcodec/mdct15.h

@@ -42,6 +42,9 @@ typedef struct MDCT15Context {
     /* 15-point FFT */
     void (*fft15)(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
 
+    /* PFA postrotate and exptab */
+    void (*postreindex)(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
+
     /* Calculate a full 2N -> N MDCT */
     void (*mdct)(struct MDCT15Context *s, float *dst, const float *src, ptrdiff_t stride);
 

libavcodec/x86/mdct15.asm

@@ -22,14 +22,21 @@
 
 %include "libavutil/x86/x86util.asm"
 
-%if ARCH_X86_64
+SECTION_RODATA 32
 
-SECTION_RODATA
+perm_neg: dd 2, 5, 3, 4, 6, 1, 7, 0
+perm_pos: dd 0, 7, 1, 6, 4, 3, 5, 2
+sign_adjust_r: times 4 dd 0x80000000, 0x00000000
 
 sign_adjust_5: dd 0x00000000, 0x80000000, 0x80000000, 0x00000000
 
 SECTION .text
 
+%if ARCH_X86_64
+
+;*****************************************************************************************
+;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
+;*****************************************************************************************
 %macro FFT5 3 ; %1 - in_offset, %2 - dst1 (64bit used), %3 - dst2
     VBROADCASTSD m0, [inq + %1]         ; in[ 0].re, in[ 0].im, in[ 0].re, in[ 0].im
     movsd   xm1, [inq + 1*16 +  8 + %1] ; in[ 3].re, in[ 3].im,         0,         0
@@ -103,9 +110,6 @@ SECTION .text
     movhps [%2q + strideq*4], xm1
 %endmacro
 
-;*****************************************************************************************
-;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
-;*****************************************************************************************
 INIT_YMM avx
 cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5
 %define out0q inq
@@ -138,4 +142,83 @@ cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5
 
     RET
 
+%endif ; ARCH_X86_64
+
+;*******************************************************************************************************
+;void ff_mdct15_postreindex(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
+;*******************************************************************************************************
+%macro LUT_LOAD_4D 3
+    mov      r4d, [lutq + %3q*4 +  0]
+    movsd  xmm%1, [inq +  r4q*8]
+    mov      r4d, [lutq + %3q*4 +  4]
+    movhps xmm%1, [inq +  r4q*8]
+%if cpuflag(avx2)
+    mov      r4d, [lutq + %3q*4 +  8]
+    movsd     %2, [inq +  r4q*8]
+    mov      r4d, [lutq + %3q*4 + 12]
+    movhps    %2, [inq +  r4q*8]
+    vinsertf128 %1, %1, %2, 1
+%endif
+%endmacro
+
+%macro POSTROTATE_FN 1
+cglobal mdct15_postreindex, 5, 7, 8 + cpuflag(avx2)*2, out, in, exp, lut, len8, offset_p, offset_n
+
+    xor offset_nq, offset_nq
+    lea offset_pq, [len8q*2 - %1]
+
+    movaps m7,  [sign_adjust_r]
+
+%if cpuflag(avx2)
+    movaps   m8, [perm_pos]
+    movaps   m9, [perm_neg]
+%endif
+
+.loop:
+    movups m0, [expq + offset_pq*8]     ; exp[p0].re, exp[p0].im, exp[p1].re, exp[p1].im, exp[p2].re, exp[p2].im, exp[p3].re, exp[p3].im
+    movups m1, [expq + offset_nq*8]     ; exp[n3].re, exp[n3].im, exp[n2].re, exp[n2].im, exp[n1].re, exp[n1].im, exp[n0].re, exp[n0].im
+
+    LUT_LOAD_4D m3, xm4, offset_p       ; in[p0].re, in[p0].im, in[p1].re, in[p1].im, in[p2].re, in[p2].im, in[p3].re, in[p3].im
+    LUT_LOAD_4D m4, xm5, offset_n       ; in[n3].re, in[n3].im, in[n2].re, in[n2].im, in[n1].re, in[n1].im, in[n0].re, in[n0].im
+
+    mulps  m5, m3, m0                   ; in[p].reim * exp[p].reim
+    mulps  m6, m4, m1                   ; in[n].reim * exp[n].reim
+
+    xorps  m5, m7                       ; in[p].re *= -1, in[p].im *= 1
+    xorps  m6, m7                       ; in[n].re *= -1, in[n].im *= 1
+
+    shufps m3, m3, m3, q2301            ; in[p].imre
+    shufps m4, m4, m4, q2301            ; in[n].imre
+
+    mulps  m3, m0                       ; in[p].imre * exp[p].reim
+    mulps  m4, m1                       ; in[n].imre * exp[n].reim
+
+    haddps m3, m6                       ; out[n0].im, out[n1].im, out[n3].re, out[n2].re, out[n2].im, out[n3].im, out[n1].re, out[n0].re
+    haddps m5, m4                       ; out[p0].re, out[p1].re, out[p3].im, out[p2].im, out[p2].re, out[p3].re, out[p1].im, out[p0].im
+
+%if cpuflag(avx2)
+    vpermps m3, m9, m3                  ; out[n3].im, out[n3].re, out[n2].im, out[n2].re, out[n1].im, out[n1].re, out[n0].im, out[n0].re
+    vpermps m5, m8, m5                  ; out[p0].re, out[p0].im, out[p1].re, out[p1].im, out[p2].re, out[p2].im, out[p3].re, out[p3].im
+%else
+    shufps m3, m3, m3, q0312
+    shufps m5, m5, m5, q2130
+%endif
+
+    movups [outq + offset_nq*8], m3
+    movups [outq + offset_pq*8], m5
+
+    sub offset_pq, %1
+    add offset_nq, %1
+    cmp offset_nq, offset_pq
+    jle .loop
+
+    REP_RET
+%endmacro
+
+INIT_XMM sse3
+POSTROTATE_FN 2
+
+%if ARCH_X86_64 && HAVE_AVX2_EXTERNAL
+INIT_YMM avx2
+POSTROTATE_FN 4
 %endif

libavcodec/x86/mdct15_init.c

@@ -25,6 +25,9 @@
 #include "libavutil/x86/cpu.h"
 #include "libavcodec/mdct15.h"
 
+void ff_mdct15_postreindex_sse3(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
+void ff_mdct15_postreindex_avx2(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
+
 void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
 
 static void perm_twiddles(MDCT15Context *s)
@@ -85,11 +88,17 @@ av_cold void ff_mdct15_init_x86(MDCT15Context *s)
     int adjust_twiddles = 0;
     int cpu_flags = av_get_cpu_flags();
 
+    if (EXTERNAL_SSE3(cpu_flags))
+        s->postreindex = ff_mdct15_postreindex_sse3;
+
     if (ARCH_X86_64 && EXTERNAL_AVX(cpu_flags)) {
         s->fft15 = ff_fft15_avx;
         adjust_twiddles = 1;
     }
 
+    if (ARCH_X86_64 && EXTERNAL_AVX2_FAST(cpu_flags))
+        s->postreindex = ff_mdct15_postreindex_avx2;
+
     if (adjust_twiddles)
         perm_twiddles(s);
 }