hevc: Add NEON add_residual for bitdepth 8

Optimized by Alexandra Hájková.

Signed-off-by: Martin Storsjö <martin@martin.st>

libavcodec/arm/hevc_idct.S

@@ -30,6 +30,94 @@ const trans, align=4
         .short 57, 43, 25, 9
 endconst
 
+function ff_hevc_add_residual_4x4_8_neon, export=1
+        vld1.16         {q0-q1}, [r1, :128]
+        vld1.32         d4[0], [r0, :32], r2
+        vld1.32         d4[1], [r0, :32], r2
+        vld1.32         d5[0], [r0, :32], r2
+        vld1.32         d5[1], [r0, :32], r2
+        sub             r0, r0, r2, lsl #2
+        vmovl.u8        q8, d4
+        vmovl.u8        q9, d5
+        vqadd.s16       q0, q0, q8
+        vqadd.s16       q1, q1, q9
+        vqmovun.s16     d0, q0
+        vqmovun.s16     d1, q1
+        vst1.32         d0[0], [r0, :32], r2
+        vst1.32         d0[1], [r0, :32], r2
+        vst1.32         d1[0], [r0, :32], r2
+        vst1.32         d1[1], [r0, :32], r2
+        bx              lr
+endfunc
+
+function ff_hevc_add_residual_8x8_8_neon, export=1
+        mov             r3,   #8
+1:      subs            r3,   #2
+        vld1.16         {q0-q1}, [r1, :128]!
+        vld1.8          {d16},   [r0, :64]
+        add             r12, r0, r2
+        vld1.8          {d17},   [r12, :64]
+        vmovl.u8        q9,   d16
+        vmovl.u8        q8,   d17
+        vqadd.s16       q0,   q9
+        vqadd.s16       q1,   q8
+        vqmovun.s16     d0,   q0
+        vqmovun.s16     d1,   q1
+        vst1.8          d0,   [r0, :64], r2
+        vst1.8          d1,   [r0, :64], r2
+        bne             1b
+        bx              lr
+endfunc
+
+function ff_hevc_add_residual_16x16_8_neon, export=1
+        mov             r3,  #16
+        add             r12, r0, r2
+        add             r2,  r2, r2
+1:      subs            r3,  #2
+        vld1.8          {q8},     [r0, :128]
+        vld1.16         {q0, q1}, [r1, :128]!
+        vld1.8          {q11},    [r12, :128]
+        vld1.16         {q2, q3}, [r1, :128]!
+        vmovl.u8        q9,  d16
+        vmovl.u8        q10, d17
+        vmovl.u8        q12, d22
+        vmovl.u8        q13, d23
+        vqadd.s16       q0,  q9
+        vqadd.s16       q1,  q10
+        vqadd.s16       q2,  q12
+        vqadd.s16       q3,  q13
+        vqmovun.s16     d0,  q0
+        vqmovun.s16     d1,  q1
+        vqmovun.s16     d2,  q2
+        vqmovun.s16     d3,  q3
+        vst1.8          {q0},     [r0, :128], r2
+        vst1.8          {q1},     [r12, :128], r2
+        bne             1b
+        bx              lr
+endfunc
+
+function ff_hevc_add_residual_32x32_8_neon, export=1
+        mov             r3,  #32
+1:      subs            r3,  #1
+        vldm            r1!, {q0-q3}
+        vld1.8          {q8, q9}, [r0, :128]
+        vmovl.u8        q10, d16
+        vmovl.u8        q11, d17
+        vmovl.u8        q12, d18
+        vmovl.u8        q13, d19
+        vqadd.s16       q0,  q10
+        vqadd.s16       q1,  q11
+        vqadd.s16       q2,  q12
+        vqadd.s16       q3,  q13
+        vqmovun.s16     d0,  q0
+        vqmovun.s16     d1,  q1
+        vqmovun.s16     d2,  q2
+        vqmovun.s16     d3,  q3
+        vst1.8          {q0, q1}, [r0, :128], r2
+        bne             1b
+        bx              lr
+endfunc
+
 .macro idct_4x4_dc bitdepth
 function ff_hevc_idct_4x4_dc_\bitdepth\()_neon, export=1
         ldrsh           r1, [r0]

libavcodec/arm/hevcdsp_init_arm.c

@@ -25,6 +25,16 @@
 
 #include "libavcodec/hevcdsp.h"
 
+
+void ff_hevc_add_residual_4x4_8_neon(uint8_t *_dst, int16_t *coeffs,
+                                     ptrdiff_t stride);
+void ff_hevc_add_residual_8x8_8_neon(uint8_t *_dst, int16_t *coeffs,
+                                     ptrdiff_t stride);
+void ff_hevc_add_residual_16x16_8_neon(uint8_t *_dst, int16_t *coeffs,
+                                       ptrdiff_t stride);
+void ff_hevc_add_residual_32x32_8_neon(uint8_t *_dst, int16_t *coeffs,
+                                       ptrdiff_t stride);
+
 void ff_hevc_idct_4x4_dc_8_neon(int16_t *coeffs);
 void ff_hevc_idct_8x8_dc_8_neon(int16_t *coeffs);
 void ff_hevc_idct_16x16_dc_8_neon(int16_t *coeffs);
@@ -47,6 +57,11 @@ av_cold void ff_hevc_dsp_init_arm(HEVCDSPContext *c, int bit_depth)
 
     if (have_neon(cpu_flags)) {
         if (bit_depth == 8) {
+            c->add_residual[0] = ff_hevc_add_residual_4x4_8_neon;
+            c->add_residual[1] = ff_hevc_add_residual_8x8_8_neon;
+            c->add_residual[2] = ff_hevc_add_residual_16x16_8_neon;
+            c->add_residual[3] = ff_hevc_add_residual_32x32_8_neon;
+
             c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_neon;
             c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_neon;
             c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon;