prores: Add a codepath for decoding errors

libavcodec/proresdec.c

@@ -369,10 +369,10 @@ static inline void decode_dc_coeffs(GetBitContext *gb, int16_t *out,
 /**
  * Decode AC coefficients for all blocks in a slice.
  */
-static inline void decode_ac_coeffs(GetBitContext *gb, int16_t *out,
-                                    int blocks_per_slice,
-                                    int plane_size_factor,
-                                    const uint8_t *scan)
+static inline int decode_ac_coeffs(GetBitContext *gb, int16_t *out,
+                                   int blocks_per_slice,
+                                   int plane_size_factor,
+                                   const uint8_t *scan)
 {
     int pos, block_mask, run, level, sign, run_cb_index, lev_cb_index;
     int max_coeffs, bits_left;
@@ -390,13 +390,13 @@ static inline void decode_ac_coeffs(GetBitContext *gb, int16_t *out,
 
         bits_left = get_bits_left(gb);
         if (bits_left <= 0 || (bits_left <= 8 && !show_bits(gb, bits_left)))
-            return;
+            return AVERROR_INVALIDDATA;
 
         run = decode_vlc_codeword(gb, ff_prores_ac_codebook[run_cb_index]);
 
         bits_left = get_bits_left(gb);
         if (bits_left <= 0 || (bits_left <= 8 && !show_bits(gb, bits_left)))
-            return;
+            return AVERROR_INVALIDDATA;
 
         level = decode_vlc_codeword(gb, ff_prores_ac_codebook[lev_cb_index]) + 1;
 
@@ -408,22 +408,24 @@ static inline void decode_ac_coeffs(GetBitContext *gb, int16_t *out,
         out[((pos & block_mask) << 6) + scan[pos >> plane_size_factor]] =
             (level ^ sign) - sign;
     }
+
+    return 0;
 }
 
 
 /**
  * Decode a slice plane (luma or chroma).
  */
-static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
-                               const uint8_t *buf,
-                               int data_size, uint16_t *out_ptr,
-                               int linesize, int mbs_per_slice,
-                               int blocks_per_mb, int plane_size_factor,
-                               const int16_t *qmat, int is_chroma)
+static int decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
+                              const uint8_t *buf,
+                              int data_size, uint16_t *out_ptr,
+                              int linesize, int mbs_per_slice,
+                              int blocks_per_mb, int plane_size_factor,
+                              const int16_t *qmat, int is_chroma)
 {
     GetBitContext gb;
     int16_t *block_ptr;
-    int mb_num, blocks_per_slice;
+    int mb_num, blocks_per_slice, ret;
 
     blocks_per_slice = mbs_per_slice * blocks_per_mb;
 
@@ -433,8 +435,10 @@ static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
 
     decode_dc_coeffs(&gb, td->blocks, blocks_per_slice);
 
-    decode_ac_coeffs(&gb, td->blocks, blocks_per_slice,
-                     plane_size_factor, ctx->scantable.permutated);
+    ret = decode_ac_coeffs(&gb, td->blocks, blocks_per_slice,
+                           plane_size_factor, ctx->scantable.permutated);
+    if (ret < 0)
+        return ret;
 
     /* inverse quantization, inverse transform and output */
     block_ptr = td->blocks;
@@ -468,6 +472,7 @@ static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
             }
         }
     }
+    return 0;
 }
 
 
@@ -560,6 +565,7 @@ static int decode_slice(AVCodecContext *avctx, void *tdata)
     int y_data_size, u_data_size, v_data_size, a_data_size;
     int y_linesize, u_linesize, v_linesize, a_linesize;
     int coff[4];
+    int ret;
 
     buf             = ctx->slice_data[slice_num].index;
     slice_data_size = ctx->slice_data[slice_num + 1].index - buf;
@@ -631,24 +637,31 @@ static int decode_slice(AVCodecContext *avctx, void *tdata)
     }
 
     /* decode luma plane */
-    decode_slice_plane(ctx, td, buf + coff[0], y_data_size,
-                       (uint16_t*) y_data, y_linesize,
-                       mbs_per_slice, 4, slice_width_factor + 2,
-                       td->qmat_luma_scaled, 0);
+    ret = decode_slice_plane(ctx, td, buf + coff[0], y_data_size,
+                             (uint16_t*) y_data, y_linesize,
+                             mbs_per_slice, 4, slice_width_factor + 2,
+                             td->qmat_luma_scaled, 0);
+
+    if (ret < 0)
+        return ret;
 
     /* decode U chroma plane */
-    decode_slice_plane(ctx, td, buf + coff[1], u_data_size,
-                       (uint16_t*) u_data, u_linesize,
-                       mbs_per_slice, ctx->num_chroma_blocks,
-                       slice_width_factor + ctx->chroma_factor - 1,
-                       td->qmat_chroma_scaled, 1);
+    ret = decode_slice_plane(ctx, td, buf + coff[1], u_data_size,
+                             (uint16_t*) u_data, u_linesize,
+                             mbs_per_slice, ctx->num_chroma_blocks,
+                             slice_width_factor + ctx->chroma_factor - 1,
+                             td->qmat_chroma_scaled, 1);
+    if (ret < 0)
+        return ret;
 
     /* decode V chroma plane */
-    decode_slice_plane(ctx, td, buf + coff[2], v_data_size,
-                       (uint16_t*) v_data, v_linesize,
-                       mbs_per_slice, ctx->num_chroma_blocks,
-                       slice_width_factor + ctx->chroma_factor - 1,
-                       td->qmat_chroma_scaled, 1);
+    ret = decode_slice_plane(ctx, td, buf + coff[2], v_data_size,
+                             (uint16_t*) v_data, v_linesize,
+                             mbs_per_slice, ctx->num_chroma_blocks,
+                             slice_width_factor + ctx->chroma_factor - 1,
+                             td->qmat_chroma_scaled, 1);
+    if (ret < 0)
+        return ret;
 
     /* decode alpha plane if available */
     if (a_data && a_data_size)