avcodec/mss1: check for overread and forward errors

Fixes: Timeout (106sec -> 14ms)
Fixes: 15576/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MSS1_fuzzer-5688080461201408

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 43015afd7c)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/mss1.c

@@ -56,6 +56,8 @@ static void arith_normalise(ArithCoder *c)
         c->low   <<= 1;
         c->high  <<= 1;
         c->high   |= 1;
+        if (get_bits_left(c->gbc.gb) < 1)
+            c->overread++;
         c->value  |= get_bits1(c->gbc.gb);
     }
 }
@@ -112,6 +114,7 @@ static void arith_init(ArithCoder *c, GetBitContext *gb)
     c->low           = 0;
     c->high          = 0xFFFF;
     c->value         = get_bits(gb, 16);
+    c->overread      = 0;
     c->gbc.gb        = gb;
     c->get_model_sym = arith_get_model_sym;
     c->get_number    = arith_get_number;

libavcodec/mss12.c

@@ -161,6 +161,8 @@ static av_always_inline int decode_pixel(ArithCoder *acoder, PixContext *pctx,
 {
     int i, val, pix;
 
+    if (acoder->overread > MAX_OVERREAD)
+        return AVERROR_INVALIDDATA;
     val = acoder->get_model_sym(acoder, &pctx->cache_model);
     if (val < pctx->num_syms) {
         if (any_ngb) {
@@ -305,6 +307,8 @@ static int decode_region(ArithCoder *acoder, uint8_t *dst, uint8_t *rgb_pic,
             else
                 p = decode_pixel_in_context(acoder, pctx, dst + i, stride,
                                             i, j, width - i - 1);
+            if (p < 0)
+                return p;
             dst[i] = p;
 
             if (rgb_pic)
@@ -397,6 +401,8 @@ static int decode_region_masked(MSS12Context const *c, ArithCoder *acoder,
                 else
                     p = decode_pixel_in_context(acoder, pctx, dst + i, stride,
                                                 i, j, width - i - 1);
+                if (p < 0)
+                    return p;
                 dst[i] = p;
                 if (c->rgb_pic)
                     AV_WB24(rgb_dst + i * 3, c->pal[p]);
@@ -472,6 +478,8 @@ static int decode_region_intra(SliceContext *sc, ArithCoder *acoder,
         uint8_t *rgb_dst = c->rgb_pic + x * 3 + y * rgb_stride;
 
         pix     = decode_pixel(acoder, &sc->intra_pix_ctx, NULL, 0, 0);
+        if (pix < 0)
+            return pix;
         rgb_pix = c->pal[pix];
         for (i = 0; i < height; i++, dst += stride, rgb_dst += rgb_stride) {
             memset(dst, pix, width);
@@ -498,6 +506,8 @@ static int decode_region_inter(SliceContext *sc, ArithCoder *acoder,
 
     if (!mode) {
         mode = decode_pixel(acoder, &sc->inter_pix_ctx, NULL, 0, 0);
+        if (mode < 0)
+            return mode;
 
         if (c->avctx->err_recognition & AV_EF_EXPLODE &&
             ( c->rgb_pic && mode != 0x01 && mode != 0x02 && mode != 0x04 ||
@@ -529,6 +539,8 @@ int ff_mss12_decode_rect(SliceContext *sc, ArithCoder *acoder,
                          int x, int y, int width, int height)
 {
     int mode, pivot;
+    if (acoder->overread > MAX_OVERREAD)
+        return AVERROR_INVALIDDATA;
 
     mode = acoder->get_model_sym(acoder, &sc->split_mode);
 

libavcodec/mss12.h

@@ -47,6 +47,8 @@ typedef struct Model {
 
 typedef struct ArithCoder {
     int low, high, value;
+    int overread;
+#define MAX_OVERREAD 16
     union {
         GetBitContext *gb;
         GetByteContext *gB;

libavcodec/mss2.c

@@ -152,6 +152,7 @@ static void arith2_init(ArithCoder *c, GetByteContext *gB)
     c->low           = 0;
     c->high          = 0xFFFFFF;
     c->value         = bytestream2_get_be24(gB);
+    c->overread      = 0;
     c->gbc.gB        = gB;
     c->get_model_sym = arith2_get_model_sym;
     c->get_number    = arith2_get_number;