avcodec/zmbv: Check decomp_size

Fixes: OOM
Fixes: 2710/clusterfuzz-testcase-minimized-4750001420894208

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Tomas Härdin <tjoppen@acc.umu.se>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 931c0ac95c)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/zmbv.c

@@ -589,6 +589,11 @@ static av_cold int decode_init(AVCodecContext *avctx)
     // Needed if zlib unused or init aborted before inflateInit
     memset(&c->zstream, 0, sizeof(z_stream));
 
+    if ((avctx->width + 255ULL) * (avctx->height + 64ULL) > FFMIN(avctx->max_pixels, INT_MAX / 4) ) {
+        av_log(avctx, AV_LOG_ERROR, "Internal buffer (decomp_size) larger than max_pixels or too large\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     c->decomp_size = (avctx->width + 255) * 4 * (avctx->height + 64);
 
     /* Allocate decompression buffer */