avcodec/brenderpix: Check input size before allocating image

An incomplete image is not supported prior to this and will
not produce any output. This commit moves the failure before
time consuming operations.

Fixes: Timeout (81sec -> 76ms)
Fixes: 15723/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BRENDER_PIX_fuzzer-5147265653538816

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 38b6c48c43)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/brenderpix.c

@@ -204,6 +204,10 @@ static int pix_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         avpriv_request_sample(avctx, "Format %d", hdr.format);
         return AVERROR_PATCHWELCOME;
     }
+    bytes_per_scanline = bytes_pp * hdr.width;
+
+    if (bytestream2_get_bytes_left(&gb) < hdr.height * bytes_per_scanline)
+        return AVERROR_INVALIDDATA;
 
     if ((ret = ff_set_dimensions(avctx, hdr.width, hdr.height)) < 0)
         return ret;
@@ -261,7 +265,6 @@ static int pix_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     bytestream2_skip(&gb, 8);
 
     // read the image data to the buffer
-    bytes_per_scanline = bytes_pp * hdr.width;
     bytes_left = bytestream2_get_bytes_left(&gb);
 
     if (chunk_type != IMAGE_DATA_CHUNK || data_len != bytes_left ||