avformat/mspdec: Check packet_size more completely

Fixes: OOM
Fixes: 28348/clusterfuzz-testcase-minimized-ffmpeg_dem_MSP_fuzzer-4612055872831488
Fixes: 28360/clusterfuzz-testcase-minimized-ffmpeg_dem_MSP_fuzzer-6245230626078720

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/mspdec.c

@@ -70,11 +70,12 @@ static int msp_read_header(AVFormatContext *s)
 
     if (st->codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
         cntx->packet_size = av_image_get_buffer_size(st->codecpar->format, st->codecpar->width, st->codecpar->height, 1);
-        if (cntx->packet_size < 0)
-            return cntx->packet_size;
     } else
         cntx->packet_size = 2 * st->codecpar->height;
 
+    if (cntx->packet_size <= 0)
+        return cntx->packet_size < 0 ? cntx->packet_size : AVERROR_INVALIDDATA;
+
     return 0;
 }