avformat/rsd: Check size and start before computing duration

Fixes: signed integer overflow: 100794754 * 28 cannot be represented in type 'int'
Fixes: 26474/clusterfuzz-testcase-minimized-ffmpeg_dem_RSD_fuzzer-5181797606096896

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/rsd.c

@@ -103,13 +103,9 @@ static int rsd_read_header(AVFormatContext *s)
         break;
     case AV_CODEC_ID_ADPCM_PSX:
         par->block_align = 16 * par->channels;
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = av_get_audio_frame_duration2(par, avio_size(pb) - start);
         break;
     case AV_CODEC_ID_ADPCM_IMA_RAD:
         par->block_align = 20 * par->channels;
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = av_get_audio_frame_duration2(par, avio_size(pb) - start);
         break;
     case AV_CODEC_ID_ADPCM_IMA_WAV:
         if (version == 2)
@@ -117,8 +113,6 @@ static int rsd_read_header(AVFormatContext *s)
 
         par->bits_per_coded_sample = 4;
         par->block_align = 36 * par->channels;
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = av_get_audio_frame_duration2(par, avio_size(pb) - start);
         break;
     case AV_CODEC_ID_ADPCM_THP_LE:
         /* RSD3GADP is mono, so only alloc enough memory
@@ -128,8 +122,6 @@ static int rsd_read_header(AVFormatContext *s)
 
         if ((ret = ff_get_extradata(s, par, s->pb, 32)) < 0)
             return ret;
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = av_get_audio_frame_duration2(par, avio_size(pb) - start);
         break;
     case AV_CODEC_ID_ADPCM_THP:
         par->block_align = 8 * par->channels;
@@ -142,18 +134,36 @@ static int rsd_read_header(AVFormatContext *s)
             avio_read(s->pb, st->codecpar->extradata + 32 * i, 32);
             avio_skip(s->pb, 8);
         }
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = (avio_size(pb) - start) / (8 * par->channels) * 14;
         break;
     case AV_CODEC_ID_PCM_S16LE:
     case AV_CODEC_ID_PCM_S16BE:
         if (version != 4)
             start = avio_rl32(pb);
 
-        if (pb->seekable & AVIO_SEEKABLE_NORMAL)
-            st->duration = (avio_size(pb) - start) / 2 / par->channels;
         break;
     }
+    if (start < 0)
+        return AVERROR_INVALIDDATA;
+
+    if (pb->seekable & AVIO_SEEKABLE_NORMAL) {
+        int64_t remaining = avio_size(pb);
+
+        if (remaining >= start && remaining - start <= INT_MAX)
+            switch (par->codec_id) {
+            case AV_CODEC_ID_ADPCM_PSX:
+            case AV_CODEC_ID_ADPCM_IMA_RAD:
+            case AV_CODEC_ID_ADPCM_IMA_WAV:
+            case AV_CODEC_ID_ADPCM_THP_LE:
+                st->duration = av_get_audio_frame_duration2(par, remaining - start);
+                break;
+            case AV_CODEC_ID_ADPCM_THP:
+                st->duration = (remaining - start) / (8 * par->channels) * 14;
+                break;
+            case AV_CODEC_ID_PCM_S16LE:
+            case AV_CODEC_ID_PCM_S16BE:
+                st->duration = (remaining - start) / 2 / par->channels;
+            }
+    }
 
     avio_skip(pb, start - avio_tell(pb));
     if (par->codec_id == AV_CODEC_ID_XMA2) {