rsd: limit number of channels

Negative values don't make sense and too large values can cause
overflows. For AV_CODEC_ID_ADPCM_THP this leads to a too small extradata
buffer being allocated, causing out-of-bounds writes.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
(cherry picked from commit ee5f0f1d35)
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>

libavformat/rsd.c

@@ -84,8 +84,10 @@ static int rsd_read_header(AVFormatContext *s)
     }
 
     par->channels = avio_rl32(pb);
-    if (!par->channels)
+    if (par->channels <= 0 || par->channels > INT_MAX / 36) {
+        av_log(s, AV_LOG_ERROR, "Invalid number of channels: %d\n", par->channels);
         return AVERROR_INVALIDDATA;
+    }
 
     avio_skip(pb, 4); // Bit depth
     par->sample_rate = avio_rl32(pb);