Reject audio tracks with invalid interleaver parameters in RM demuxer.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 4907f81358)
(cherry picked from commit 24e0a9e451)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/rmdec.c

@@ -154,8 +154,9 @@ static int rm_read_audio_stream_info(AVFormatContext *s, ByteIOContext *pb,
             ast->audio_framesize = st->codec->block_align;
             st->codec->block_align = coded_framesize;
 
-            if(ast->audio_framesize >= UINT_MAX / sub_packet_h){
-                av_log(s, AV_LOG_ERROR, "ast->audio_framesize * sub_packet_h too large\n");
+            if (ast->audio_framesize <= 0 || sub_packet_h <= 0 ||
+                ast->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "ast->audio_framesize * sub_packet_h is invalid\n");
                 return -1;
             }
 
@@ -185,8 +186,9 @@ static int rm_read_audio_stream_info(AVFormatContext *s, ByteIOContext *pb,
             ast->audio_framesize = st->codec->block_align;
             st->codec->block_align = ast->sub_packet_size;
 
-            if(ast->audio_framesize >= UINT_MAX / sub_packet_h){
-                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h too large\n");
+            if (ast->audio_framesize <= 0 || sub_packet_h <= 0 ||
+                ast->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h is invalid\n");
                 return -1;
             }