qdm2dec: fix buffer overflow.

Fixes NGS00144

This also adds a few lines of code from master that are needed for this fix.

Thanks to Phillip for suggestions to improve the patch.
Found-by: Phillip Langlois
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit a6a61a6d1d)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/qdm2.c

@@ -77,6 +77,7 @@ do { \
 #define SAMPLES_NEEDED_2(why) \
      av_log (NULL,AV_LOG_INFO,"This file triggers some missing code. Please contact the developers.\nPosition: %s\n",why);
 
+#define QDM2_MAX_FRAME_SIZE 512
 
 typedef int8_t sb_int8_array[2][30][64];
 
@@ -169,7 +170,7 @@ typedef struct {
     /// I/O data
     const uint8_t *compressed_data;
     int compressed_size;
-    float output_buffer[1024];
+    float output_buffer[QDM2_MAX_FRAME_SIZE * MPA_MAX_CHANNELS * 2];
 
     /// Synthesis filter
     DECLARE_ALIGNED_16(MPA_INT, synth_buf[MPA_MAX_CHANNELS][512*2]);
@@ -1845,7 +1846,8 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx)
     // something like max decodable tones
     s->group_order = av_log2(s->group_size) + 1;
     s->frame_size = s->group_size / 16; // 16 iterations per super block
-    if (s->frame_size > FF_ARRAY_ELEMS(s->output_buffer) / 2)
+
+    if (s->frame_size > QDM2_MAX_FRAME_SIZE)
         return AVERROR_INVALIDDATA;
 
     s->sub_sampling = s->fft_order - 7;
@@ -1915,6 +1917,9 @@ static void qdm2_decode (QDM2Context *q, const uint8_t *in, int16_t *out)
     int ch, i;
     const int frame_size = (q->frame_size * q->channels);
 
+    if((unsigned)frame_size > FF_ARRAY_ELEMS(q->output_buffer)/2)
+        return -1;
+
     /* select input buffer */
     q->compressed_data = in;
     q->compressed_size = q->checksum_size;