Remove VLA in ff_kbd_window_init, limit window size to 1024

Originally committed as revision 23755 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/fft.h

@@ -150,11 +150,16 @@ static inline void ff_mdct_calc(FFTContext *s, FFTSample *output,
     s->mdct_calc(s, output, input);
 }
 
+/**
+ * Maximum window size for ff_kbd_window_init.
+ */
+#define FF_KBD_WINDOW_MAX 1024
+
 /**
  * Generate a Kaiser-Bessel Derived Window.
  * @param   window  pointer to half window
  * @param   alpha   determines window shape
- * @param   n       size of half window
+ * @param   n       size of half window, max FF_KBD_WINDOW_MAX
  */
 void ff_kbd_window_init(float *window, float alpha, int n);
 

libavcodec/mdct.c

@@ -36,9 +36,11 @@ av_cold void ff_kbd_window_init(float *window, float alpha, int n)
 {
    int i, j;
    double sum = 0.0, bessel, tmp;
-   double local_window[n];
+   double local_window[FF_KBD_WINDOW_MAX];
    double alpha2 = (alpha * M_PI / n) * (alpha * M_PI / n);
 
+   assert(n <= FF_KBD_WINDOW_MAX);
+
    for (i = 0; i < n; i++) {
        tmp = i * (n - i) * alpha2;
        bessel = 1.0;