mjpegdec: check for pixel format changes

Fixes possible invalid memory access.

Based on code by Michael Niedermayer <michaelni@gmx.at>

CC: libav-stable@libav.org
Bug-ID: CVE-2014-8541
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
(cherry picked from commit 809c3023b6)
Signed-off-by: Anton Khirnov <anton@khirnov.net>

libavcodec/mjpegdec.c

@@ -220,18 +220,20 @@ int ff_mjpeg_decode_dht(MJpegDecodeContext *s)
 
 int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
 {
-    int len, nb_components, i, width, height, pix_fmt_id, ret;
+    int h_count[MAX_COMPONENTS] = { 0 };
+    int v_count[MAX_COMPONENTS] = { 0 };
+    int len, nb_components, i, width, height, bits, pix_fmt_id, ret;
 
     /* XXX: verify len field validity */
     len     = get_bits(&s->gb, 16);
-    s->bits = get_bits(&s->gb, 8);
+    bits    = get_bits(&s->gb, 8);
 
     if (s->pegasus_rct)
-        s->bits = 9;
-    if (s->bits == 9 && !s->pegasus_rct)
+        bits = 9;
+    if (bits == 9 && !s->pegasus_rct)
         s->rct  = 1;    // FIXME ugly
 
-    if (s->bits != 8 && !s->lossless) {
+    if (bits != 8 && !s->lossless) {
         av_log(s->avctx, AV_LOG_ERROR, "only 8 bits/component accepted\n");
         return -1;
     }
@@ -258,7 +260,7 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
             return AVERROR_INVALIDDATA;
         }
     }
-    if (s->ls && !(s->bits <= 8 || nb_components == 1)) {
+    if (s->ls && !(bits <= 8 || nb_components == 1)) {
         avpriv_report_missing_feature(s->avctx,
                                       "JPEG-LS that is not <= 8 "
                                       "bits/component or 16-bit gray");
@@ -270,25 +272,25 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
     for (i = 0; i < nb_components; i++) {
         /* component id */
         s->component_id[i] = get_bits(&s->gb, 8) - 1;
-        s->h_count[i]      = get_bits(&s->gb, 4);
-        s->v_count[i]      = get_bits(&s->gb, 4);
+        h_count[i]         = get_bits(&s->gb, 4);
+        v_count[i]         = get_bits(&s->gb, 4);
         /* compute hmax and vmax (only used in interleaved case) */
-        if (s->h_count[i] > s->h_max)
-            s->h_max = s->h_count[i];
-        if (s->v_count[i] > s->v_max)
-            s->v_max = s->v_count[i];
+        if (h_count[i] > s->h_max)
+            s->h_max = h_count[i];
+        if (v_count[i] > s->v_max)
+            s->v_max = v_count[i];
         s->quant_index[i] = get_bits(&s->gb, 8);
         if (s->quant_index[i] >= 4)
             return AVERROR_INVALIDDATA;
-        if (!s->h_count[i] || !s->v_count[i]) {
+        if (!h_count[i] || !v_count[i]) {
             av_log(s->avctx, AV_LOG_ERROR,
                    "Invalid sampling factor in component %d %d:%d\n",
-                   i, s->h_count[i], s->v_count[i]);
+                   i, h_count[i], v_count[i]);
             return AVERROR_INVALIDDATA;
         }
 
         av_log(s->avctx, AV_LOG_DEBUG, "component %d %d:%d id: %d quant:%d\n",
-               i, s->h_count[i], s->v_count[i],
+               i, h_count[i], v_count[i],
                s->component_id[i], s->quant_index[i]);
     }
 
@@ -301,10 +303,14 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
         s->rgb = 1;
 
     /* if different size, realloc/alloc picture */
-    /* XXX: also check h_count and v_count */
-    if (width != s->width || height != s->height) {
+    if (width != s->width || height != s->height || bits != s->bits ||
+        memcmp(s->h_count, h_count, sizeof(h_count))                ||
+        memcmp(s->v_count, v_count, sizeof(v_count))) {
         s->width      = width;
         s->height     = height;
+        s->bits       = bits;
+        memcpy(s->h_count, h_count, sizeof(h_count));
+        memcpy(s->v_count, v_count, sizeof(v_count));
         s->interlaced = 0;
 
         /* test interlaced mode */