Merge remote-tracking branch 'qatar/master'

* qatar/master:
  indeo: Bound-check before applying transform

Conflicts:
	libavcodec/indeo4.c
	libavcodec/indeo5.c
	libavcodec/ivi_common.c

See: af388237093ed6df6f5118b34ef938a2ca2ffbda, 0846719dd1
Merged-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/indeo4.c

@@ -352,7 +352,14 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band,
             band->inv_transform = transforms[transform_id].inv_trans;
             band->dc_transform  = transforms[transform_id].dc_trans;
             band->is_2d_trans   = transforms[transform_id].is_2d_trans;
-            band->transform_size= (transform_id < 10) ? 8 : 4;
+
+            if (transform_id < 10)
+                band->transform_size = 8;
+            else
+                band->transform_size = 4;
+
+            if (band->blk_size != band->transform_size)
+                return AVERROR_INVALIDDATA;
 
             scan_indx = get_bits(&ctx->gb, 4);
             if (scan_indx == 15) {

libavcodec/indeo5.c

@@ -153,44 +153,47 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx)
             /* select transform function and scan pattern according to plane and band number */
             switch ((p << 2) + i) {
             case 0:
-                band->inv_transform = ff_ivi_inverse_slant_8x8;
-                band->dc_transform  = ff_ivi_dc_slant_2d;
-                band->scan          = ff_zigzag_direct;
-                band->transform_size= 8;
+                band->inv_transform  = ff_ivi_inverse_slant_8x8;
+                band->dc_transform   = ff_ivi_dc_slant_2d;
+                band->scan           = ff_zigzag_direct;
+                band->transform_size = 8;
                 break;
 
             case 1:
-                band->inv_transform = ff_ivi_row_slant8;
-                band->dc_transform  = ff_ivi_dc_row_slant;
-                band->scan          = ff_ivi_vertical_scan_8x8;
-                band->transform_size= 8;
+                band->inv_transform  = ff_ivi_row_slant8;
+                band->dc_transform   = ff_ivi_dc_row_slant;
+                band->scan           = ff_ivi_vertical_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 2:
-                band->inv_transform = ff_ivi_col_slant8;
-                band->dc_transform  = ff_ivi_dc_col_slant;
-                band->scan          = ff_ivi_horizontal_scan_8x8;
-                band->transform_size= 8;
+                band->inv_transform  = ff_ivi_col_slant8;
+                band->dc_transform   = ff_ivi_dc_col_slant;
+                band->scan           = ff_ivi_horizontal_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 3:
-                band->inv_transform = ff_ivi_put_pixels_8x8;
-                band->dc_transform  = ff_ivi_put_dc_pixel_8x8;
-                band->scan          = ff_ivi_horizontal_scan_8x8;
-                band->transform_size= 8;
+                band->inv_transform  = ff_ivi_put_pixels_8x8;
+                band->dc_transform   = ff_ivi_put_dc_pixel_8x8;
+                band->scan           = ff_ivi_horizontal_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 4:
-                band->inv_transform = ff_ivi_inverse_slant_4x4;
-                band->dc_transform  = ff_ivi_dc_slant_2d;
-                band->scan          = ff_ivi_direct_scan_4x4;
-                band->transform_size= 4;
+                band->inv_transform  = ff_ivi_inverse_slant_4x4;
+                band->dc_transform   = ff_ivi_dc_slant_2d;
+                band->scan           = ff_ivi_direct_scan_4x4;
+                band->transform_size = 4;
                 break;
             }
 
             band->is_2d_trans = band->inv_transform == ff_ivi_inverse_slant_8x8 ||
                                 band->inv_transform == ff_ivi_inverse_slant_4x4;
 
+            if (band->transform_size != band->blk_size)
+                return AVERROR_INVALIDDATA;
+
             /* select dequant matrix according to plane and band number */
             if (!p) {
                 quant_mat = (pic_conf.luma_bands > 1) ? i+1 : 0;

libavcodec/ivi_common.c

@@ -415,6 +415,20 @@ static int ivi_dec_tile_data_size(GetBitContext *gb)
     return len;
 }
 
+static int ivi_dc_transform(IVIBandDesc *band, int *prev_dc, int buf_offs,
+                            int blk_size)
+{
+    int buf_size = band->pitch * band->aheight - buf_offs;
+    int min_size = (blk_size - 1) * band->pitch + blk_size;
+
+    if (min_size > buf_size)
+        return AVERROR_INVALIDDATA;
+
+    band->dc_transform(prev_dc, band->buf + buf_offs,
+                       band->pitch, blk_size);
+
+    return 0;
+}
 
 static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band,
                                    ivi_mc_func mc, int mv_x, int mv_y,
@@ -432,6 +446,12 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band,
     int num_coeffs = blk_size * blk_size;
     int col_mask   = blk_size - 1;
     int scan_pos   = -1;
+    int min_size   = band->pitch * (band->transform_size - 1) +
+                     band->transform_size;
+    int buf_size   = band->pitch * band->aheight - offs;
+
+    if (min_size > buf_size)
+        return AVERROR_INVALIDDATA;
 
     if (!band->scan) {
         av_log(avctx, AV_LOG_ERROR, "Scan pattern is not set.\n");
@@ -602,8 +622,9 @@ static int ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band,
                 /* for intra blocks apply the dc slant transform */
                 /* for inter - perform the motion compensation without delta */
                 if (is_intra) {
-                        band->dc_transform(&prev_dc, band->buf + buf_offs,
-                                           band->pitch, blk_size);
+                    ret = ivi_dc_transform(band, &prev_dc, buf_offs, blk_size);
+                    if (ret < 0)
+                        return ret;
                 } else {
                     ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf,
                                  buf_offs, mv_x, mv_y, band->pitch, mc_type);

libavcodec/ivi_common.h

@@ -160,9 +160,9 @@ typedef struct IVIBandDesc {
     int             num_tiles;      ///< number of tiles in this band
     IVITile         *tiles;         ///< array of tile descriptors
     InvTransformPtr *inv_transform;
+    int             transform_size;
     DCTransformPtr  *dc_transform;
     int             is_2d_trans;    ///< 1 indicates that the two-dimensional inverse transform is used
-    int             transform_size; ///< block size of the transform
     int32_t         checksum;       ///< for debug purposes
     int             checksum_present;
     int             bufsize;        ///< band buffer size in bytes