Merge commit '3beb9cbad35218ed1fb3473eeb3cfc97a931bff4'

* commit '3beb9cbad35218ed1fb3473eeb3cfc97a931bff4':
  roqvideo: check memory allocations and propagate errors

Conflicts:
	libavcodec/roqvideoenc.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/roqvideoenc.c

@@ -245,11 +245,13 @@ typedef struct RoqTempData
 /**
  * Initialize cel evaluators and set their source coordinates
  */
-static void create_cel_evals(RoqContext *enc, RoqTempdata *tempData)
+static int create_cel_evals(RoqContext *enc, RoqTempdata *tempData)
 {
     int n=0, x, y, i;
 
     tempData->cel_evals = av_malloc_array(enc->width*enc->height/64, sizeof(CelEvaluation));
+    if (!tempData->cel_evals)
+        return AVERROR(ENOMEM);
 
     /* Map to the ROQ quadtree order */
     for (y=0; y<enc->height; y+=16)
@@ -258,6 +260,8 @@ static void create_cel_evals(RoqContext *enc, RoqTempdata *tempData)
                 tempData->cel_evals[n  ].sourceX = x + (i&1)*8;
                 tempData->cel_evals[n++].sourceY = y + (i&2)*4;
             }
+
+    return 0;
 }
 
 /**
@@ -792,26 +796,36 @@ static void create_clusters(const AVFrame *frame, int w, int h, uint8_t *yuvClus
         }
 }
 
-static void generate_codebook(RoqContext *enc, RoqTempdata *tempdata,
-                              int *points, int inputCount, roq_cell *results,
-                              int size, int cbsize)
+static int generate_codebook(RoqContext *enc, RoqTempdata *tempdata,
+                             int *points, int inputCount, roq_cell *results,
+                             int size, int cbsize)
 {
-    int i, j, k;
+    int i, j, k, ret = 0;
     int c_size = size*size/4;
     int *buf;
     int *codebook = av_malloc_array(6*c_size, cbsize*sizeof(int));
     int *closest_cb;
 
-    if (size == 4)
+    if (!codebook)
+        return AVERROR(ENOMEM);
+
+    if (size == 4) {
         closest_cb = av_malloc_array(6*c_size, inputCount*sizeof(int));
-    else
+        if (!closest_cb) {
+            ret = AVERROR(ENOMEM);
+            goto out;
+        }
+    } else
         closest_cb = tempdata->closest_cb2;
 
-    avpriv_init_elbg(points, 6*c_size, inputCount, codebook, cbsize, 1, closest_cb, &enc->randctx);
-    avpriv_do_elbg(points, 6*c_size, inputCount, codebook, cbsize, 1, closest_cb, &enc->randctx);
-
-    if (size == 4)
-        av_free(closest_cb);
+    ret = avpriv_init_elbg(points, 6 * c_size, inputCount, codebook,
+                       cbsize, 1, closest_cb, &enc->randctx);
+    if (ret < 0)
+        goto out;
+    ret = avpriv_do_elbg(points, 6 * c_size, inputCount, codebook,
+                     cbsize, 1, closest_cb, &enc->randctx);
+    if (ret < 0)
+        goto out;
 
     buf = codebook;
     for (i=0; i<cbsize; i++)
@@ -823,13 +837,16 @@ static void generate_codebook(RoqContext *enc, RoqTempdata *tempdata,
             results->v =    (*buf++ + CHROMA_BIAS/2)/CHROMA_BIAS;
             results++;
         }
-
+out:
+    if (size == 4)
+        av_free(closest_cb);
     av_free(codebook);
+    return ret;
 }
 
-static void generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData)
+static int generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData)
 {
-    int i,j;
+    int i, j, ret = 0;
     RoqCodebooks *codebooks = &tempData->codebooks;
     int max = enc->width*enc->height/16;
     uint8_t mb2[3*4];
@@ -838,6 +855,11 @@ static void generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData)
     int *points = av_malloc_array(max, 6*4*sizeof(int));
     int bias;
 
+    if (!results4 || !yuvClusters || !points) {
+        ret = AVERROR(ENOMEM);
+        goto out;
+    }
+
     /* Subsample YUV data */
     create_clusters(enc->frame_to_enc, enc->width, enc->height, yuvClusters);
 
@@ -848,14 +870,22 @@ static void generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData)
     }
 
     /* Create 4x4 codebooks */
-    generate_codebook(enc, tempData, points, max, results4, 4, (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4));
+    if ((ret = generate_codebook(enc, tempData, points, max,
+                                 results4, 4, (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4))) < 0)
+        goto out;
 
     codebooks->numCB4 = (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4);
 
     tempData->closest_cb2 = av_malloc_array(max, 4*sizeof(int));
+    if (!tempData->closest_cb2) {
+        ret = AVERROR(ENOMEM);
+        goto out;
+    }
 
     /* Create 2x2 codebooks */
-    generate_codebook(enc, tempData, points, max*4, enc->cb2x2, 2, MAX_CBS_2x2);
+    if ((ret = generate_codebook(enc, tempData, points, max * 4,
+                                 enc->cb2x2, 2, MAX_CBS_2x2)) < 0)
+        goto out;
 
     codebooks->numCB2 = MAX_CBS_2x2;
 
@@ -875,22 +905,27 @@ static void generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData)
         enlarge_roq_mb4(codebooks->unpacked_cb4 + i*4*4*3,
                         codebooks->unpacked_cb4_enlarged + i*8*8*3);
     }
-
+out:
     av_free(yuvClusters);
     av_free(points);
     av_free(results4);
+    return ret;
 }
 
 static int roq_encode_video(RoqContext *enc)
 {
     RoqTempdata *tempData = enc->tmpData;
-    int i;
+    int i, ret;
 
     memset(tempData, 0, sizeof(*tempData));
 
-    create_cel_evals(enc, tempData);
+    ret = create_cel_evals(enc, tempData);
+    if (ret < 0)
+        return ret;
 
-    generate_new_codebooks(enc, tempData);
+    ret = generate_new_codebooks(enc, tempData);
+    if (ret < 0)
+        return ret;
 
     if (enc->framesSinceKeyframe >= 1) {
         motion_search(enc, 8);
@@ -1075,7 +1110,8 @@ static int roq_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     }
 
     /* Encode the actual frame */
-    if ((ret = roq_encode_video(enc)) < 0)
+    ret = roq_encode_video(enc);
+    if (ret < 0)
         return ret;
 
     pkt->size   = enc->out_buf - pkt->data;