avcodec/jpeglsenc: Check for memory allocation failures

Fixes CID1271044

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/jpeglsenc.c

@@ -253,8 +253,9 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
     const int near         = avctx->prediction_method;
     PutBitContext pb, pb2;
     GetBitContext gb;
-    uint8_t *buf2, *zero, *cur, *last;
-    JLSState *state;
+    uint8_t *buf2 = NULL;
+    uint8_t *zero, *cur, *last;
+    JLSState *state = NULL;
     int i, size, ret;
     int comps;
 
@@ -269,6 +270,8 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
         return ret;
 
     buf2 = av_malloc(pkt->size);
+    if (!buf2)
+        goto fail;
 
     init_put_bits(&pb, pkt->data, pkt->size);
     init_put_bits(&pb2, buf2, pkt->size);
@@ -299,6 +302,8 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
     put_bits(&pb, 8, 0);  // point transform: none
 
     state = av_mallocz(sizeof(JLSState));
+    if (!state)
+        goto fail;
     /* initialize JPEG-LS state from JPEG parameters */
     state->near = near;
     state->bpp  = (avctx->pix_fmt == AV_PIX_FMT_GRAY16) ? 16 : 8;
@@ -308,10 +313,9 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
     ls_store_lse(state, &pb);
 
     zero = av_mallocz(FFABS(p->linesize[0]));
-    if (!zero) {
-        av_free(state);
-        return AVERROR(ENOMEM);
-    }
+    if (!zero)
+        goto fail;
+
     last = zero;
     cur  = p->data[0];
     if (avctx->pix_fmt == AV_PIX_FMT_GRAY8) {
@@ -384,7 +388,7 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
         }
     }
     avpriv_align_put_bits(&pb);
-    av_free(buf2);
+    av_freep(&buf2);
 
     /* End of image */
     put_marker(&pb, EOI);
@@ -396,6 +400,11 @@ static int encode_picture_ls(AVCodecContext *avctx, AVPacket *pkt,
     pkt->flags |= AV_PKT_FLAG_KEY;
     *got_packet = 1;
     return 0;
+fail:
+    av_freep(&buf2);
+    av_freep(&state);
+
+    return AVERROR(ENOMEM);
 }
 
 static av_cold int encode_close(AVCodecContext *avctx)