From 700fb8c8dd0622fcadb1a35eb2363e1e4c94e663 Mon Sep 17 00:00:00 2001
From: Luca Barbato <lu_zero@gentoo.org>
Date: Fri, 14 Dec 2012 08:22:06 +0100
Subject: [PATCH] vp56: make parse_header return standard error codes

Returning 0 for failure is misleading.

CC: libav-stable@libav.org

(cherry picked from commit bb675d3ac6d722d5e117ae9042a996b55ca05b1d)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
---
 libavcodec/vp5.c  | 12 ++++++------
 libavcodec/vp56.c |  8 ++++----
 libavcodec/vp56.h |  2 ++
 libavcodec/vp6.c  | 12 ++++++------
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/libavcodec/vp5.c b/libavcodec/vp5.c
index 2b975801d0..bb6c224309 100644
--- a/libavcodec/vp5.c
+++ b/libavcodec/vp5.c
@@ -47,18 +47,18 @@ static int vp5_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     {
         vp56_rac_gets(c, 8);
         if(vp56_rac_gets(c, 5) > 5)
-            return 0;
+            return AVERROR_INVALIDDATA;
         vp56_rac_gets(c, 2);
         if (vp56_rac_get(c)) {
             av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n");
-            return 0;
+            return AVERROR_PATCHWELCOME;
         }
         rows = vp56_rac_gets(c, 8);  /* number of stored macroblock rows */
         cols = vp56_rac_gets(c, 8);  /* number of stored macroblock cols */
         if (!rows || !cols) {
             av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n",
                    cols << 4, rows << 4);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
         vp56_rac_gets(c, 8);  /* number of displayed macroblock rows */
         vp56_rac_gets(c, 8);  /* number of displayed macroblock cols */
@@ -67,11 +67,11 @@ static int vp5_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
             16*cols != s->avctx->coded_width ||
             16*rows != s->avctx->coded_height) {
             avcodec_set_dimensions(s->avctx, 16*cols, 16*rows);
-            return 2;
+            return VP56_SIZE_CHANGE;
         }
     } else if (!s->macroblocks)
-        return 0;
-    return 1;
+        return AVERROR_INVALIDDATA;
+    return 0;
 }
 
 static void vp5_parse_vector_adjustment(VP56Context *s, VP56mv *vect)
diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index dd9dd77527..96f40a1bd0 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -511,10 +511,10 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         s->modelp = &s->models[is_alpha];
 
         res = s->parse_header(s, buf, remaining_buf_size, &golden_frame);
-        if (!res)
-            return -1;
+        if (res < 0)
+            return res;
 
-        if (res == 2) {
+        if (res == VP56_SIZE_CHANGE) {
             int i;
             for (i = 0; i < 4; i++) {
                 if (s->frames[i].data[0])
@@ -533,7 +533,7 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
                 return -1;
             }
 
-            if (res == 2)
+            if (res == VP56_SIZE_CHANGE)
                 if (vp56_size_changed(avctx)) {
                     avctx->release_buffer(avctx, p);
                     return -1;
diff --git a/libavcodec/vp56.h b/libavcodec/vp56.h
index 0c34438c8f..ad6977351a 100644
--- a/libavcodec/vp56.h
+++ b/libavcodec/vp56.h
@@ -38,6 +38,8 @@ typedef struct {
     int16_t y;
 } DECLARE_ALIGNED(4, , VP56mv);
 
+#define VP56_SIZE_CHANGE 1
+
 typedef void (*VP56ParseVectorAdjustment)(VP56Context *s,
                                           VP56mv *vect);
 typedef void (*VP56Filter)(VP56Context *s, uint8_t *dst, uint8_t *src,
diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index b0d8642465..7347f9fac2 100644
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -50,7 +50,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     int vrt_shift = 0;
     int sub_version;
     int rows, cols;
-    int res = 1;
+    int res = 0;
     int separated_coeff = buf[0] & 1;
 
     s->framep[VP56_FRAME_CURRENT]->key_frame = !(buf[0] & 0x80);
@@ -59,7 +59,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     if (s->framep[VP56_FRAME_CURRENT]->key_frame) {
         sub_version = buf[1] >> 3;
         if (sub_version > 8)
-            return 0;
+            return AVERROR_INVALIDDATA;
         s->filter_header = buf[1] & 0x06;
         if (buf[1] & 1) {
             av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n");
@@ -77,7 +77,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         /* buf[5] is number of displayed macroblock cols */
         if (!rows || !cols) {
             av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n", cols << 4, rows << 4);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
 
         if (!s->macroblocks || /* first frame */
@@ -88,7 +88,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
                 s->avctx->width  -= s->avctx->extradata[0] >> 4;
                 s->avctx->height -= s->avctx->extradata[0] & 0x0F;
             }
-            res = 2;
+            res = VP56_SIZE_CHANGE;
         }
 
         ff_vp56_init_range_decoder(c, buf+6, buf_size-6);
@@ -100,7 +100,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         s->sub_version = sub_version;
     } else {
         if (!s->sub_version || !s->avctx->coded_width || !s->avctx->coded_height)
-            return 0;
+            return AVERROR_INVALIDDATA;
 
         if (separated_coeff || !s->filter_header) {
             coeff_offset = AV_RB16(buf+1) - 2;
@@ -144,7 +144,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         if (buf_size < 0) {
             if (s->framep[VP56_FRAME_CURRENT]->key_frame)
                 avcodec_set_dimensions(s->avctx, 0, 0);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
         if (s->use_huffman) {
             s->parse_coeff = vp6_parse_coeff_huffman;