vorbis: Validate that the floor 1 X values contain no duplicates.

Duplicate values in this vector are explicitly banned by the Vorbis I spec
and cause divide-by-zero crashes later on.
(cherry picked from commit ecf79c4d3e)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

libavcodec/vorbis.c

@@ -119,7 +119,8 @@ int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num)
     return 0;
 }
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values)
 {
     int i;
     list[0].sort = 0;
@@ -143,6 +144,11 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     for (i = 0; i < values - 1; i++) {
         int j;
         for (j = i + 1; j < values; j++) {
+            if (list[i].x == list[j].x) {
+                av_log(avccontext, AV_LOG_ERROR,
+                       "Duplicate value found in floor 1 X coordinates\n");
+                return AVERROR_INVALIDDATA;
+            }
             if (list[list[i].sort].x > list[list[j].sort].x) {
                 int tmp = list[i].sort;
                 list[i].sort = list[j].sort;
@@ -150,6 +156,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
             }
         }
     }
+    return 0;
 }
 
 static inline void render_line_unrolled(intptr_t x, int y, int x1,

libavcodec/vorbis.h

@@ -36,7 +36,8 @@ typedef struct {
     uint16_t high;
 } vorbis_floor1_entry;
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values);
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values);
 unsigned int ff_vorbis_nth_root(unsigned int x, unsigned int n); // x^(1/n)
 int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num);
 void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,

libavcodec/vorbisdec.c

@@ -574,7 +574,11 @@ static int vorbis_parse_setup_hdr_floors(vorbis_context *vc)
             }
 
 // Precalculate order of x coordinates - needed for decode
-            ff_vorbis_ready_floor1_list(floor_setup->data.t1.list, floor_setup->data.t1.x_list_dim);
+            if (ff_vorbis_ready_floor1_list(vc->avccontext,
+                                            floor_setup->data.t1.list,
+                                            floor_setup->data.t1.x_list_dim)) {
+                return AVERROR_INVALIDDATA;
+            }
         } else if (floor_setup->floor_type == 0) {
             unsigned max_codebook_dim = 0;
 

libavcodec/vorbisenc.c

@@ -336,7 +336,8 @@ static int create_vorbis_context(vorbis_enc_context *venc,
         };
         fc->list[i].x = a[i - 2];
     }
-    ff_vorbis_ready_floor1_list(fc->list, fc->values);
+    if (ff_vorbis_ready_floor1_list(avccontext, fc->list, fc->values))
+        return AVERROR_BUG;
 
     venc->nresidues = 1;
     venc->residues  = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues);