g722enc: validate AVCodecContext.trellis

libavcodec/g722enc.c

@@ -36,6 +36,11 @@
    problems, so we limit it to a reasonable value */
 #define MAX_FRAME_SIZE 32768
 
+/* We clip the value of avctx->trellis to prevent data type overflows and
+   undefined behavior. Using larger values is insanely slow anyway. */
+#define MIN_TRELLIS 0
+#define MAX_TRELLIS 16
+
 static av_cold int g722_encode_init(AVCodecContext * avctx)
 {
     G722Context *c = avctx->priv_data;
@@ -83,6 +88,17 @@ static av_cold int g722_encode_init(AVCodecContext * avctx)
         avctx->frame_size = 320;
     }
 
+    if (avctx->trellis) {
+        /* validate trellis */
+        if (avctx->trellis < MIN_TRELLIS || avctx->trellis > MAX_TRELLIS) {
+            int new_trellis = av_clip(avctx->trellis, MIN_TRELLIS, MAX_TRELLIS);
+            av_log(avctx, AV_LOG_WARNING, "Requested trellis value is not "
+                   "allowed. Using %d instead of %d\n", new_trellis,
+                   avctx->trellis);
+            avctx->trellis = new_trellis;
+        }
+    }
+
     return 0;
 }