avcodec/mpegvideo_enc: Ignore QMAX if VBV constraints are exceeded

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/mpegvideo.h

@@ -539,6 +539,7 @@ typedef struct MpegEncContext {
     float rc_buffer_aggressivity;
     float border_masking;
     int lmin, lmax;
+    int vbv_ignore_qmax;
 
     char *rc_eq;
 

libavcodec/mpegvideo_enc.c

@@ -173,7 +173,8 @@ static inline void update_qscale(MpegEncContext *s)
 
         for (i = 0 ; i<FF_ARRAY_ELEMS(non_linear_qscale); i++) {
             int diff = FFABS((non_linear_qscale[i]<<(FF_LAMBDA_SHIFT + 7)) - (int)s->lambda * 139);
-            if (non_linear_qscale[i] < s->avctx->qmin || non_linear_qscale[i] > s->avctx->qmax)
+            if (non_linear_qscale[i] < s->avctx->qmin ||
+                (non_linear_qscale[i] > s->avctx->qmax && !s->vbv_ignore_qmax))
                 continue;
             if (diff < bestdiff) {
                 bestdiff = diff;
@@ -184,7 +185,7 @@ static inline void update_qscale(MpegEncContext *s)
     } else {
         s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
                     (FF_LAMBDA_SHIFT + 7);
-        s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
+        s->qscale = av_clip(s->qscale, s->avctx->qmin, s->vbv_ignore_qmax ? 31 : s->avctx->qmax);
     }
 
     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
@@ -1748,6 +1749,8 @@ int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
     int i, stuffing_count, ret;
     int context_count = s->slice_context_count;
 
+    s->vbv_ignore_qmax = 0;
+
     s->picture_in_gop_number++;
 
     if (load_input_picture(s, pic_arg) < 0)
@@ -1844,6 +1847,7 @@ vbv_retry:
                     PutBitContext *pb = &s->thread_context[i]->pb;
                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
                 }
+                s->vbv_ignore_qmax = 1;
                 av_log(s->avctx, AV_LOG_VERBOSE, "reencoding frame due to VBV\n");
                 goto vbv_retry;
             }