vaapi_h264: Enable VBR mode

Default to using VBR when a target bitrate is set, unless the max rate
is also set and matches the target.  Changes to the Intel driver mean
that min_qp is also respected in this case, so set a codec default to
unset the value rather than using the current default inherited from
the MPEG-4 part 2 encoder.

libavcodec/vaapi_encode_h264.c

@@ -1126,13 +1126,15 @@ static av_cold int vaapi_encode_h264_configure(AVCodecContext *avctx)
                "%d / %d / %d for IDR- / P- / B-frames.\n",
                priv->fixed_qp_idr, priv->fixed_qp_p, priv->fixed_qp_b);
 
-    } else if (ctx->va_rc_mode == VA_RC_CBR) {
+    } else if (ctx->va_rc_mode == VA_RC_CBR ||
+               ctx->va_rc_mode == VA_RC_VBR) {
         // These still need to be  set for pic_init_qp/slice_qp_delta.
         priv->fixed_qp_idr = 26;
         priv->fixed_qp_p   = 26;
         priv->fixed_qp_b   = 26;
 
-        av_log(avctx, AV_LOG_DEBUG, "Using constant-bitrate = %d bps.\n",
+        av_log(avctx, AV_LOG_DEBUG, "Using %s-bitrate = %d bps.\n",
+               ctx->va_rc_mode == VA_RC_CBR ? "constant" : "variable",
                avctx->bit_rate);
 
     } else {
@@ -1241,9 +1243,12 @@ static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
     // Only 8-bit encode is supported.
     ctx->va_rt_format = VA_RT_FORMAT_YUV420;
 
-    if (avctx->bit_rate > 0)
-        ctx->va_rc_mode = VA_RC_CBR;
-    else
+    if (avctx->bit_rate > 0) {
+        if (avctx->rc_max_rate == avctx->bit_rate)
+            ctx->va_rc_mode = VA_RC_CBR;
+        else
+            ctx->va_rc_mode = VA_RC_VBR;
+    } else
         ctx->va_rc_mode = VA_RC_CQP;
 
     ctx->va_packed_headers =
@@ -1281,6 +1286,7 @@ static const AVCodecDefault vaapi_encode_h264_defaults[] = {
     { "i_qoffset",      "0.0" },
     { "b_qfactor",      "1.2" },
     { "b_qoffset",      "0.0" },
+    { "qmin",           "0"   },
     { NULL },
 };