vaapi_encode: Support VBR mode

This includes a backward-compatibility hack to choose CBR anyway on
old drivers which have no CBR support, so that existing programs will
continue to work their options now map to VBR.

libavcodec/vaapi_encode.c

@@ -1034,6 +1034,19 @@ static av_cold int vaapi_encode_config_attributes(AVCodecContext *avctx)
             };
             break;
         case VAConfigAttribRateControl:
+            // Hack for backward compatibility: CBR was the only
+            // usable RC mode for a long time, so old drivers will
+            // only have it.  Normal default options may now choose
+            // VBR and then fail, however, so override it here with
+            // CBR if that is the only supported mode.
+            if (ctx->va_rc_mode == VA_RC_VBR &&
+                !(attr[i].value & VA_RC_VBR) &&
+                (attr[i].value & VA_RC_CBR)) {
+                av_log(avctx, AV_LOG_WARNING, "VBR rate control is "
+                       "not supported with this driver version; "
+                       "using CBR instead.\n");
+                ctx->va_rc_mode = VA_RC_CBR;
+            }
             if (!(ctx->va_rc_mode & attr[i].value)) {
                 av_log(avctx, AV_LOG_ERROR, "Rate control mode %#x "
                        "is not supported (mask: %#x).\n",
@@ -1098,6 +1111,9 @@ fail:
 static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
+    int rc_bits_per_second;
+    int rc_target_percentage;
+    int rc_window_size;
     int hrd_buffer_size;
     int hrd_initial_buffer_fullness;
 
@@ -1110,13 +1126,29 @@ static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
     else
         hrd_initial_buffer_fullness = hrd_buffer_size * 3 / 4;
 
+    if (ctx->va_rc_mode == VA_RC_CBR) {
+        rc_bits_per_second   = avctx->bit_rate;
+        rc_target_percentage = 100;
+        rc_window_size       = 1000;
+    } else {
+        if (avctx->rc_max_rate < avctx->bit_rate) {
+            // Max rate is unset or invalid, just use the normal bitrate.
+            rc_bits_per_second   = avctx->bit_rate;
+            rc_target_percentage = 100;
+        } else {
+            rc_bits_per_second   = avctx->rc_max_rate;
+            rc_target_percentage = (avctx->bit_rate * 100) / rc_bits_per_second;
+        }
+        rc_window_size = (hrd_buffer_size * 1000) / avctx->bit_rate;
+    }
+
     ctx->rc_params.misc.type = VAEncMiscParameterTypeRateControl;
     ctx->rc_params.rc = (VAEncMiscParameterRateControl) {
-        .bits_per_second   = avctx->bit_rate,
-        .target_percentage = 66,
-        .window_size       = 1000,
-        .initial_qp        = (avctx->qmax >= 0 ? avctx->qmax : 40),
-        .min_qp            = (avctx->qmin >= 0 ? avctx->qmin : 18),
+        .bits_per_second   = rc_bits_per_second,
+        .target_percentage = rc_target_percentage,
+        .window_size       = rc_window_size,
+        .initial_qp        = 0,
+        .min_qp            = (avctx->qmin > 0 ? avctx->qmin : 0),
         .basic_unit_size   = 0,
     };
     ctx->global_params[ctx->nb_global_params] =