diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
index 9b35476f2a..2a305abd07 100644
--- a/libavcodec/cuvid.c
+++ b/libavcodec/cuvid.c
@@ -143,6 +143,19 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
 
     avctx->pix_fmt = surface_fmt;
 
+    // Update our hwframe ctx, as the get_format callback might have refreshed it!
+    if (avctx->hw_frames_ctx) {
+        av_buffer_unref(&ctx->hwframe);
+
+        ctx->hwframe = av_buffer_ref(avctx->hw_frames_ctx);
+        if (!ctx->hwframe) {
+            ctx->internal_error = AVERROR(ENOMEM);
+            return 0;
+        }
+
+        hwframe_ctx = (AVHWFramesContext*)ctx->hwframe->data;
+    }
+
     avctx->width = format->display_area.right;
     avctx->height = format->display_area.bottom;