vda: Signal 4 byte NAL headers to the decoder regardless of what's in the extradata

Signed-off-by: Martin Storsjö <martin@martin.st>

libavcodec/vda.c

@@ -142,6 +142,26 @@ int ff_vda_create_decoder(struct vda_context *vda_ctx,
 
     pthread_mutex_init(&vda_ctx->queue_mutex, NULL);
 
+    /* Each VCL NAL in the bistream sent to the decoder
+     * is preceeded by a 4 bytes length header.
+     * Change the avcC atom header if needed, to signal headers of 4 bytes. */
+    if (extradata_size >= 4 && (extradata[4] & 0x03) != 0x03) {
+        uint8_t *rw_extradata;
+
+        if (!(rw_extradata = av_malloc(extradata_size)))
+            return AVERROR(ENOMEM);
+
+        memcpy(rw_extradata, extradata, extradata_size);
+
+        rw_extradata[4] |= 0x03;
+
+        avc_data = CFDataCreate(kCFAllocatorDefault, rw_extradata, extradata_size);
+
+        av_freep(&rw_extradata);
+    } else {
+        avc_data = CFDataCreate(kCFAllocatorDefault, extradata, extradata_size);
+    }
+
     config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
                                             4,
                                             &kCFTypeDictionaryKeyCallBacks,
@@ -150,7 +170,6 @@ int ff_vda_create_decoder(struct vda_context *vda_ctx,
     height   = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->height);
     width    = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->width);
     format   = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->format);
-    avc_data = CFDataCreate(kCFAllocatorDefault, extradata, extradata_size);
 
     CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height);
     CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width);