avformat/matroskaenc: write a CRC32 element on each Cluster

Implements part of ticket #4347

Tested-by: Dave Rice <dave@dericed.com>
Tested-by: Jerome Martinez <jerome@mediaarea.net>
Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: James Almer <jamrial@gmail.com>

libavformat/matroskaenc.c

@@ -37,6 +37,7 @@
 
 #include "libavutil/avstring.h"
 #include "libavutil/channel_layout.h"
+#include "libavutil/crc.h"
 #include "libavutil/dict.h"
 #include "libavutil/intfloat.h"
 #include "libavutil/intreadwrite.h"
@@ -305,6 +306,45 @@ static void end_ebml_master(AVIOContext *pb, ebml_master master)
     avio_seek(pb, pos, SEEK_SET);
 }
 
+static int start_ebml_master_crc32(AVIOContext *pb, AVIOContext **dyn_cp, ebml_master *master,
+                                   unsigned int elementid, uint64_t expectedsize)
+{
+    int ret;
+
+    if (ret = avio_open_dyn_buf(dyn_cp) < 0)
+        return ret;
+
+    if (pb->seekable)
+        *master = start_ebml_master(pb, elementid, expectedsize);
+    else
+        *master = start_ebml_master(*dyn_cp, elementid, expectedsize);
+
+    return 0;
+}
+
+static void end_ebml_master_crc32(AVIOContext *pb, AVIOContext **dyn_cp, MatroskaMuxContext *mkv,
+                                  ebml_master master)
+{
+    uint8_t *buf, crc[4];
+    int size;
+
+    if (pb->seekable) {
+        size = avio_close_dyn_buf(*dyn_cp, &buf);
+        if (mkv->mode != MODE_WEBM) {
+            AV_WL32(crc, av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), UINT32_MAX, buf, size) ^ UINT32_MAX);
+            put_ebml_binary(pb, EBML_ID_CRC32, crc, sizeof(crc));
+        }
+        avio_write(pb, buf, size);
+        end_ebml_master(pb, master);
+    } else {
+        end_ebml_master(*dyn_cp, master);
+        size = avio_close_dyn_buf(*dyn_cp, &buf);
+        avio_write(pb, buf, size);
+    }
+    av_free(buf);
+    *dyn_cp = NULL;
+}
+
 static void put_xiph_size(AVIOContext *pb, int size)
 {
     ffio_fill(pb, 255, size / 255);
@@ -1924,26 +1964,12 @@ static int mkv_write_vtt_blocks(AVFormatContext *s, AVIOContext *pb, AVPacket *p
     return pkt->duration;
 }
 
-static void mkv_flush_dynbuf(AVFormatContext *s)
-{
-    MatroskaMuxContext *mkv = s->priv_data;
-    int bufsize;
-    uint8_t *dyn_buf;
-
-    bufsize = avio_close_dyn_buf(mkv->dyn_bc, &dyn_buf);
-    avio_write(s->pb, dyn_buf, bufsize);
-    av_free(dyn_buf);
-    mkv->dyn_bc = NULL;
-}
-
 static void mkv_start_new_cluster(AVFormatContext *s, AVPacket *pkt)
 {
     MatroskaMuxContext *mkv = s->priv_data;
-    AVIOContext *pb = mkv->dyn_bc;
 
-    end_ebml_master(pb, mkv->cluster);
+    end_ebml_master_crc32(s->pb, &mkv->dyn_bc, mkv, mkv->cluster);
     mkv->cluster_pos = -1;
-    mkv_flush_dynbuf(s);
     if (s->pb->seekable)
         av_log(s, AV_LOG_DEBUG,
                "Starting new cluster at offset %" PRIu64 " bytes, "
@@ -1982,23 +2008,17 @@ static int mkv_write_packet_internal(AVFormatContext *s, AVPacket *pkt, int add_
         }
     }
 
-    if (!mkv->dyn_bc) {
-        ret = avio_open_dyn_buf(&mkv->dyn_bc);
-        if (ret < 0) {
-            av_log(s, AV_LOG_ERROR, "Failed to open dynamic buffer\n");
-            return ret;
-        }
-    }
-    pb = mkv->dyn_bc;
-
     if (mkv->cluster_pos == -1) {
         mkv->cluster_pos = avio_tell(s->pb);
-        mkv->cluster     = start_ebml_master(pb, MATROSKA_ID_CLUSTER, 0);
-        put_ebml_uint(pb, MATROSKA_ID_CLUSTERTIMECODE, FFMAX(0, ts));
+        ret = start_ebml_master_crc32(s->pb, &mkv->dyn_bc, &mkv->cluster, MATROSKA_ID_CLUSTER, 0);
+        if (ret < 0)
+            return ret;
+        put_ebml_uint(mkv->dyn_bc, MATROSKA_ID_CLUSTERTIMECODE, FFMAX(0, ts));
         mkv->cluster_pts = FFMAX(0, ts);
     }
+    pb = mkv->dyn_bc;
 
-    relative_packet_pos = avio_tell(pb) - mkv->cluster.pos;
+    relative_packet_pos = avio_tell(s->pb) - mkv->cluster.pos + avio_tell(pb);
 
     if (par->codec_type != AVMEDIA_TYPE_SUBTITLE) {
         mkv_write_block(s, pb, MATROSKA_ID_SIMPLEBLOCK, pkt, keyframe);
@@ -2118,12 +2138,11 @@ static int mkv_write_packet(AVFormatContext *s, AVPacket *pkt)
 static int mkv_write_flush_packet(AVFormatContext *s, AVPacket *pkt)
 {
     MatroskaMuxContext *mkv = s->priv_data;
-    AVIOContext *pb = mkv->dyn_bc;
+
     if (!pkt) {
         if (mkv->cluster_pos != -1) {
-            end_ebml_master(pb, mkv->cluster);
+            end_ebml_master_crc32(s->pb, &mkv->dyn_bc, mkv, mkv->cluster);
             mkv->cluster_pos = -1;
-            mkv_flush_dynbuf(s);
             if (s->pb->seekable)
                 av_log(s, AV_LOG_DEBUG,
                        "Flushing cluster at offset %" PRIu64 " bytes\n",
@@ -2156,8 +2175,7 @@ static int mkv_write_trailer(AVFormatContext *s)
     }
 
     if (mkv->dyn_bc) {
-        end_ebml_master(mkv->dyn_bc, mkv->cluster);
-        mkv_flush_dynbuf(s);
+        end_ebml_master_crc32(pb, &mkv->dyn_bc, mkv, mkv->cluster);
     }
 
     if (mkv->mode != MODE_WEBM) {

tests/ref/fate/rgb24-mkv

@@ -1,5 +1,5 @@
-0d081c8e151a922435830f95000d3c71 *tests/data/fate/rgb24-mkv.matroska
-58321 tests/data/fate/rgb24-mkv.matroska
+7761f59d660edcc835a1186cc3875332 *tests/data/fate/rgb24-mkv.matroska
+58327 tests/data/fate/rgb24-mkv.matroska
 #tb 0: 1/10
 #media_type 0: video
 #codec_id 0: rawvideo

tests/ref/lavf/mka

@@ -1,3 +1,3 @@
-b2e3746787b885d0191a1a26f3faa58f *./tests/data/lavf/lavf.mka
-43654 ./tests/data/lavf/lavf.mka
+56dab37c9f48bf0f9f16cfed8f1f6c17 *./tests/data/lavf/lavf.mka
+43660 ./tests/data/lavf/lavf.mka
 ./tests/data/lavf/lavf.mka CRC=0x3a1da17e

tests/ref/lavf/mkv

@@ -1,6 +1,6 @@
-5b982c8dfbadc71f51b206cbd10b9a71 *./tests/data/lavf/lavf.mkv
-472875 ./tests/data/lavf/lavf.mkv
+0c2b156a749ef4fd22cef29685d6b058 *./tests/data/lavf/lavf.mkv
+472893 ./tests/data/lavf/lavf.mkv
 ./tests/data/lavf/lavf.mkv CRC=0xec6c3c68
-b4a295bae8e6cf536563cb840386f3a4 *./tests/data/lavf/lavf.mkv
-320551 ./tests/data/lavf/lavf.mkv
+cf768eb8de3dccc6cf8c4a281e600edc *./tests/data/lavf/lavf.mkv
+320569 ./tests/data/lavf/lavf.mkv
 ./tests/data/lavf/lavf.mkv CRC=0xec6c3c68

tests/ref/seek/lavf-mkv

@@ -1,48 +1,48 @@
-ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    800 size:   208
+ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    806 size:   208
 ret: 0         st:-1 flags:0  ts:-1.000000
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret: 0         st:-1 flags:1  ts: 1.894167
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret: 0         st: 0 flags:0  ts: 0.788000
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret: 0         st: 0 flags:1  ts:-0.317000
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret:-1         st: 1 flags:0  ts: 2.577000
 ret: 0         st: 1 flags:1  ts: 1.471000
-ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320271 size:   209
+ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320289 size:   209
 ret: 0         st:-1 flags:0  ts: 0.365002
-ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146983 size: 27925
+ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146995 size: 27925
 ret: 0         st:-1 flags:1  ts:-0.740831
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret:-1         st: 0 flags:0  ts: 2.153000
 ret: 0         st: 0 flags:1  ts: 1.048000
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret: 0         st: 1 flags:0  ts:-0.058000
-ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    800 size:   208
+ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    806 size:   208
 ret: 0         st: 1 flags:1  ts: 2.836000
-ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320271 size:   209
+ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320289 size:   209
 ret:-1         st:-1 flags:0  ts: 1.730004
 ret: 0         st:-1 flags:1  ts: 0.624171
-ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146983 size: 27925
+ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146995 size: 27925
 ret: 0         st: 0 flags:0  ts:-0.482000
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret: 0         st: 0 flags:1  ts: 2.413000
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret:-1         st: 1 flags:0  ts: 1.307000
 ret: 0         st: 1 flags:1  ts: 0.201000
-ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    800 size:   208
+ret: 0         st: 1 flags:1 dts: 0.000000 pts: 0.000000 pos:    806 size:   208
 ret: 0         st:-1 flags:0  ts:-0.904994
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret: 0         st:-1 flags:1  ts: 1.989173
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret: 0         st: 0 flags:0  ts: 0.883000
-ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292430 size: 27834
+ret: 0         st: 0 flags:1 dts: 0.971000 pts: 0.971000 pos: 292448 size: 27834
 ret: 0         st: 0 flags:1  ts:-0.222000
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837
 ret:-1         st: 1 flags:0  ts: 2.672000
 ret: 0         st: 1 flags:1  ts: 1.566000
-ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320271 size:   209
+ret: 0         st: 1 flags:1 dts: 0.993000 pts: 0.993000 pos: 320289 size:   209
 ret: 0         st:-1 flags:0  ts: 0.460008
-ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146983 size: 27925
+ret: 0         st: 0 flags:1 dts: 0.491000 pts: 0.491000 pos: 146995 size: 27925
 ret: 0         st:-1 flags:1  ts:-0.645825
-ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1016 size: 27837
+ret: 0         st: 0 flags:1 dts: 0.011000 pts: 0.011000 pos:   1022 size: 27837