avformat: add hash and framehash muxers

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Moritz Barsnick <barsnick@gmx.net>
Signed-off-by: James Almer <jamrial@gmail.com>

Changelog

@@ -22,6 +22,7 @@ version <next>:
 - musx demuxer
 - aix demuxer
 - remap filter
+- hash and framehash muxers
 
 version 3.0:
 - Common Encryption (CENC) MP4 encoding and decoding support

doc/muxers.texi

@@ -174,30 +174,70 @@ ffmpeg -i INPUT -c:a pcm_u8 -c:v mpeg2video -f framecrc -
 
 See also the @ref{crc} muxer.
 
-@anchor{framemd5}
-@section framemd5
+@anchor{framehash}
+@section framehash
 
-Per-packet MD5 testing format.
+Per-packet hash testing format.
 
-This muxer computes and prints the MD5 hash for each audio
-and video packet. By default audio frames are converted to signed
-16-bit raw audio and video frames to raw video before computing the
-hash.
+This muxer computes and prints a cryptographic hash for each audio
+and video packet. This can be used for packet-by-packet equality
+checks without having to individually do a binary comparison on each.
+
+By default audio frames are converted to signed 16-bit raw audio and
+video frames to raw video before computing the hash, but the output
+of explicit conversions to other codecs can also be used. It uses the
+SHA-256 cryptographic hash function by default, but supports several
+other algorithms.
 
 The output of the muxer consists of a line for each audio and video
 packet of the form:
 @example
-@var{stream_index}, @var{packet_dts}, @var{packet_pts}, @var{packet_duration}, @var{packet_size}, @var{MD5}
+@var{stream_index}, @var{packet_dts}, @var{packet_pts}, @var{packet_duration}, @var{packet_size}, @var{hash}
 @end example
 
-@var{MD5} is a hexadecimal number representing the computed MD5 hash
+@var{hash} is a hexadecimal number representing the computed hash
 for the packet.
 
+@table @option
+@item hash @var{algorithm}
+Use the cryptographic hash function specified by the string @var{algorithm}.
+Supported values include @code{MD5}, @code{murmur3}, @code{RIPEMD128},
+@code{RIPEMD160}, @code{RIPEMD256}, @code{RIPEMD320}, @code{SHA160},
+@code{SHA224}, @code{SHA256} (default), @code{SHA512/224}, @code{SHA512/256},
+@code{SHA384}, @code{SHA512}, @code{CRC32} and @code{adler32}.
+
+@end table
+
 @subsection Examples
 
-For example to compute the MD5 of the audio and video frames in
-@file{INPUT}, converted to raw audio and video packets, and store it
-in the file @file{out.md5}:
+To compute the SHA-256 hash of the audio and video frames in @file{INPUT},
+converted to raw audio and video packets, and store it in the file
+@file{out.sha256}:
+@example
+ffmpeg -i INPUT -f framehash out.sha256
+@end example
+
+To print the information to stdout, using the MD5 hash function, use
+the command:
+@example
+ffmpeg -i INPUT -f framehash -hash md5 -
+@end example
+
+See also the @ref{hash} muxer.
+
+@anchor{framemd5}
+@section framemd5
+
+Per-packet MD5 testing format.
+
+This is a variant of the @ref{framehash} muxer. Unlike that muxer,
+it defaults to using the MD5 hash function.
+
+@subsection Examples
+
+To compute the MD5 hash of the audio and video frames in @file{INPUT},
+converted to raw audio and video packets, and store it in the file
+@file{out.md5}:
 @example
 ffmpeg -i INPUT -f framemd5 out.md5
 @end example
@@ -207,7 +247,7 @@ To print the information to stdout, use the command:
 ffmpeg -i INPUT -f framemd5 -
 @end example
 
-See also the @ref{md5} muxer.
+See also the @ref{framehash} and @ref{md5} muxers.
 
 @anchor{gif}
 @section gif
@@ -243,6 +283,51 @@ ffmpeg -i INPUT -c:v gif -f image2 "out%d.gif"
 Note 2: the GIF format has a very small time base: the delay between two frames
 can not be smaller than one centi second.
 
+@anchor{hash}
+@section hash
+
+Hash testing format.
+
+This muxer computes and prints a cryptographic hash of all the input
+audio and video frames. This can be used for equality checks without
+having to do a complete binary comparison.
+
+By default audio frames are converted to signed 16-bit raw audio and
+video frames to raw video before computing the hash, but the output
+of explicit conversions to other codecs can also be used. Timestamps
+are ignored. It uses the SHA-256 cryptographic hash function by default,
+but supports several other algorithms.
+
+The output of the muxer consists of a single line of the form:
+@var{algo}=@var{hash}, where @var{algo} is a short string representing
+the hash function used, and @var{hash} is a hexadecimal number
+representing the computed hash.
+
+@table @option
+@item hash @var{algorithm}
+Use the cryptographic hash function specified by the string @var{algorithm}.
+Supported values include @code{MD5}, @code{murmur3}, @code{RIPEMD128},
+@code{RIPEMD160}, @code{RIPEMD256}, @code{RIPEMD320}, @code{SHA160},
+@code{SHA224}, @code{SHA256} (default), @code{SHA512/224}, @code{SHA512/256},
+@code{SHA384}, @code{SHA512}, @code{CRC32} and @code{adler32}.
+
+@end table
+
+@subsection Examples
+
+To compute the SHA-256 hash of the input converted to raw audio and
+video, and store it in the file @file{out.sha256}:
+@example
+ffmpeg -i INPUT -f hash out.sha256
+@end example
+
+To print an MD5 hash to stdout use the command:
+@example
+ffmpeg -i INPUT -f hash -hash md5 -
+@end example
+
+See also the @ref{framehash} muxer.
+
 @anchor{hls}
 @section hls
 
@@ -629,16 +714,12 @@ have no effect if it is not.
 
 MD5 testing format.
 
-This muxer computes and prints the MD5 hash of all the input audio
-and video frames. By default audio frames are converted to signed
-16-bit raw audio and video frames to raw video before computing the
-hash. Timestamps are ignored.
+This is a variant of the @ref{hash} muxer. Unlike that muxer, it
+defaults to using the MD5 hash function.
 
-The output of the muxer consists of a single line of the form:
-MD5=@var{MD5}, where @var{MD5} is a hexadecimal number representing
-the computed MD5 hash.
+@subsection Examples
 
-For example to compute the MD5 hash of the input converted to raw
+To compute the MD5 hash of the input converted to raw
 audio and video, and store it in the file @file{out.md5}:
 @example
 ffmpeg -i INPUT -f md5 out.md5
@@ -649,7 +730,7 @@ You can print the MD5 to stdout with the command:
 ffmpeg -i INPUT -f md5 -
 @end example
 
-See also the @ref{framemd5} muxer.
+See also the @ref{hash} and @ref{framemd5} muxers.
 
 @section mov, mp4, ismv
 

libavformat/Makefile

@@ -175,7 +175,8 @@ OBJS-$(CONFIG_LIVE_FLV_DEMUXER)          += flvdec.o
 OBJS-$(CONFIG_FLV_MUXER)                 += flvenc.o avc.o
 OBJS-$(CONFIG_FOURXM_DEMUXER)            += 4xm.o
 OBJS-$(CONFIG_FRAMECRC_MUXER)            += framecrcenc.o framehash.o
-OBJS-$(CONFIG_FRAMEMD5_MUXER)            += md5enc.o framehash.o
+OBJS-$(CONFIG_FRAMEHASH_MUXER)           += hashenc.o framehash.o
+OBJS-$(CONFIG_FRAMEMD5_MUXER)            += hashenc.o framehash.o
 OBJS-$(CONFIG_FRM_DEMUXER)               += frmdec.o
 OBJS-$(CONFIG_FSB_DEMUXER)               += fsb.o
 OBJS-$(CONFIG_GIF_MUXER)                 += gif.o
@@ -196,6 +197,7 @@ OBJS-$(CONFIG_H263_DEMUXER)              += h263dec.o rawdec.o
 OBJS-$(CONFIG_H263_MUXER)                += rawenc.o
 OBJS-$(CONFIG_H264_DEMUXER)              += h264dec.o rawdec.o
 OBJS-$(CONFIG_H264_MUXER)                += rawenc.o
+OBJS-$(CONFIG_HASH_MUXER)                += hashenc.o
 OBJS-$(CONFIG_HDS_MUXER)                 += hdsenc.o
 OBJS-$(CONFIG_HEVC_DEMUXER)              += hevcdec.o rawdec.o
 OBJS-$(CONFIG_HEVC_MUXER)                += rawenc.o
@@ -259,7 +261,7 @@ OBJS-$(CONFIG_MATROSKA_MUXER)            += matroskaenc.o matroska.o \
                                             avc.o hevc.o \
                                             flacenc_header.o avlanguage.o vorbiscomment.o wv.o \
                                             webmdashenc.o webm_chunk.o
-OBJS-$(CONFIG_MD5_MUXER)                 += md5enc.o
+OBJS-$(CONFIG_MD5_MUXER)                 += hashenc.o
 OBJS-$(CONFIG_MGSTS_DEMUXER)             += mgsts.o
 OBJS-$(CONFIG_MICRODVD_DEMUXER)          += microdvddec.o subtitles.o
 OBJS-$(CONFIG_MICRODVD_MUXER)            += microdvdenc.o

libavformat/allformats.c

@@ -131,6 +131,7 @@ void av_register_all(void)
     REGISTER_DEMUXER (LIVE_FLV,         live_flv);
     REGISTER_DEMUXER (FOURXM,           fourxm);
     REGISTER_MUXER   (FRAMECRC,         framecrc);
+    REGISTER_MUXER   (FRAMEHASH,        framehash);
     REGISTER_MUXER   (FRAMEMD5,         framemd5);
     REGISTER_DEMUXER (FRM,              frm);
     REGISTER_DEMUXER (FSB,              fsb);
@@ -144,6 +145,7 @@ void av_register_all(void)
     REGISTER_MUXDEMUX(H261,             h261);
     REGISTER_MUXDEMUX(H263,             h263);
     REGISTER_MUXDEMUX(H264,             h264);
+    REGISTER_MUXER   (HASH,             hash);
     REGISTER_MUXER   (HDS,              hds);
     REGISTER_MUXDEMUX(HEVC,             hevc);
     REGISTER_MUXDEMUX(HLS,              hls);

libavformat/md5enc.c → libavformat/hashenc.c

@@ -1,5 +1,5 @@
 /*
- * MD5 encoder (for codec/format testing)
+ * Hash/MD5 encoder (for codec/format testing)
  * Copyright (c) 2009 Reimar Döffinger, based on crcenc (c) 2002 Fabrice Bellard
  *
  * This file is part of FFmpeg.
@@ -26,23 +26,23 @@
 #include "avformat.h"
 #include "internal.h"
 
-struct MD5Context {
+struct HashContext {
     const AVClass *avclass;
     struct AVHashContext *hash;
     char *hash_name;
     int format_version;
 };
 
-static void md5_finish(struct AVFormatContext *s, char *buf)
+static void hash_finish(struct AVFormatContext *s, char *buf)
 {
-    struct MD5Context *c = s->priv_data;
-    uint8_t md5[AV_HASH_MAX_SIZE];
+    struct HashContext *c = s->priv_data;
+    uint8_t hash[AV_HASH_MAX_SIZE];
     int i, offset = strlen(buf);
     int len = av_hash_get_size(c->hash);
-    av_assert0(len > 0 && len <= sizeof(md5));
-    av_hash_final(c->hash, md5);
+    av_assert0(len > 0 && len <= sizeof(hash));
+    av_hash_final(c->hash, hash);
     for (i = 0; i < len; i++) {
-        snprintf(buf + offset, 3, "%02"PRIx8, md5[i]);
+        snprintf(buf + offset, 3, "%02"PRIx8, hash[i]);
         offset += 2;
     }
     buf[offset] = '\n';
@@ -52,25 +52,28 @@ static void md5_finish(struct AVFormatContext *s, char *buf)
     avio_flush(s->pb);
 }
 
-#define OFFSET(x) offsetof(struct MD5Context, x)
+#define OFFSET(x) offsetof(struct HashContext, x)
 #define ENC AV_OPT_FLAG_ENCODING_PARAM
+#if CONFIG_HASH_MUXER || CONFIG_FRAMEHASH_MUXER
 static const AVOption hash_options[] = {
-    { "hash", "set hash to use", OFFSET(hash_name), AV_OPT_TYPE_STRING, {.str = "md5"}, 0, 0, ENC },
+    { "hash", "set hash to use", OFFSET(hash_name), AV_OPT_TYPE_STRING, {.str = "sha256"}, 0, 0, ENC },
     { "format_version", "file format version", OFFSET(format_version), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 1, ENC },
     { NULL },
 };
+#endif
 
-static const AVClass md5enc_class = {
-    .class_name = "hash encoder class",
-    .item_name  = av_default_item_name,
-    .option     = hash_options,
-    .version    = LIBAVUTIL_VERSION_INT,
+#if CONFIG_MD5_MUXER || CONFIG_FRAMEMD5_MUXER
+static const AVOption md5_options[] = {
+    { "hash", "set hash to use", OFFSET(hash_name), AV_OPT_TYPE_STRING, {.str = "md5"}, 0, 0, ENC },
+    { "format_version", "file format version", OFFSET(format_version), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 1, ENC },
+    { NULL },
 };
+#endif
 
-#if CONFIG_MD5_MUXER
-static int write_header(struct AVFormatContext *s)
+#if CONFIG_HASH_MUXER || CONFIG_MD5_MUXER
+static int hash_write_header(struct AVFormatContext *s)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     int res = av_hash_alloc(&c->hash, c->hash_name);
     if (res < 0)
         return res;
@@ -78,45 +81,77 @@ static int write_header(struct AVFormatContext *s)
     return 0;
 }
 
-static int write_packet(struct AVFormatContext *s, AVPacket *pkt)
+static int hash_write_packet(struct AVFormatContext *s, AVPacket *pkt)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     av_hash_update(c->hash, pkt->data, pkt->size);
     return 0;
 }
 
-static int write_trailer(struct AVFormatContext *s)
+static int hash_write_trailer(struct AVFormatContext *s)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     char buf[256];
     av_strlcpy(buf, av_hash_get_name(c->hash), sizeof(buf) - 200);
     av_strlcat(buf, "=", sizeof(buf) - 200);
 
-    md5_finish(s, buf);
+    hash_finish(s, buf);
 
     av_hash_freep(&c->hash);
     return 0;
 }
+#endif
+
+#if CONFIG_HASH_MUXER
+static const AVClass hashenc_class = {
+    .class_name = "hash encoder class",
+    .item_name  = av_default_item_name,
+    .option     = hash_options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+AVOutputFormat ff_hash_muxer = {
+    .name              = "hash",
+    .long_name         = NULL_IF_CONFIG_SMALL("Hash testing"),
+    .priv_data_size    = sizeof(struct HashContext),
+    .audio_codec       = AV_CODEC_ID_PCM_S16LE,
+    .video_codec       = AV_CODEC_ID_RAWVIDEO,
+    .write_header      = hash_write_header,
+    .write_packet      = hash_write_packet,
+    .write_trailer     = hash_write_trailer,
+    .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT |
+                         AVFMT_TS_NEGATIVE,
+    .priv_class        = &hashenc_class,
+};
+#endif
+
+#if CONFIG_MD5_MUXER
+static const AVClass md5enc_class = {
+    .class_name = "MD5 encoder class",
+    .item_name  = av_default_item_name,
+    .option     = md5_options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
 
 AVOutputFormat ff_md5_muxer = {
     .name              = "md5",
     .long_name         = NULL_IF_CONFIG_SMALL("MD5 testing"),
-    .priv_data_size    = sizeof(struct MD5Context),
+    .priv_data_size    = sizeof(struct HashContext),
     .audio_codec       = AV_CODEC_ID_PCM_S16LE,
     .video_codec       = AV_CODEC_ID_RAWVIDEO,
-    .write_header      = write_header,
-    .write_packet      = write_packet,
-    .write_trailer     = write_trailer,
+    .write_header      = hash_write_header,
+    .write_packet      = hash_write_packet,
+    .write_trailer     = hash_write_trailer,
     .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT |
                          AVFMT_TS_NEGATIVE,
     .priv_class        = &md5enc_class,
 };
 #endif
 
-#if CONFIG_FRAMEMD5_MUXER
-static int framemd5_write_header(struct AVFormatContext *s)
+#if CONFIG_FRAMEHASH_MUXER || CONFIG_FRAMEMD5_MUXER
+static int framehash_write_header(struct AVFormatContext *s)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     int res = av_hash_alloc(&c->hash, c->hash_name);
     if (res < 0)
         return res;
@@ -128,42 +163,67 @@ static int framemd5_write_header(struct AVFormatContext *s)
     return 0;
 }
 
-static int framemd5_write_packet(struct AVFormatContext *s, AVPacket *pkt)
+static int framehash_write_packet(struct AVFormatContext *s, AVPacket *pkt)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     char buf[256];
     av_hash_init(c->hash);
     av_hash_update(c->hash, pkt->data, pkt->size);
 
     snprintf(buf, sizeof(buf) - 64, "%d, %10"PRId64", %10"PRId64", %8"PRId64", %8d, ",
              pkt->stream_index, pkt->dts, pkt->pts, pkt->duration, pkt->size);
-    md5_finish(s, buf);
+    hash_finish(s, buf);
     return 0;
 }
 
-static int framemd5_write_trailer(struct AVFormatContext *s)
+static int framehash_write_trailer(struct AVFormatContext *s)
 {
-    struct MD5Context *c = s->priv_data;
+    struct HashContext *c = s->priv_data;
     av_hash_freep(&c->hash);
     return 0;
 }
+#endif
 
-static const AVClass framemd5_class = {
+#if CONFIG_FRAMEHASH_MUXER
+static const AVClass framehash_class = {
     .class_name = "frame hash encoder class",
     .item_name  = av_default_item_name,
     .option     = hash_options,
     .version    = LIBAVUTIL_VERSION_INT,
 };
 
+AVOutputFormat ff_framehash_muxer = {
+    .name              = "framehash",
+    .long_name         = NULL_IF_CONFIG_SMALL("Per-frame hash testing"),
+    .priv_data_size    = sizeof(struct HashContext),
+    .audio_codec       = AV_CODEC_ID_PCM_S16LE,
+    .video_codec       = AV_CODEC_ID_RAWVIDEO,
+    .write_header      = framehash_write_header,
+    .write_packet      = framehash_write_packet,
+    .write_trailer     = framehash_write_trailer,
+    .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT |
+                         AVFMT_TS_NEGATIVE,
+    .priv_class        = &framehash_class,
+};
+#endif
+
+#if CONFIG_FRAMEMD5_MUXER
+static const AVClass framemd5_class = {
+    .class_name = "frame hash encoder class",
+    .item_name  = av_default_item_name,
+    .option     = md5_options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
 AVOutputFormat ff_framemd5_muxer = {
     .name              = "framemd5",
     .long_name         = NULL_IF_CONFIG_SMALL("Per-frame MD5 testing"),
-    .priv_data_size    = sizeof(struct MD5Context),
+    .priv_data_size    = sizeof(struct HashContext),
     .audio_codec       = AV_CODEC_ID_PCM_S16LE,
     .video_codec       = AV_CODEC_ID_RAWVIDEO,
-    .write_header      = framemd5_write_header,
-    .write_packet      = framemd5_write_packet,
-    .write_trailer     = framemd5_write_trailer,
+    .write_header      = framehash_write_header,
+    .write_packet      = framehash_write_packet,
+    .write_trailer     = framehash_write_trailer,
     .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT |
                          AVFMT_TS_NEGATIVE,
     .priv_class        = &framemd5_class,

libavformat/version.h

@@ -30,7 +30,7 @@
 #include "libavutil/version.h"
 
 #define LIBAVFORMAT_VERSION_MAJOR  57
-#define LIBAVFORMAT_VERSION_MINOR  33
+#define LIBAVFORMAT_VERSION_MINOR  34
 #define LIBAVFORMAT_VERSION_MICRO 100
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \