avformat/aacdec: add a custom read_packet function

Atempt to read and propagate only full ADTS frames and not other data,
like id3v1 or APETags at the end of the file.

Fixes ticket #6437.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: James Almer <jamrial@gmail.com>

libavformat/aacdec.c

@@ -23,10 +23,11 @@
 #include "libavutil/intreadwrite.h"
 #include "avformat.h"
 #include "internal.h"
-#include "rawdec.h"
 #include "id3v1.h"
 #include "apetag.h"
 
+#define ADTS_HEADER_SIZE 7
+
 static int adts_aac_probe(AVProbeData *p)
 {
     int max_frames = 0, first_frames = 0;
@@ -79,6 +80,7 @@ static int adts_aac_probe(AVProbeData *p)
 static int adts_aac_read_header(AVFormatContext *s)
 {
     AVStream *st;
+    uint16_t state;
 
     st = avformat_new_stream(s, NULL);
     if (!st)
@@ -96,18 +98,56 @@ static int adts_aac_read_header(AVFormatContext *s)
         avio_seek(s->pb, cur, SEEK_SET);
     }
 
+    // skip data until the first ADTS frame is found
+    state = avio_r8(s->pb);
+    while (!avio_feof(s->pb) && avio_tell(s->pb) < s->probesize) {
+        state = (state << 8) | avio_r8(s->pb);
+        if ((state >> 4) != 0xFFF)
+            continue;
+        avio_seek(s->pb, -2, SEEK_CUR);
+        break;
+    }
+    if ((state >> 4) != 0xFFF)
+        return AVERROR_INVALIDDATA;
+
     // LCM of all possible ADTS sample rates
     avpriv_set_pts_info(st, 64, 1, 28224000);
 
     return 0;
 }
 
+static int adts_aac_read_packet(AVFormatContext *s, AVPacket *pkt)
+{
+    int ret, fsize;
+
+    ret = av_get_packet(s->pb, pkt, ADTS_HEADER_SIZE);
+    if (ret < 0)
+        return ret;
+    if (ret < ADTS_HEADER_SIZE) {
+        av_packet_unref(pkt);
+        return AVERROR(EIO);
+    }
+
+    if ((AV_RB16(pkt->data) >> 4) != 0xfff) {
+        av_packet_unref(pkt);
+        return AVERROR_INVALIDDATA;
+    }
+
+    fsize = (AV_RB32(pkt->data + 3) >> 13) & 0x1FFF;
+    if (fsize < ADTS_HEADER_SIZE) {
+        av_packet_unref(pkt);
+        return AVERROR_INVALIDDATA;
+    }
+
+    return av_append_packet(s->pb, pkt, fsize - ADTS_HEADER_SIZE);
+}
+
 AVInputFormat ff_aac_demuxer = {
     .name         = "aac",
     .long_name    = NULL_IF_CONFIG_SMALL("raw ADTS AAC (Advanced Audio Coding)"),
     .read_probe   = adts_aac_probe,
     .read_header  = adts_aac_read_header,
-    .read_packet  = ff_raw_read_partial_packet,
+    .read_packet  = adts_aac_read_packet,
     .flags        = AVFMT_GENERIC_INDEX,
     .extensions   = "aac",
     .mime_type    = "audio/aac,audio/aacp,audio/x-aac",