rtpenc: Don't set max_frames_per_packet based on the packet frame size or frame rate

Instead check the timestamps while muxing, to avoid buffering a
too long timestamp range into one single packet.

This makes the AMR and AAC packetization slightly less efficient,
since we set a possibly unnecessarily high max_frames_per_packet.
(These packetizers end up doing a memmove of the TOC bytes if
sending a packet before max_frames_per_packet is achieved, and
we end up setting max_frames_per_packet to a value that should
be high enough for most uses.)

All packetizers that use max_frames_per_packet now set it either
to a default value, or to a value calculated based on other
parameters, so none of them rely on the previous default setting.

For iLBC, copy one frame at a time, to allow checking the timestamp
range for each of them - basically doing potentially multiple
loops to simplify the code instead of trying to calculate the
number of frames to buffer while honoring s1->max_delay.

This is in preparation for reducing the coupling between libavformat
and libavcodec, by not having the muxers use the encoder field
frame_size (which may not be available during e.g. stream copy).

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/rtpenc.c

@@ -149,33 +149,6 @@ static int rtp_write_header(AVFormatContext *s1)
     }
     s->max_payload_size = s1->packet_size - 12;
 
-    s->max_frames_per_packet = 0;
-    if (s1->max_delay > 0) {
-        if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
-            int frame_size = av_get_audio_frame_duration(st->codec, 0);
-            if (!frame_size)
-                frame_size = st->codec->frame_size;
-            if (frame_size == 0) {
-                av_log(s1, AV_LOG_ERROR, "Cannot respect max delay: frame size = 0\n");
-            } else {
-                s->max_frames_per_packet =
-                        av_rescale_q_rnd(s1->max_delay,
-                                         AV_TIME_BASE_Q,
-                                         (AVRational){ frame_size, st->codec->sample_rate },
-                                         AV_ROUND_DOWN);
-            }
-        }
-        if (st->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
-            /* FIXME: We should round down here... */
-            if (st->avg_frame_rate.num > 0 && st->avg_frame_rate.den > 0) {
-                s->max_frames_per_packet = av_rescale_q(s1->max_delay,
-                                                        (AVRational){1, 1000000},
-                                                        av_inv_q(st->avg_frame_rate));
-            } else
-                s->max_frames_per_packet = 1;
-        }
-    }
-
     if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
         avpriv_set_pts_info(st, 32, 1, st->codec->sample_rate);
     } else {
@@ -225,9 +198,7 @@ static int rtp_write_header(AVFormatContext *s1)
         break;
     case AV_CODEC_ID_VORBIS:
     case AV_CODEC_ID_THEORA:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 15;
-        s->max_frames_per_packet = av_clip(s->max_frames_per_packet, 1, 15);
+        s->max_frames_per_packet = 15;
         break;
     case AV_CODEC_ID_ADPCM_G722:
         /* Due to a historical error, the clock rate for G722 in RTP is
@@ -249,15 +220,11 @@ static int rtp_write_header(AVFormatContext *s1)
             av_log(s1, AV_LOG_ERROR, "Incorrect iLBC block size specified\n");
             goto fail;
         }
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 1;
-        s->max_frames_per_packet = FFMIN(s->max_frames_per_packet,
-                                         s->max_payload_size / st->codec->block_align);
+        s->max_frames_per_packet = s->max_payload_size / st->codec->block_align;
         break;
     case AV_CODEC_ID_AMR_NB:
     case AV_CODEC_ID_AMR_WB:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 12;
+        s->max_frames_per_packet = 50;
         if (st->codec->codec_id == AV_CODEC_ID_AMR_NB)
             n = 31;
         else
@@ -273,8 +240,7 @@ static int rtp_write_header(AVFormatContext *s1)
         }
         break;
     case AV_CODEC_ID_AAC:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 5;
+        s->max_frames_per_packet = 50;
         break;
     default:
         break;
@@ -493,18 +459,23 @@ static int rtp_send_ilbc(AVFormatContext *s1, const uint8_t *buf, int size)
     int frames = size / frame_size;
 
     while (frames > 0) {
-        int n = FFMIN(s->max_frames_per_packet - s->num_frames, frames);
+        if (s->num_frames > 0 &&
+            av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                          s1->max_delay, AV_TIME_BASE_Q) >= 0) {
+            ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 1);
+            s->num_frames = 0;
+        }
 
         if (!s->num_frames) {
             s->buf_ptr = s->buf;
             s->timestamp = s->cur_timestamp;
         }
-        memcpy(s->buf_ptr, buf, n * frame_size);
-        frames           -= n;
-        s->num_frames    += n;
-        s->buf_ptr       += n * frame_size;
-        buf              += n * frame_size;
-        s->cur_timestamp += n * frame_duration;
+        memcpy(s->buf_ptr, buf, frame_size);
+        frames--;
+        s->num_frames++;
+        s->buf_ptr       += frame_size;
+        buf              += frame_size;
+        s->cur_timestamp += frame_duration;
 
         if (s->num_frames == s->max_frames_per_packet) {
             ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 1);

libavformat/rtpenc_aac.c

@@ -27,6 +27,7 @@
 void ff_rtp_send_aac(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s = s1->priv_data;
+    AVStream *st = s1->streams[0];
     const int max_au_headers_size = 2 + 2 * s->max_frames_per_packet;
     int len, max_packet_size = s->max_payload_size - max_au_headers_size;
     uint8_t *p;
@@ -41,7 +42,9 @@ void ff_rtp_send_aac(AVFormatContext *s1, const uint8_t *buff, int size)
     len = (s->buf_ptr - s->buf);
     if (s->num_frames &&
         (s->num_frames == s->max_frames_per_packet ||
-         (len + size) > s->max_payload_size)) {
+         (len + size) > s->max_payload_size ||
+         av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                       s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
         int au_size = s->num_frames * 2;
 
         p = s->buf + max_au_headers_size - au_size - 2;

libavformat/rtpenc_amr.c

@@ -30,6 +30,7 @@
 void ff_rtp_send_amr(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s          = s1->priv_data;
+    AVStream *st              = s1->streams[0];
     int max_header_toc_size   = 1 + s->max_frames_per_packet;
     uint8_t *p;
     int len;
@@ -38,7 +39,9 @@ void ff_rtp_send_amr(AVFormatContext *s1, const uint8_t *buff, int size)
     len = s->buf_ptr - s->buf;
     if (s->num_frames &&
         (s->num_frames == s->max_frames_per_packet ||
-         len + size - 1 > s->max_payload_size)) {
+         len + size - 1 > s->max_payload_size ||
+         av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                       s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
         int header_size = s->num_frames + 1;
         p = s->buf + max_header_toc_size - header_size;
         if (p != s->buf)

libavformat/rtpenc_xiph.c

@@ -32,6 +32,7 @@
 void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s = s1->priv_data;
+    AVStream *st = s1->streams[0];
     int max_pkt_size, xdt, frag;
     uint8_t *q;
 
@@ -77,8 +78,10 @@ void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
         assert(s->num_frames <= s->max_frames_per_packet);
         if (s->num_frames > 0 &&
             (remaining < 0 ||
-             s->num_frames == s->max_frames_per_packet)) {
-            // send previous packets now; no room for new data
+             s->num_frames == s->max_frames_per_packet ||
+             av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                           s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
+            // send previous packets now; no room for new data, or too much delay
             ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 0);
             s->num_frames = 0;
         }