rtmpproto: Only prepend @setDataFrame for onMetaData and |RtmpSampleAccess

Currently, when streaming to an RTMP server, any time a packet of type
RTMP_PT_NOTIFY is encountered, the packet is prepended with @setDataFrame
before it gets sent to the server. This is incorrect; only packets for
onMetaData and |RtmpSampleAccess should invoke @setDataFrame on the RTMP
server. Specifically, the current bug manifests itself when trying to
stream onTextData or onCuePoint invocations.

This fix addresses that problem and ensures that the @setDataFrame is
only prepended for onMetaData and |RtmpSampleAccess.

Since data is fed to the rtmp_write function in smaller pieces (depending
on the calling IO buffer size), we can't generally assume that the
whole packet (or even the whole command string) is available at once,
therefore we can only check the command string once the full packet
has been transferred to us for sending.

Based on a patch by Jeffrey Wescott.

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/rtmpproto.c

@@ -2970,8 +2970,6 @@ static int rtmp_write(URLContext *s, const uint8_t *buf, int size)
 
             if (((pkttype == RTMP_PT_VIDEO || pkttype == RTMP_PT_AUDIO) && ts == 0) ||
                 pkttype == RTMP_PT_NOTIFY) {
-                if (pkttype == RTMP_PT_NOTIFY)
-                    pktsize += 16;
                 if ((ret = ff_rtmp_check_alloc_array(&rt->prev_pkt[1],
                                                      &rt->nb_prev_pkt[1],
                                                      channel)) < 0)
@@ -2989,9 +2987,6 @@ static int rtmp_write(URLContext *s, const uint8_t *buf, int size)
 
             rt->out_pkt.extra = rt->stream_id;
             rt->flv_data = rt->out_pkt.data;
-
-            if (pkttype == RTMP_PT_NOTIFY)
-                ff_amf_write_string(&rt->flv_data, "@setDataFrame");
         }
 
         copy = FFMIN(rt->flv_size - rt->flv_off, size_temp);
@@ -3002,6 +2997,33 @@ static int rtmp_write(URLContext *s, const uint8_t *buf, int size)
         if (rt->flv_off == rt->flv_size) {
             rt->skip_bytes = 4;
 
+            if (rt->out_pkt.type == RTMP_PT_NOTIFY) {
+                // For onMetaData and |RtmpSampleAccess packets, we want
+                // @setDataFrame prepended to the packet before it gets sent.
+                // However, not all RTMP_PT_NOTIFY packets (e.g., onTextData
+                // and onCuePoint).
+                uint8_t commandbuffer[64];
+                int stringlen = 0;
+                GetByteContext gbc;
+
+                bytestream2_init(&gbc, rt->flv_data, rt->flv_size);
+                if (!ff_amf_read_string(&gbc, commandbuffer, sizeof(commandbuffer),
+                                        &stringlen)) {
+                    if (!strcmp(commandbuffer, "onMetaData") ||
+                        !strcmp(commandbuffer, "|RtmpSampleAccess")) {
+                        uint8_t *ptr;
+                        if ((ret = av_reallocp(&rt->out_pkt.data, rt->out_pkt.size + 16)) < 0) {
+                            rt->flv_size = rt->flv_off = rt->flv_header_bytes = 0;
+                            return ret;
+                        }
+                        memmove(rt->out_pkt.data + 16, rt->out_pkt.data, rt->out_pkt.size);
+                        rt->out_pkt.size += 16;
+                        ptr = rt->out_pkt.data;
+                        ff_amf_write_string(&ptr, "@setDataFrame");
+                    }
+                }
+            }
+
             if ((ret = rtmp_send_packet(rt, &rt->out_pkt, 0)) < 0)
                 return ret;
             rt->flv_size = 0;