tls_gnutls: Readd support for nonblocking operation

The rtmp protocol uses nonblocking reads, to poll for incoming
messages from the server while publishing a stream.

Prior to 94599a6de3 and
d13b124eaf452b267480074b2e6946538ed03a6e, the tls protocol
handled the nonblocking flag, mostly as a side effect from not
using custom IO callbacks for reading from the socket. When custom
IO callbacks were taken into use in
d15eec4d6bdfa3bd4c4b5b7dd2dbd699ba253d02, the handling of a nonblocking
socket wasn't necessary for the default blocking mode any longer.

The code was simplified, since it was overlooked that other code
within libavformat actually used the tls protocol in nonblocking mode.

This fixes publishing over rtmps, with the gnutls backend.

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/tls_gnutls.c

@@ -61,6 +61,7 @@ static int print_tls_error(URLContext *h, int ret)
 {
     switch (ret) {
     case GNUTLS_E_AGAIN:
+        return AVERROR(EAGAIN);
     case GNUTLS_E_INTERRUPTED:
         break;
     case GNUTLS_E_WARNING_ALERT_RECEIVED:
@@ -97,7 +98,10 @@ static ssize_t gnutls_url_pull(gnutls_transport_ptr_t transport,
         return ret;
     if (ret == AVERROR_EXIT)
         return 0;
-    errno = EIO;
+    if (ret == AVERROR(EAGAIN))
+        errno = EAGAIN;
+    else
+        errno = EIO;
     return -1;
 }
 
@@ -110,7 +114,10 @@ static ssize_t gnutls_url_push(gnutls_transport_ptr_t transport,
         return ret;
     if (ret == AVERROR_EXIT)
         return 0;
-    errno = EIO;
+    if (ret == AVERROR(EAGAIN))
+        errno = EAGAIN;
+    else
+        errno = EIO;
     return -1;
 }
 
@@ -202,7 +209,11 @@ fail:
 static int tls_read(URLContext *h, uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = gnutls_record_recv(c->session, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = gnutls_record_recv(c->session, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)
@@ -213,7 +224,11 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
 static int tls_write(URLContext *h, const uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = gnutls_record_send(c->session, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = gnutls_record_send(c->session, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)