diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index 9de6bd1c2d..0dc879a58c 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -476,8 +476,10 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
     if (!s->got_header) {
         if ((ret = read_header(s)) < 0)
             return ret;
-        *got_frame_ptr = 0;
-        goto finish_frame;
+        if (avpkt->size) {
+            *got_frame_ptr = 0;
+            goto finish_frame;
+        }
     }
 
     /* if quit command was read previously, don't decode anything */