diff --git a/libavcodec/pthread.c b/libavcodec/pthread.c
index 6ae763da80..c58222b490 100644
--- a/libavcodec/pthread.c
+++ b/libavcodec/pthread.c
@@ -390,7 +390,7 @@ static attribute_align_arg void *frame_worker_thread(void *arg)
 
         pthread_mutex_lock(&p->progress_mutex);
         for (i = 0; i < MAX_BUFFERS; i++)
-            if (p->progress_used[i]) {
+            if (p->progress_used[i] && (p->got_frame || p->result<0 || avctx->codec_id != CODEC_ID_H264)) {
                 p->progress[i][0] = INT_MAX;
                 p->progress[i][1] = INT_MAX;
             }