diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 52eef0dd5a..6d719d7985 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -893,6 +893,10 @@ typedef struct AVStream {
      */
     int pts_wrap_behavior;
 
+    /**
+     * Internal data to prevent doing update_initial_durations() twice
+     */
+    int update_initial_durations_done;
 } AVStream;
 
 AVRational av_stream_get_r_frame_rate(const AVStream *s);
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 40d886f58a..b0cb427968 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -998,6 +998,9 @@ static void update_initial_durations(AVFormatContext *s, AVStream *st,
     int64_t cur_dts= RELATIVE_TS_BASE;
 
     if(st->first_dts != AV_NOPTS_VALUE){
+        if (st->update_initial_durations_done)
+            return;
+        st->update_initial_durations_done = 1;
         cur_dts= st->first_dts;
         for(; pktl; pktl= get_next_pkt(s, st, pktl)){
             if(pktl->pkt.stream_index == stream_index){