diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 44d84d134d..d83758de5f 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -625,6 +625,7 @@ int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count)
                      */
 
             if (h->long_ref[mmco[i].long_arg] != h->cur_pic_ptr) {
+                av_assert0(!h->cur_pic_ptr->long_ref);
                 remove_long(h, mmco[i].long_arg, 0);
                 if (remove_short(h, h->cur_pic_ptr->frame_num, 0)) {
                     av_log(h->avctx, AV_LOG_ERROR, "mmco: cannot assign current picture to short and long at the same time\n");