diff --git a/libavfilter/vf_pad.c b/libavfilter/vf_pad.c
index 413fde1f6c..08a1f8e668 100644
--- a/libavfilter/vf_pad.c
+++ b/libavfilter/vf_pad.c
@@ -284,7 +284,7 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
           )
             break;
     }
-    pad->needs_copy= plane < 4 && outpicref->data[plane];
+    pad->needs_copy= plane < 4 && outpicref->data[plane] || !(outpicref->perms & AV_PERM_WRITE);
     if(pad->needs_copy){
         av_log(inlink->dst, AV_LOG_DEBUG, "Direct padding impossible allocating new frame\n");
         avfilter_unref_buffer(outpicref);