avfilter/vf_vectorscope: make color mode more useful

Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavfilter/vf_vectorscope.c

@@ -305,16 +305,6 @@ static void vectorscope(VectorscopeContext *s, AVFrame *in, AVFrame *out, int pd
                 }
             }
         }
-        if (s->mode == COLOR) {
-            for (i = 0; i < out->height; i++) {
-                for (j = 0; j < out->width; j++) {
-                    if (!dpd[i * out->linesize[pd] + j]) {
-                        dpx[i * out->linesize[px] + j] = j;
-                        dpy[i * out->linesize[py] + j] = i;
-                    }
-                }
-            }
-        }
         break;
     case COLOR2:
         if (s->is_yuv) {
@@ -393,6 +383,18 @@ static void vectorscope(VectorscopeContext *s, AVFrame *in, AVFrame *out, int pd
     }
 
     envelope(s, out);
+
+    if (s->mode == COLOR) {
+        for (i = 0; i < out->height; i++) {
+            for (j = 0; j < out->width; j++) {
+                if (!dpd[i * out->linesize[pd] + j]) {
+                    dpx[i * out->linesize[px] + j] = j;
+                    dpy[i * out->linesize[py] + j] = i;
+                    dpd[i * out->linesize[pd] + j] = 128;
+                }
+            }
+        }
+    }
 }
 
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)

tests/ref/fate/filter-vectorscope_color

@@ -1,4 +1,4 @@
 #tb 0: 1/25
-0,          0,          0,        1,   196608, 0x9d705c63
-0,          1,          1,        1,   196608, 0x9fbf6f2c
-0,          2,          2,        1,   196608, 0x257b7290
+0,          0,          0,        1,   196608, 0x1890aa30
+0,          1,          1,        1,   196608, 0xa490acf9
+0,          2,          2,        1,   196608, 0x404a775d