sws/fill_rgb2yuv_table: increase precission to 32bit

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libswscale/utils.c

@@ -801,7 +801,7 @@ static void getSubSampleFactors(int *h, int *v, enum AVPixelFormat format)
 
 static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange)
 {
-    int64_t W,V,Z;
+    int64_t W, V, Z, Cy, Cu, Cv;
     int64_t vr =  table[0];
     int64_t ub =  table[1];
     int64_t ug = -table[2];
@@ -858,21 +858,25 @@ static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange)
         ug = ug * 224 / 255;
         vg = vg * 224 / 255;
     }
-    W = ONE*ug/ub;
-    V = ONE*vg/vr;
-    Z = ONE-W-V;
+    W = ONE*ONE*ug/ub;
+    V = ONE*ONE*vg/vr;
+    Z = ONE*ONE-W-V;
 
-    c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*V/(Z*cy);
-    c->input_rgb2yuv_table[GY_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*cy);
-    c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*W/(Z*cy);
+    Cy = cy*Z / ONE;
+    Cu = ub*Z / ONE;
+    Cv = vr*Z / ONE;
 
-    c->input_rgb2yuv_table[RU_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*V/(Z*ub);
-    c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*ub);
-    c->input_rgb2yuv_table[BU_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*(Z+W)/(Z*ub);
+    c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*V/Cy;
+    c->input_rgb2yuv_table[GY_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*ONE/Cy;
+    c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*W/Cy;
 
-    c->input_rgb2yuv_table[RV_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*(V+Z)/(Z*vr);
-    c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*vr);
-    c->input_rgb2yuv_table[BV_IDX] =  (1 << RGB2YUV_SHIFT)*ONE*W/(Z*vr);
+    c->input_rgb2yuv_table[RU_IDX] =  (1 << RGB2YUV_SHIFT)*V/Cu;
+    c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cu;
+    c->input_rgb2yuv_table[BU_IDX] =  (1 << RGB2YUV_SHIFT)*(Z+W)/Cu;
+
+    c->input_rgb2yuv_table[RV_IDX] =  (1 << RGB2YUV_SHIFT)*(V+Z)/Cv;
+    c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cv;
+    c->input_rgb2yuv_table[BV_IDX] =  (1 << RGB2YUV_SHIFT)*W/Cv;
 
     if(/*!dstRange && */table == ff_yuv2rgb_coeffs[SWS_CS_DEFAULT]) {
         c->input_rgb2yuv_table[BY_IDX] =  ((int)(0.114 * 219 / 255 * (1 << RGB2YUV_SHIFT) + 0.5));