avfilter/vf_rotate: increase fixed point precision

This ensures int_sins output doesnt have "random" lsbs

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavfilter/vf_rotate.c

@@ -252,11 +252,12 @@ static int config_props(AVFilterLink *outlink)
 }
 
 #define FIXP (1<<16)
-#define INT_PI 205887 //(M_PI * FIXP)
+#define FIXP2 (1<<20)
+#define INT_PI 3294199 //(M_PI * FIXP2)
 
 /**
  * Compute the sin of a using integer values.
- * Input and output values are scaled by FIXP.
+ * Input is scaled by FIXP2 and output values are scaled by FIXP.
  */
 static int64_t int_sin(int64_t a)
 {
@@ -268,13 +269,13 @@ static int64_t int_sin(int64_t a)
     if (a >= INT_PI*3/2) a -= 2*INT_PI;  // -PI/2 .. 3PI/2
     if (a >= INT_PI/2  ) a = INT_PI - a; // -PI/2 ..  PI/2
 
-    /* compute sin using Taylor series approximated to the third term */
-    a2 = (a*a)/FIXP;
-    for (i = 2; i < 7; i += 2) {
+    /* compute sin using Taylor series approximated to the fifth term */
+    a2 = (a*a)/(FIXP2);
+    for (i = 2; i < 11; i += 2) {
         res += a;
-        a = -a*a2 / (FIXP*i*(i+1));
+        a = -a*a2 / (FIXP2*i*(i+1));
     }
-    return res;
+    return (res + 8)>>4;
 }
 
 /**
@@ -402,7 +403,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     av_log(ctx, AV_LOG_DEBUG, "n:%f time:%f angle:%f/PI\n",
            rot->var_values[VAR_N], rot->var_values[VAR_T], rot->angle/M_PI);
 
-    angle_int = res * FIXP;
+    angle_int = res * FIXP * 16;
     s = int_sin(angle_int);
     c = int_sin(angle_int + INT_PI/2);
 

tests/ref/fate/filter-pixfmts-rotate

@@ -1,20 +1,20 @@
-0bgr                1040a5c4645582fc271f7be40ea5aaf7
-0rgb                7f21fcf8fd658de854b75dd8c47b0b00
-abgr                24f441d2e6e67cae2d3451aa1dad23a7
-argb                993002f41f621d04cd76278e466c03eb
-bgr0                efe11efe2840fa84ee95cdb913463bc1
-bgr24               d0f449e8b38e07c947bd808f441a8ace
-bgra                54cebf01881cb63ec3727f7cc23b0a6b
-gbrap               6d69c0cd0cba6300065f8d990e35b081
-gbrp                db3b6345d2a5c0fb524f93486d97193e
-gray                5a896c38449a0fb08129a7394952eb31
-rgb0                c29f92ff5224044c7272c763fa5321e6
-rgb24               739f0eb47e76ce5c87354d5631ac2d5b
-rgba                f25570a798f24e8174729d978872c272
-yuv410p             22e673170464119cafb1a973c5a8080c
-yuv420p             afed4567a1b2d54ca9bc87bbdfff9a34
-yuv444p             df57aba68928092b54f6b75ab01e3110
-yuva420p            ad9bdb4f21855550a6f94e2d96588097
-yuva444p            b6a345f46ef75814033f733ccb4da42e
-yuvj420p            ccc0724c2ff91daa3848db1f4d91d010
-yuvj444p            9165b6e7c647c93cd24011d931890edb
+0bgr                8bce6fc2b0532e6dceee3bae394c170b
+0rgb                b1f893e98c87f32d0131f7f019bd3c10
+abgr                a5b742253d41be91ee1726980c39fca5
+argb                deb903538e653598c119bfcee2aee651
+bgr0                bdf68c2709492abc5d05cbae373dee7d
+bgr24               e669ad05a12b8a5f853b599f38fa9725
+bgra                2353862686c14bbfe3a103aaec8ab6ed
+gbrap               4d19ce999a450498fd71c3764b2f9aa7
+gbrp                149393d82ef85a102f62a78aeccc86b6
+gray                b40cc829f4310691c717a8162c945fc2
+rgb0                e7b835d6ddfcfe87632604a81c2468e7
+rgb24               1b1b62bd26d83988f4ec51584ebea300
+rgba                f0bdbad87cace97926025917a010ffa2
+yuv410p             98b7126c5e00687e033da047ec44dc59
+yuv420p             c67848288b31393a1045efb335c533ce
+yuv444p             2d017f98f986810246068d2a3af3049b
+yuva420p            1ee052f2688c6caf9f08cc9ede7c133e
+yuva444p            dd7bccf95744e374295be468e882b96d
+yuvj420p            34aa87584344e7ba0dbe92f89d865de7
+yuvj444p            d1b2eb2ea933839a2cb587043a789313