jpeg2000dwt: remove floats from mixed float/int 9/7 dwt

This should fix some fate failures

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/jpeg2000dwt.c

@@ -145,31 +145,31 @@ static void dwt_encode53(DWTContext *s, int *t)
     }
 }
 
-static void sd_1d97(float *p, int i0, int i1)
+static void sd_1d97_int(int *p, int i0, int i1)
 {
     int i;
 
     if (i1 == i0 + 1)
         return;
 
-    extend97_float(p, i0, i1);
+    extend97_int(p, i0, i1);
     i0++; i1++;
 
     for (i = i0/2 - 2; i < i1/2 + 1; i++)
-        p[2*i+1] -= 1.586134 * (p[2*i] + p[2*i+2]);
+        p[2 * i + 1] -= (I_LFTG_ALPHA * (p[2 * i]     + p[2 * i + 2]) + (1 << 15)) >> 16;
     for (i = i0/2 - 1; i < i1/2 + 1; i++)
-        p[2*i] -= 0.052980 * (p[2*i-1] + p[2*i+1]);
+        p[2 * i]     -= (I_LFTG_BETA  * (p[2 * i - 1] + p[2 * i + 1]) + (1 << 15)) >> 16;
     for (i = i0/2 - 1; i < i1/2; i++)
-        p[2*i+1] += 0.882911 * (p[2*i] + p[2*i+2]);
+        p[2 * i + 1] += (I_LFTG_GAMMA * (p[2 * i]     + p[2 * i + 2]) + (1 << 15)) >> 16;
     for (i = i0/2; i < i1/2; i++)
-        p[2*i] += 0.443506 * (p[2*i-1] + p[2*i+1]);
+        p[2 * i]     += (I_LFTG_DELTA * (p[2 * i - 1] + p[2 * i + 1]) + (1 << 15)) >> 16;
 }
 
 static void dwt_encode97_int(DWTContext *s, int *t)
 {
     int lev,
         w = s->linelen[s->ndeclevels-1][0];
-    float *line = s->i_linebuf;
+    int *line = s->i_linebuf;
     line += 5;
 
     for (lev = s->ndeclevels-1; lev >= 0; lev--){
@@ -178,7 +178,7 @@ static void dwt_encode97_int(DWTContext *s, int *t)
             mh = s->mod[lev][0],
             mv = s->mod[lev][1],
             lp;
-        float *l;
+        int *l;
 
         // HOR_SD
         l = line + mh;
@@ -188,13 +188,13 @@ static void dwt_encode97_int(DWTContext *s, int *t)
             for (i = 0; i < lh; i++)
                 l[i] = t[w*lp + i];
 
-            sd_1d97(line, mh, mh + lh);
+            sd_1d97_int(line, mh, mh + lh);
 
             // copy back and deinterleave
             for (i =   mh; i < lh; i+=2, j++)
-                t[w*lp + j] = F_LFTG_X * l[i] / 2;
+                t[w*lp + j] = ((l[i] * I_LFTG_X) + (1 << 16)) >> 17;
             for (i = 1-mh; i < lh; i+=2, j++)
-                t[w*lp + j] = F_LFTG_K * l[i] / 2;
+                t[w*lp + j] = ((l[i] * I_LFTG_K) + (1 << 16)) >> 17;
         }
 
         // VER_SD
@@ -205,13 +205,13 @@ static void dwt_encode97_int(DWTContext *s, int *t)
             for (i = 0; i < lv; i++)
                 l[i] = t[w*i + lp];
 
-            sd_1d97(line, mv, mv + lv);
+            sd_1d97_int(line, mv, mv + lv);
 
             // copy back and deinterleave
             for (i =   mv; i < lv; i+=2, j++)
-                t[w*j + lp] = F_LFTG_X * l[i] / 2;
+                t[w*j + lp] = ((l[i] * I_LFTG_X) + (1 << 16)) >> 17;
             for (i = 1-mv; i < lv; i+=2, j++)
-                t[w*j + lp] = F_LFTG_K * l[i] / 2;
+                t[w*j + lp] = ((l[i] * I_LFTG_K) + (1 << 16)) >> 17;
         }
     }
 }

tests/ref/vsynth/vsynth1-j2k-97

@@ -1,4 +1,4 @@
-1222a8f36e34ce161a04afc1a0be37ec *tests/data/fate/vsynth1-j2k-97.avi
-2034226 tests/data/fate/vsynth1-j2k-97.avi
-42bb661ff1927fd6691ed99a755df5ea *tests/data/fate/vsynth1-j2k-97.out.rawvideo
-stddev:    7.58 PSNR: 30.53 MAXDIFF:  104 bytes:  7603200/  7603200
+c135eb14e9f219242180270c2a242634 *tests/data/fate/vsynth1-j2k-97.avi
+2243132 tests/data/fate/vsynth1-j2k-97.avi
+c8dc404072bf57d3ca823c0adaac013c *tests/data/fate/vsynth1-j2k-97.out.rawvideo
+stddev:    6.79 PSNR: 31.49 MAXDIFF:   77 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth2-j2k-97

@@ -1,4 +1,4 @@
-82b35e3f587babe909da920597fcf4a8 *tests/data/fate/vsynth2-j2k-97.avi
-1004984 tests/data/fate/vsynth2-j2k-97.avi
-3d29c50c8f2ae433c39cb46855fe8a34 *tests/data/fate/vsynth2-j2k-97.out.rawvideo
-stddev:    6.57 PSNR: 31.77 MAXDIFF:   61 bytes:  7603200/  7603200
+3ac3e49a89136bddde9e44bac3e5b4ed *tests/data/fate/vsynth2-j2k-97.avi
+1118952 tests/data/fate/vsynth2-j2k-97.avi
+a9434a726d6976facf53ff63d1466a99 *tests/data/fate/vsynth2-j2k-97.out.rawvideo
+stddev:    5.81 PSNR: 32.84 MAXDIFF:   59 bytes:  7603200/  7603200