avcodec/jpeg2000dwt: Fix value of F_LFTG_X and I_LFTG_X

Even if the jpeg2000 spec uses a wrong value this does not
make mathematics work this way, also this has been corrected in the 2004
version AFAIK

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/jpeg2000dwt.c

@@ -38,10 +38,7 @@
 #define F_LFTG_GAMMA  0.882911075530934f
 #define F_LFTG_DELTA  0.443506852043971f
 #define F_LFTG_K      1.230174104914001f
-#define F_LFTG_X      1.625732422f
-/* FIXME: Why use 1.625732422 instead of 1/F_LFTG_K?
- * Incorrect value in JPEG2000 norm.
- * see (ISO/IEC 15444:1 (version 2002) F.3.8.2 */
+#define F_LFTG_X      1.625786132231922f
 
 /* Lifting parameters in integer format.
  * Computed as param = (float param) * (1 << 16) */
@@ -50,7 +47,7 @@
 #define I_LFTG_GAMMA   57862
 #define I_LFTG_DELTA   29066
 #define I_LFTG_K       80621
-#define I_LFTG_X      106544
+#define I_LFTG_X      106548
 
 static inline void extend53(int *p, int i0, int i1)
 {

tests/ref/fate/j2k-dwt

@@ -1,60 +1,60 @@
 5/3i, decomp:15 border 151 170 140 183 milli-err2:        0
-9/7i, decomp:15 border 151 170 140 183 milli-err2:    13030
-9/7f, decomp:15 border 151 170 140 183 err2:              0.2626
+9/7i, decomp:15 border 151 170 140 183 milli-err2:    13160
+9/7f, decomp:15 border 151 170 140 183 err2:              0.0001
 5/3i, decomp:21 border 173 201  81 189 milli-err2:        0
-9/7i, decomp:21 border 173 201  81 189 milli-err2:    21587
-9/7f, decomp:21 border 173 201  81 189 err2:              0.3460
+9/7i, decomp:21 border 173 201  81 189 milli-err2:    24635
+9/7f, decomp:21 border 173 201  81 189 err2:              0.0001
 5/3i, decomp:22 border 213 227  76 245 milli-err2:        0
-9/7i, decomp:22 border 213 227  76 245 milli-err2:    13454
-9/7f, decomp:22 border 213 227  76 245 err2:              0.2393
+9/7i, decomp:22 border 213 227  76 245 milli-err2:    15253
+9/7f, decomp:22 border 213 227  76 245 err2:              0.0001
 5/3i, decomp:13 border 134 157 184 203 milli-err2:        0
-9/7i, decomp:13 border 134 157 184 203 milli-err2:    19745
-9/7f, decomp:13 border 134 157 184 203 err2:              0.1715
+9/7i, decomp:13 border 134 157 184 203 milli-err2:    21846
+9/7f, decomp:13 border 134 157 184 203 err2:              0.0001
 5/3i, decomp: 1 border 204 237   6 106 milli-err2:        0
-9/7i, decomp: 1 border 204 237   6 106 milli-err2:     2236
-9/7f, decomp: 1 border 204 237   6 106 err2:              0.0063
+9/7i, decomp: 1 border 204 237   6 106 milli-err2:     2317
+9/7f, decomp: 1 border 204 237   6 106 err2:              0.0000
 5/3i, decomp:28 border  76 211  13 210 milli-err2:        0
-9/7i, decomp:28 border  76 211  13 210 milli-err2:    22341
-9/7f, decomp:28 border  76 211  13 210 err2:              0.3931
+9/7i, decomp:28 border  76 211  13 210 milli-err2:    21360
+9/7f, decomp:28 border  76 211  13 210 err2:              0.0002
 5/3i, decomp:21 border  76  99  43 123 milli-err2:        0
-9/7i, decomp:21 border  76  99  43 123 milli-err2:    15450
-9/7f, decomp:21 border  76  99  43 123 err2:              0.2005
+9/7i, decomp:21 border  76  99  43 123 milli-err2:    13513
+9/7f, decomp:21 border  76  99  43 123 err2:              0.0001
 5/3i, decomp:15 border 192 243 174 204 milli-err2:        0
-9/7i, decomp:15 border 192 243 174 204 milli-err2:    56308
-9/7f, decomp:15 border 192 243 174 204 err2:              0.3373
+9/7i, decomp:15 border 192 243 174 204 milli-err2:    13302
+9/7f, decomp:15 border 192 243 174 204 err2:              0.0001
 5/3i, decomp:21 border  17  68  93 204 milli-err2:        0
-9/7i, decomp:21 border  17  68  93 204 milli-err2:    26234
-9/7f, decomp:21 border  17  68  93 204 err2:              0.2765
+9/7i, decomp:21 border  17  68  93 204 milli-err2:    22975
+9/7f, decomp:21 border  17  68  93 204 err2:              0.0001
 5/3i, decomp:11 border 142 168  82 174 milli-err2:        0
-9/7i, decomp:11 border 142 168  82 174 milli-err2:    14802
-9/7f, decomp:11 border 142 168  82 174 err2:              0.2064
+9/7i, decomp:11 border 142 168  82 174 milli-err2:    11936
+9/7f, decomp:11 border 142 168  82 174 err2:              0.0001
 5/3i, decomp:23 border 142 209 171 235 milli-err2:        0
-9/7i, decomp:23 border 142 209 171 235 milli-err2:    26230
-9/7f, decomp:23 border 142 209 171 235 err2:              0.3092
+9/7i, decomp:23 border 142 209 171 235 milli-err2:    19428
+9/7f, decomp:23 border 142 209 171 235 err2:              0.0001
 5/3i, decomp:30 border  37 185  79 245 milli-err2:        0
-9/7i, decomp:30 border  37 185  79 245 milli-err2:    18994
-9/7f, decomp:30 border  37 185  79 245 err2:              0.3921
+9/7i, decomp:30 border  37 185  79 245 milli-err2:    22981
+9/7f, decomp:30 border  37 185  79 245 err2:              0.0002
 5/3i, decomp: 5 border 129 236  30 243 milli-err2:        0
-9/7i, decomp: 5 border 129 236  30 243 milli-err2:    14600
-9/7f, decomp: 5 border 129 236  30 243 err2:              0.1252
+9/7i, decomp: 5 border 129 236  30 243 milli-err2:    16632
+9/7f, decomp: 5 border 129 236  30 243 err2:              0.0001
 5/3i, decomp:10 border   5 160 146 247 milli-err2:        0
-9/7i, decomp:10 border   5 160 146 247 milli-err2:    43905
-9/7f, decomp:10 border   5 160 146 247 err2:              0.2753
+9/7i, decomp:10 border   5 160 146 247 milli-err2:    39669
+9/7f, decomp:10 border   5 160 146 247 err2:              0.0002
 5/3i, decomp: 5 border 104 162   6  47 milli-err2:        0
-9/7i, decomp: 5 border 104 162   6  47 milli-err2:    12942
-9/7f, decomp: 5 border 104 162   6  47 err2:              0.1237
+9/7i, decomp: 5 border 104 162   6  47 milli-err2:    18290
+9/7f, decomp: 5 border 104 162   6  47 err2:              0.0001
 5/3i, decomp:24 border  78 250 102 218 milli-err2:        0
-9/7i, decomp:24 border  78 250 102 218 milli-err2:    24997
-9/7f, decomp:24 border  78 250 102 218 err2:              0.3907
+9/7i, decomp:24 border  78 250 102 218 milli-err2:    36623
+9/7f, decomp:24 border  78 250 102 218 err2:              0.0002
 5/3i, decomp:28 border  86  98  56  79 milli-err2:        0
-9/7i, decomp:28 border  86  98  56  79 milli-err2:     8112
-9/7f, decomp:28 border  86  98  56  79 err2:              0.3094
+9/7i, decomp:28 border  86  98  56  79 milli-err2:     9891
+9/7f, decomp:28 border  86  98  56  79 err2:              0.0001
 5/3i, decomp: 6 border  95 238 197 214 milli-err2:        0
-9/7i, decomp: 6 border  95 238 197 214 milli-err2:    12925
-9/7f, decomp: 6 border  95 238 197 214 err2:              0.1481
+9/7i, decomp: 6 border  95 238 197 214 milli-err2:    17172
+9/7f, decomp: 6 border  95 238 197 214 err2:              0.0001
 5/3i, decomp:17 border  77 169  93 165 milli-err2:        0
-9/7i, decomp:17 border  77 169  93 165 milli-err2:    24949
-9/7f, decomp:17 border  77 169  93 165 err2:              0.4816
+9/7i, decomp:17 border  77 169  93 165 milli-err2:    23573
+9/7f, decomp:17 border  77 169  93 165 err2:              0.0001
 5/3i, decomp:22 border 178 187   7 119 milli-err2:        0
-9/7i, decomp:22 border 178 187   7 119 milli-err2:     9838
-9/7f, decomp:22 border 178 187   7 119 err2:              0.0999
+9/7i, decomp:22 border 178 187   7 119 milli-err2:    11305
+9/7f, decomp:22 border 178 187   7 119 err2:              0.0000

tests/ref/fate/jpeg2000-dcinema

@@ -1,3 +1,3 @@
 #tb 0: 1/24
-0,          0,          0,        1, 12441600, 0xf0de508b
-0,          1,          1,        1, 12441600, 0x8e50c249
+0,          0,          0,        1, 12441600, 0x35b017fd
+0,          1,          1,        1, 12441600, 0xe94551b9

tests/ref/vsynth/vsynth1-jpeg2000-97

@@ -1,4 +1,4 @@
-0bbcba25aaf987d1719a12c3f6e5a333 *tests/data/fate/vsynth1-jpeg2000-97.avi
-2188416 tests/data/fate/vsynth1-jpeg2000-97.avi
-9dd595c219b4ab5219a96b08cd18851f *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
-stddev:    6.19 PSNR: 32.30 MAXDIFF:   57 bytes:  7603200/  7603200
+4cac63aaf880ab0ee199ac44e520640a *tests/data/fate/vsynth1-jpeg2000-97.avi
+2188956 tests/data/fate/vsynth1-jpeg2000-97.avi
+d9227e7a7413cb48362654d879df985a *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
+stddev:    6.18 PSNR: 32.30 MAXDIFF:   57 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth2-jpeg2000-97

@@ -1,4 +1,4 @@
-1894f3c438f817b6101a444e2af430d3 *tests/data/fate/vsynth2-jpeg2000-97.avi
-1448224 tests/data/fate/vsynth2-jpeg2000-97.avi
-c95f2c7138ec20a6f388d1d945cf0cf4 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
-stddev:    5.35 PSNR: 33.56 MAXDIFF:   55 bytes:  7603200/  7603200
+352d3dc81b5c78638a2e7b575ce468c2 *tests/data/fate/vsynth2-jpeg2000-97.avi
+1448342 tests/data/fate/vsynth2-jpeg2000-97.avi
+d7b18b6dcac0139858e64e00073db1c6 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
+stddev:    5.35 PSNR: 33.56 MAXDIFF:   56 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth3-jpeg2000-97

@@ -1,4 +1,4 @@
-324eac3241a5580725ba817a9d60f3f3 *tests/data/fate/vsynth3-jpeg2000-97.avi
-66568 tests/data/fate/vsynth3-jpeg2000-97.avi
-eb32ff15b5a23fc29cedd43430e1d46a *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
-stddev:    6.15 PSNR: 32.35 MAXDIFF:   47 bytes:    86700/    86700
+36dddf487c4919d890db3484ab5d6486 *tests/data/fate/vsynth3-jpeg2000-97.avi
+66522 tests/data/fate/vsynth3-jpeg2000-97.avi
+332c0924bc1477a41eb413455c5b9d90 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
+stddev:    6.17 PSNR: 32.32 MAXDIFF:   51 bytes:    86700/    86700

tests/ref/vsynth/vsynth_lena-jpeg2000-97

@@ -1,4 +1,4 @@
-aebe3eaa1fa5c6698cd1abc8d0569d7f *tests/data/fate/vsynth_lena-jpeg2000-97.avi
-1105382 tests/data/fate/vsynth_lena-jpeg2000-97.avi
-32069295493de2b3e348ecbed2ff41ab *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
-stddev:    4.85 PSNR: 34.41 MAXDIFF:   58 bytes:  7603200/  7603200
+a39e052e7d109e33714329800c9df90f *tests/data/fate/vsynth_lena-jpeg2000-97.avi
+1105046 tests/data/fate/vsynth_lena-jpeg2000-97.avi
+c151476d2ed807575122064ada37d5ef *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
+stddev:    4.85 PSNR: 34.41 MAXDIFF:   59 bytes:  7603200/  7603200