avcodec/cfhd: improve decompanding quality with reference implementation

libavcodec/cfhd.c

@@ -117,8 +117,8 @@ static inline int dequant_and_decompand(int level, int quantisation, int codeboo
 {
     if (codebook == 0 || codebook == 1) {
         int64_t abslevel = abs(level);
-        if (level < 264)
-            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) *
+        if (abslevel < 256)
+            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (256 * 256 * 256))) *
                FFSIGN(level) * quantisation;
         else
             return level * quantisation;

tests/ref/fate/cfhd-3

@@ -3,13 +3,13 @@
 #codec_id 0: rawvideo
 #dimensions 0: 496x241
 #sar 0: 0/1
-0,          0,          0,        1,   478144, 0x48a01dbb
-0,          1,          1,        1,   478144, 0x48a01dbb
-0,          2,          2,        1,   478144, 0x48a01dbb
-0,          3,          3,        1,   478144, 0xb978a72f
-0,          4,          4,        1,   478144, 0x7bbb4679
-0,          5,          5,        1,   478144, 0xc3fd3f59
-0,          6,          6,        1,   478144, 0xfd2a4816
-0,          7,          7,        1,   478144, 0x207f65d3
-0,          8,          8,        1,   478144, 0x207f65d3
-0,          9,          9,        1,   478144, 0x207f65d3
+0,          0,          0,        1,   478144, 0x1e5a0d6c
+0,          1,          1,        1,   478144, 0x1e5a0d6c
+0,          2,          2,        1,   478144, 0x1e5a0d6c
+0,          3,          3,        1,   478144, 0x88788c7d
+0,          4,          4,        1,   478144, 0x78643db8
+0,          5,          5,        1,   478144, 0x84303909
+0,          6,          6,        1,   478144, 0x8ddd4828
+0,          7,          7,        1,   478144, 0xc0845d58
+0,          8,          8,        1,   478144, 0xc0845d58
+0,          9,          9,        1,   478144, 0xc0845d58