j2k: redesign vert_causal_ctx_csty implementation

The old implementation was incomplete and could not have worked
This also fixes some warnings
New code is untested as i dont seem to have a sample file that uses this.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/j2k.c

@@ -103,20 +103,18 @@ static void tag_tree_zero(Jpeg2000TgtNode *t, int w, int h)
     }
 }
 
-static int getnbctxno(int flag, int bandno, int vert_causal_ctx_csty_symbol)
+static int getnbctxno(int flag, int bandno)
 {
     int h, v, d;
 
     h = ((flag & JPEG2000_T1_SIG_E) ? 1:0)+
         ((flag & JPEG2000_T1_SIG_W) ? 1:0);
     v = ((flag & JPEG2000_T1_SIG_N) ? 1:0);
-    if (!vert_causal_ctx_csty_symbol)
-         v = v + ((flag & JPEG2000_T1_SIG_S) ? 1:0);
+    v = v + ((flag & JPEG2000_T1_SIG_S) ? 1:0);
     d = ((flag & JPEG2000_T1_SIG_NE) ? 1:0)+
         ((flag & JPEG2000_T1_SIG_NW) ? 1:0);
-    if (!vert_causal_ctx_csty_symbol)
-        d = d + ((flag & JPEG2000_T1_SIG_SE) ? 1:0)+
-                ((flag & JPEG2000_T1_SIG_SW) ? 1:0);
+    d = d + ((flag & JPEG2000_T1_SIG_SE) ? 1:0)+
+            ((flag & JPEG2000_T1_SIG_SW) ? 1:0);
     if (bandno < 3){
             if (bandno == 1)
                 FFSWAP(int, h, v);
@@ -168,7 +166,7 @@ void ff_j2k_init_tier1_luts(void)
     int i, j;
     for (i = 0; i < 256; i++)
         for (j = 0; j < 4; j++)
-            ff_jpeg2000_sigctxno_lut[i][j] = getnbctxno(i, j, 0);
+            ff_jpeg2000_sigctxno_lut[i][j] = getnbctxno(i, j);
     for (i = 0; i < 16; i++)
         for (j = 0; j < 16; j++)
             ff_jpeg2000_sgnctxno_lut[i][j] = getsgnctxno(i + (j << 8), &ff_jpeg2000_xorbit_lut[i][j]);

libavcodec/j2k.h

@@ -213,7 +213,7 @@ void ff_j2k_set_significant(Jpeg2000T1Context *t1, int x, int y, int negative);
 
 extern uint8_t ff_jpeg2000_sigctxno_lut[256][4];
 
-static inline int ff_j2k_getnbctxno(int flag, int bandno, int vert_causal_ctx_csty_symbol)
+static inline int ff_j2k_getnbctxno(int flag, int bandno)
 {
     return ff_jpeg2000_sigctxno_lut[flag&255][bandno];
 }

libavcodec/j2kdec.c

@@ -628,9 +628,10 @@ static void decode_sigpass(Jpeg2000T1Context *t1, int width, int height, int bpn
             for (y = y0; y < height && y < y0+4; y++){
                 if ((t1->flags[y+1][x+1] & JPEG2000_T1_SIG_NB)
                 && !(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))){
-                    int vert_causal_ctx_csty_loc_symbol = vert_causal_ctx_csty_symbol && (x == 3 && y == 3);
-                    if (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno,
-                                      vert_causal_ctx_csty_loc_symbol))){
+                    int flags_mask = -1;
+                    if (vert_causal_ctx_csty_symbol && y == y0 + 3)
+                        flags_mask &= ~(JPEG2000_T1_SIG_S | JPEG2000_T1_SIG_SW | JPEG2000_T1_SIG_SE);
+                    if (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_j2k_getnbctxno(t1->flags[y+1][x+1] & flags_mask, bandno))){
                         int xorbit, ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
                         if (bpass_csty_symbol)
                              t1->data[y][x] = ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ? -mask : mask;
@@ -666,7 +667,7 @@ static void decode_refpass(Jpeg2000T1Context *t1, int width, int height, int bpn
 }
 
 static void decode_clnpass(Jpeg2000DecoderContext *s, Jpeg2000T1Context *t1, int width, int height,
-                           int bpno, int bandno, int seg_symbols)
+                           int bpno, int bandno, int seg_symbols, int vert_causal_ctx_csty_symbol)
 {
     int mask = 3 << (bpno - 1), y0, x, y, runlen, dec;
 
@@ -689,9 +690,13 @@ static void decode_clnpass(Jpeg2000DecoderContext *s, Jpeg2000T1Context *t1, int
 
             for (y = y0 + runlen; y < y0 + 4 && y < height; y++){
                 if (!dec){
-                    if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS)))
-                        dec = ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_j2k_getnbctxno(t1->flags[y+1][x+1],
-                                                                                             bandno, 0));
+                    if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))) {
+                        int flags_mask = -1;
+                        if (vert_causal_ctx_csty_symbol && y == y0 + 3)
+                            flags_mask &= ~(JPEG2000_T1_SIG_S | JPEG2000_T1_SIG_SW | JPEG2000_T1_SIG_SE);
+                        dec = ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_j2k_getnbctxno(t1->flags[y+1][x+1] & flags_mask,
+                                                                                             bandno));
+                    }
                 }
                 if (dec){
                     int xorbit, ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
@@ -742,7 +747,7 @@ static int decode_cblk(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *codsty, J
                         ff_mqc_initdec(&t1->mqc, cblk->data);
                     break;
             case 2: decode_clnpass(s, t1, width, height, bpno+1, bandpos,
-                                   codsty->cblk_style & JPEG2000_CBLK_SEGSYM);
+                                   codsty->cblk_style & JPEG2000_CBLK_SEGSYM, vert_causal_ctx_csty_symbol);
                     clnpass_cnt = clnpass_cnt + 1;
                     if (bpass_csty_symbol && clnpass_cnt >= 4)
                        ff_mqc_initdec(&t1->mqc, cblk->data);

libavcodec/j2kenc.c

@@ -476,12 +476,11 @@ static int getnmsedec_ref(int x, int bpno)
 static void encode_sigpass(Jpeg2000T1Context *t1, int width, int height, int bandno, int *nmsedec, int bpno)
 {
     int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
-    int vert_causal_ctx_csty_loc_symbol;
     for (y0 = 0; y0 < height; y0 += 4)
         for (x = 0; x < width; x++)
             for (y = y0; y < height && y < y0+4; y++){
                 if (!(t1->flags[y+1][x+1] & JPEG2000_T1_SIG) && (t1->flags[y+1][x+1] & JPEG2000_T1_SIG_NB)){
-                    int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno, vert_causal_ctx_csty_loc_symbol),
+                    int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno),
                         bit = t1->data[y][x] & mask ? 1 : 0;
                     ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, bit);
                     if (bit){
@@ -513,7 +512,6 @@ static void encode_refpass(Jpeg2000T1Context *t1, int width, int height, int *nm
 static void encode_clnpass(Jpeg2000T1Context *t1, int width, int height, int bandno, int *nmsedec, int bpno)
 {
     int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
-    int vert_causal_ctx_csty_loc_symbol;
     for (y0 = 0; y0 < height; y0 += 4)
         for (x = 0; x < width; x++){
             if (y0 + 3 < height && !(
@@ -534,7 +532,7 @@ static void encode_clnpass(Jpeg2000T1Context *t1, int width, int height, int ban
                 ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + MQC_CX_UNI, rlen & 1);
                 for (y = y0 + rlen; y < y0 + 4; y++){
                     if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))){
-                        int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno, vert_causal_ctx_csty_loc_symbol);
+                        int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno);
                         if (y > y0 + rlen)
                             ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, t1->data[y][x] & mask ? 1:0);
                         if (t1->data[y][x] & mask){ // newly significant
@@ -550,7 +548,7 @@ static void encode_clnpass(Jpeg2000T1Context *t1, int width, int height, int ban
             } else{
                 for (y = y0; y < y0 + 4 && y < height; y++){
                     if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))){
-                        int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno, vert_causal_ctx_csty_loc_symbol);
+                        int ctxno = ff_j2k_getnbctxno(t1->flags[y+1][x+1], bandno);
                         ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, t1->data[y][x] & mask ? 1:0);
                         if (t1->data[y][x] & mask){ // newly significant
                             int xorbit;