Ensure that the palette is set in data[1] for all 8bit formats.

Also document it.

Originally committed as revision 17018 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/imgconvert.c

@@ -443,6 +443,43 @@ void avcodec_pix_fmt_string (char *buf, int buf_size, int pix_fmt)
     }
 }
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt){
+    int i;
+
+    for(i=0; i<256; i++){
+        int r,g,b;
+
+        switch(pix_fmt) {
+        case PIX_FMT_RGB8:
+            r= (i>>5    )*36;
+            g= ((i>>2)&7)*36;
+            b= (i&3     )*85;
+            break;
+        case PIX_FMT_BGR8:
+            b= (i>>6    )*85;
+            g= ((i>>3)&7)*36;
+            r= (i&7     )*36;
+            break;
+        case PIX_FMT_RGB4_BYTE:
+            r= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            b= (i&1     )*255;
+            break;
+        case PIX_FMT_BGR4_BYTE:
+            b= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            r= (i&1     )*255;
+            break;
+        case PIX_FMT_GRAY8:
+            r=b=g= i;
+            break;
+        }
+        pal[i] =  b + (g<<8) + (r<<16);
+    }
+
+    return 0;
+}
+
 int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
 {
     int w2;
@@ -505,13 +542,6 @@ int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
     case PIX_FMT_UYYVYY411:
         picture->linesize[0] = width + width/2;
         break;
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
-        picture->linesize[0] = width;
-        break;
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
         picture->linesize[0] = width / 2;
@@ -521,6 +551,11 @@ int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
         picture->linesize[0] = (width + 7) >> 3;
         break;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         picture->linesize[0] = width;
         picture->linesize[1] = 4;
         break;
@@ -588,11 +623,6 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt,
     case PIX_FMT_YUYV422:
     case PIX_FMT_UYVY422:
     case PIX_FMT_UYYVYY411:
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
     case PIX_FMT_MONOWHITE:
@@ -603,6 +633,11 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt,
         picture->data[3] = NULL;
         return size;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         size2 = (size + 3) & ~3;
         picture->data[0] = ptr;
         picture->data[1] = ptr + size2; /* palette is stored here as 256 32 bit words */
@@ -2000,6 +2035,9 @@ int avpicture_alloc(AVPicture *picture,
     if (!ptr)
         goto fail;
     avpicture_fill(picture, ptr, pix_fmt, width, height);
+    if(picture->data[1] && !picture->data[2])
+        ff_set_systematic_pal((uint32_t*)picture->data[1], pix_fmt);
+
     return 0;
  fail:
     memset(picture, 0, sizeof(AVPicture));

libavcodec/imgconvert.h

@@ -33,6 +33,8 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt, int height);
 
 int ff_get_plane_bytewidth(enum PixelFormat pix_fmt, int width, int plane);
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt);
+
 int img_convert(AVPicture *dst, int dst_pix_fmt, const AVPicture *src,
                 int src_pix_fmt, int src_width, int src_height);
 

libavcodec/utils.c

@@ -154,6 +154,8 @@ void avcodec_align_dimensions(AVCodecContext *s, int *width, int *height){
             h_align=4;
         }
     case PIX_FMT_PAL8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB8:
         if(s->codec_id == CODEC_ID_SMC){
             w_align=4;
             h_align=4;
@@ -281,12 +283,14 @@ int avcodec_default_get_buffer(AVCodecContext *s, AVFrame *pic){
             if(buf->base[i]==NULL) return -1;
             memset(buf->base[i], 128, size[i]);
 
-            // no edge if EDEG EMU or not planar YUV, we check for PAL8 redundantly to protect against a exploitable bug regression ...
-            if((s->flags&CODEC_FLAG_EMU_EDGE) || (s->pix_fmt == PIX_FMT_PAL8) || !size[2])
+            // no edge if EDEG EMU or not planar YUV
+            if((s->flags&CODEC_FLAG_EMU_EDGE) || !size[2])
                 buf->data[i] = buf->base[i];
             else
                 buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), stride_align[i]);
         }
+        if(size[1] && !size[2])
+            ff_set_systematic_pal((uint32_t*)buf->data[1], s->pix_fmt);
         buf->width  = s->width;
         buf->height = s->height;
         buf->pix_fmt= s->pix_fmt;

libavutil/avutil.h

@@ -76,6 +76,10 @@ unsigned avutil_version(void);
  * components stored in AVFrame.data[1] should be in the range 0..255.
  * This is important as many custom PAL8 video codecs that were designed
  * to run on the IBM VGA graphics adapter use 6-bit palette components.
+ *
+ * For all the 8bit per pixel formats, an RGB32 palette is in data[1] like
+ * for pal8. This palette is filled in automatically by the function
+ * allocating the picture.
  */
 enum PixelFormat {
     PIX_FMT_NONE= -1,

tests/libav.regression.ref

@@ -41,7 +41,7 @@ af195c31e8f49de61e3851ccde4c3ebd *./tests/data/b-pbmpipe.pbm
 ./tests/data/b-pbmpipe.pbm CRC=0x1ac46c70
 aff140ce80a1c86c1bf54118ad23da7b *./tests/data/b-pgmpipe.pgm
 2534775 ./tests/data/b-pgmpipe.pgm
-./tests/data/b-pgmpipe.pgm CRC=0xf485870f
+./tests/data/b-pgmpipe.pgm CRC=0x0e82c482
 94939357f0cb9502c474e3b017bd745a *./tests/data/b-ppmpipe.ppm
 7603575 ./tests/data/b-ppmpipe.ppm
 ./tests/data/b-ppmpipe.ppm CRC=0x80b9c1bc
@@ -50,7 +50,7 @@ b1450712a8dbb81602320d59e40ec3db *./tests/data/b-libav.gif
 b977a4fedff90a79baf70c8e02986820 *./tests/data/b-libav.y4m
 3801810 ./tests/data/b-libav.y4m
 0a6d74b54396884f117669965b57d3b5 *./tests/data/b-libav02.pgm
-./tests/data/b-libav%02d.pgm CRC=0x7e552eb1
+./tests/data/b-libav%02d.pgm CRC=0xc8032eb1
 101391 ./tests/data/b-libav02.pgm
 eb5b4ff1352f952234164ca15bd9d9e8 *./tests/data/b-libav02.ppm
 ./tests/data/b-libav%02d.ppm CRC=0x13da003d