Add pixel formats for 9- and 10-bit yuv420p.

Also add support for these formats in libswscale.

Needed for high bit depth h264 decoding.

Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>

libavcodec/utils.c

@@ -136,6 +136,10 @@ void avcodec_align_dimensions2(AVCodecContext *s, int *width, int *height, int l
     case PIX_FMT_YUVJ440P:
     case PIX_FMT_YUVJ444P:
     case PIX_FMT_YUVA420P:
+    case PIX_FMT_YUV420P9LE:
+    case PIX_FMT_YUV420P9BE:
+    case PIX_FMT_YUV420P10LE:
+    case PIX_FMT_YUV420P10BE:
         w_align= 16; //FIXME check for non mpeg style codecs and use less alignment
         h_align= 16;
         if(s->codec_id == CODEC_ID_MPEG2VIDEO || s->codec_id == CODEC_ID_MJPEG || s->codec_id == CODEC_ID_AMV || s->codec_id == CODEC_ID_THP || s->codec_id == CODEC_ID_H264)

libavutil/pixdesc.c

@@ -740,6 +740,52 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
         .log2_chroma_h = 1,
         .flags = PIX_FMT_HWACCEL,
     },
+    [PIX_FMT_YUV420P9LE] = {
+        .name = "yuv420p9le",
+        .nb_components= 3,
+        .log2_chroma_w= 1,
+        .log2_chroma_h= 1,
+        .comp = {
+            {0,1,1,0,8},        /* Y */
+            {1,1,1,0,8},        /* U */
+            {2,1,1,0,8},        /* V */
+        },
+    },
+    [PIX_FMT_YUV420P9BE] = {
+        .name = "yuv420p9be",
+        .nb_components= 3,
+        .log2_chroma_w= 1,
+        .log2_chroma_h= 1,
+        .comp = {
+            {0,1,1,0,8},        /* Y */
+            {1,1,1,0,8},        /* U */
+            {2,1,1,0,8},        /* V */
+        },
+        .flags = PIX_FMT_BE,
+    },
+    [PIX_FMT_YUV420P10LE] = {
+        .name = "yuv420p10le",
+        .nb_components= 3,
+        .log2_chroma_w= 1,
+        .log2_chroma_h= 1,
+        .comp = {
+            {0,1,1,0,9},        /* Y */
+            {1,1,1,0,9},        /* U */
+            {2,1,1,0,9},        /* V */
+        },
+    },
+    [PIX_FMT_YUV420P10BE] = {
+        .name = "yuv420p10be",
+        .nb_components= 3,
+        .log2_chroma_w= 1,
+        .log2_chroma_h= 1,
+        .comp = {
+            {0,1,1,0,9},        /* Y */
+            {1,1,1,0,9},        /* U */
+            {2,1,1,0,9},        /* V */
+        },
+        .flags = PIX_FMT_BE,
+    },
     [PIX_FMT_YUV420P16LE] = {
         .name = "yuv420p16le",
         .nb_components= 3,

libavutil/pixfmt.h

@@ -135,6 +135,10 @@ enum PixelFormat {
     PIX_FMT_Y400A,     ///< 8bit gray, 8bit alpha
     PIX_FMT_BGR48BE,   ///< packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as big-endian
     PIX_FMT_BGR48LE,   ///< packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as little-endian
+    PIX_FMT_YUV420P9BE, ///< planar YUV 4:2:0, 13.5bpp, (1 Cr & Cb sample per 2x2 Y samples), big-endian
+    PIX_FMT_YUV420P9LE, ///< planar YUV 4:2:0, 13.5bpp, (1 Cr & Cb sample per 2x2 Y samples), little-endian
+    PIX_FMT_YUV420P10BE,///< planar YUV 4:2:0, 15bpp, (1 Cr & Cb sample per 2x2 Y samples), big-endian
+    PIX_FMT_YUV420P10LE,///< planar YUV 4:2:0, 15bpp, (1 Cr & Cb sample per 2x2 Y samples), little-endian
     PIX_FMT_NB,        ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions
 };
 
@@ -159,6 +163,8 @@ enum PixelFormat {
 #define PIX_FMT_BGR555 PIX_FMT_NE(BGR555BE, BGR555LE)
 #define PIX_FMT_BGR444 PIX_FMT_NE(BGR444BE, BGR444LE)
 
+#define PIX_FMT_YUV420P9  PIX_FMT_NE(YUV420P9BE , YUV420P9LE)
+#define PIX_FMT_YUV420P10 PIX_FMT_NE(YUV420P10BE, YUV420P10LE)
 #define PIX_FMT_YUV420P16 PIX_FMT_NE(YUV420P16BE, YUV420P16LE)
 #define PIX_FMT_YUV422P16 PIX_FMT_NE(YUV422P16BE, YUV422P16LE)
 #define PIX_FMT_YUV444P16 PIX_FMT_NE(YUV444P16BE, YUV444P16LE)

libswscale/swscale.c

@@ -1669,7 +1669,28 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t* src[], int srcStride[
                 length*=2;
             fillPlane(dst[plane], dstStride[plane], length, height, y, (plane==3) ? 255 : 128);
         } else {
-            if(is16BPS(c->srcFormat) && !is16BPS(c->dstFormat)) {
+            if(isNBPS(c->srcFormat)) {
+                const int depth = av_pix_fmt_descriptors[c->srcFormat].comp[plane].depth_minus1+1;
+                uint16_t *srcPtr2 = (uint16_t*)srcPtr;
+
+                if (is16BPS(c->dstFormat)) {
+                    uint16_t *dstPtr2 = (uint16_t*)dstPtr;
+                    for (i = 0; i < height; i++) {
+                        for (j = 0; j < length; j++)
+                            dstPtr2[j] = (srcPtr2[j]<<(16-depth)) | (srcPtr2[j]>>(2*depth-16));
+                        dstPtr2 += dstStride[plane]/2;
+                        srcPtr2 += srcStride[plane]/2;
+                    }
+                } else {
+                    // FIXME Maybe dither instead.
+                    for (i = 0; i < height; i++) {
+                        for (j = 0; j < length; j++)
+                            dstPtr[j] = srcPtr2[j]>>(depth-8);
+                        dstPtr  += dstStride[plane];
+                        srcPtr2 += srcStride[plane]/2;
+                    }
+                }
+            } else if(is16BPS(c->srcFormat) && !is16BPS(c->dstFormat)) {
                 if (!isBE(c->srcFormat)) srcPtr++;
                 for (i=0; i<height; i++) {
                     for (j=0; j<length; j++) dstPtr[j] = srcPtr[j<<1];

libswscale/swscale_internal.h

@@ -354,6 +354,12 @@ const char *sws_format_name(enum PixelFormat format);
         || (x)==PIX_FMT_YUV422P16BE \
         || (x)==PIX_FMT_YUV444P16BE \
     )
+#define isNBPS(x)       (           \
+           (x)==PIX_FMT_YUV420P9LE  \
+        || (x)==PIX_FMT_YUV420P9BE  \
+        || (x)==PIX_FMT_YUV420P10LE \
+        || (x)==PIX_FMT_YUV420P10BE \
+    )
 #define isBE(x) ((x)&1)
 #define isPlanar8YUV(x) (           \
            (x)==PIX_FMT_YUV410P     \
@@ -368,9 +374,13 @@ const char *sws_format_name(enum PixelFormat format);
     )
 #define isPlanarYUV(x)  (           \
         isPlanar8YUV(x)             \
+        || (x)==PIX_FMT_YUV420P9LE  \
+        || (x)==PIX_FMT_YUV420P10LE \
         || (x)==PIX_FMT_YUV420P16LE \
         || (x)==PIX_FMT_YUV422P16LE \
         || (x)==PIX_FMT_YUV444P16LE \
+        || (x)==PIX_FMT_YUV420P9BE  \
+        || (x)==PIX_FMT_YUV420P10BE \
         || (x)==PIX_FMT_YUV420P16BE \
         || (x)==PIX_FMT_YUV422P16BE \
         || (x)==PIX_FMT_YUV444P16BE \

libswscale/swscale_template.c

@@ -225,6 +225,32 @@ static inline void nv21ToUV_c(uint8_t *dstU, uint8_t *dstV,
     nvXXtoUV_c(dstV, dstU, src1, width);
 }
 
+// FIXME Maybe dither instead.
+#define YUV_NBPS(depth) \
+static inline void yuv ## depth ## ToUV_c(uint8_t *dstU, uint8_t *dstV, \
+                                          const uint8_t *_srcU, const uint8_t *_srcV, \
+                                          long width, uint32_t *unused) \
+{ \
+    int i; \
+    const uint16_t *srcU = (const uint16_t*)_srcU; \
+    const uint16_t *srcV = (const uint16_t*)_srcV; \
+    for (i = 0; i < width; i++) { \
+        dstU[i] = srcU[i]>>(depth-8); \
+        dstV[i] = srcV[i]>>(depth-8); \
+    } \
+} \
+\
+static inline void yuv ## depth ## ToY_c(uint8_t *dstY, const uint8_t *_srcY, long width, uint32_t *unused) \
+{ \
+    int i; \
+    const uint16_t *srcY = (const uint16_t*)_srcY; \
+    for (i = 0; i < width; i++) \
+        dstY[i] = srcY[i]>>(depth-8); \
+} \
+
+YUV_NBPS( 9)
+YUV_NBPS(10)
+
 static inline void bgr24ToY_c(uint8_t *dst, const uint8_t *src,
                               long width, uint32_t *unused)
 {
@@ -790,6 +816,10 @@ static void sws_init_swScale_c(SwsContext *c)
         case PIX_FMT_PAL8     :
         case PIX_FMT_BGR4_BYTE:
         case PIX_FMT_RGB4_BYTE: c->chrToYV12 = palToUV; break;
+        case PIX_FMT_YUV420P9BE:
+        case PIX_FMT_YUV420P9LE: c->chrToYV12 = yuv9ToUV_c; break;
+        case PIX_FMT_YUV420P10BE:
+        case PIX_FMT_YUV420P10LE: c->chrToYV12 = yuv10ToUV_c; break;
         case PIX_FMT_YUV420P16BE:
         case PIX_FMT_YUV422P16BE:
         case PIX_FMT_YUV444P16BE: c->chrToYV12 = BEToUV_c; break;
@@ -836,6 +866,10 @@ static void sws_init_swScale_c(SwsContext *c)
     c->lumToYV12 = NULL;
     c->alpToYV12 = NULL;
     switch (srcFormat) {
+    case PIX_FMT_YUV420P9BE:
+    case PIX_FMT_YUV420P9LE: c->lumToYV12 = yuv9ToY_c; break;
+    case PIX_FMT_YUV420P10BE:
+    case PIX_FMT_YUV420P10LE: c->lumToYV12 = yuv10ToY_c; break;
     case PIX_FMT_YUYV422  :
     case PIX_FMT_YUV420P16BE:
     case PIX_FMT_YUV422P16BE:

libswscale/utils.c

@@ -106,9 +106,13 @@ const char *swscale_license(void)
         || (x)==PIX_FMT_YUV440P     \
         || (x)==PIX_FMT_MONOWHITE   \
         || (x)==PIX_FMT_MONOBLACK   \
+        || (x)==PIX_FMT_YUV420P9LE    \
+        || (x)==PIX_FMT_YUV420P10LE   \
         || (x)==PIX_FMT_YUV420P16LE   \
         || (x)==PIX_FMT_YUV422P16LE   \
         || (x)==PIX_FMT_YUV444P16LE   \
+        || (x)==PIX_FMT_YUV420P9BE    \
+        || (x)==PIX_FMT_YUV420P10BE   \
         || (x)==PIX_FMT_YUV420P16BE   \
         || (x)==PIX_FMT_YUV422P16BE   \
         || (x)==PIX_FMT_YUV444P16BE   \