Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5:
  lavfi: avfilter_merge_formats: handle case where inputs are same
  mpegvideo: Don't use ff_mspel_motion() for vc1
  imgconvert: avoid undefined left shift in avcodec_find_best_pix_fmt
  nuv: check RTjpeg header for validity
  vc1dec: add flush function for WMV9 and VC-1 decoders

Merged-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/imgconvert.c

@@ -890,7 +890,8 @@ static int avcodec_find_best_pix_fmt1(int64_t pix_fmt_mask,
     /* find exact color match with smallest size */
     dst_pix_fmt = -1;
     min_dist = 0x7fffffff;
-    for(i = 0;i < PIX_FMT_NB; i++) {
+    /* test only the first 64 pixel formats to avoid undefined behaviour */
+    for (i = 0; i < 64; i++) {
         if (pix_fmt_mask & (1ULL << i)) {
             loss = avcodec_get_pix_fmt_loss(i, src_pix_fmt, has_alpha) & loss_mask;
             if (loss == 0) {

libavcodec/mpegvideo_common.h

@@ -727,7 +727,7 @@ static av_always_inline void MPV_motion_internal(MpegEncContext *s,
                         0, 0, 0,
                         ref_picture, pix_op, qpix_op,
                         s->mv[dir][0][0], s->mv[dir][0][1], 16);
-        }else if(!is_mpeg12 && CONFIG_WMV2 && s->mspel){
+        }else if(!is_mpeg12 && CONFIG_WMV2 && s->mspel && s->codec_id == CODEC_ID_WMV2){
             ff_mspel_motion(s, dest_y, dest_cb, dest_cr,
                         ref_picture, pix_op,
                         s->mv[dir][0][0], s->mv[dir][0][1], 16);

libavcodec/nuv.c

@@ -182,17 +182,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     }
     if (c->codec_frameheader) {
         int w, h, q;
-        if (buf_size < 12) {
+        if (buf_size < RTJPEG_HEADER_SIZE || buf[4] != RTJPEG_HEADER_SIZE ||
+            buf[5] != RTJPEG_FILE_VERSION) {
             av_log(avctx, AV_LOG_ERROR, "invalid nuv video frame\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         w = AV_RL16(&buf[6]);
         h = AV_RL16(&buf[8]);
         q = buf[10];
         if (!codec_reinit(avctx, w, h, q))
             return -1;
-        buf = &buf[12];
-        buf_size -= 12;
+        buf = &buf[RTJPEG_HEADER_SIZE];
+        buf_size -= RTJPEG_HEADER_SIZE;
     }
 
     if (keyframe && c->pic.data[0])

libavcodec/rtjpeg.h

@@ -25,6 +25,9 @@
 #include <stdint.h>
 #include "dsputil.h"
 
+#define RTJPEG_FILE_VERSION 0
+#define RTJPEG_HEADER_SIZE 12
+
 typedef struct {
     int w, h;
     DSPContext *dsp;

libavcodec/vc1.c

@@ -4347,6 +4347,7 @@ AVCodec vc1_decoder = {
     vc1_decode_frame,
     CODEC_CAP_DELAY,
     NULL,
+    .flush          = ff_mpeg_flush,
     .long_name = NULL_IF_CONFIG_SMALL("SMPTE VC-1"),
     .pix_fmts = ff_pixfmt_list_420
 };
@@ -4362,6 +4363,7 @@ AVCodec wmv3_decoder = {
     vc1_decode_frame,
     CODEC_CAP_DELAY,
     NULL,
+    .flush          = ff_mpeg_flush,
     .long_name = NULL_IF_CONFIG_SMALL("Windows Media Video 9"),
     .pix_fmts = ff_pixfmt_list_420
 };

libavfilter/formats.c

@@ -43,6 +43,9 @@ AVFilterFormats *avfilter_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
     AVFilterFormats *ret;
     unsigned i, j, k = 0;
 
+    if (a == b)
+        return a;
+
     ret = av_mallocz(sizeof(AVFilterFormats));
 
     /* merge list of formats */