h264: improve parsing of broken AVC SPS

Parsing the entire NAL as SPS fixes decoding of some AVC bitstreams
with broken escaping. Since the size of the NAL unit is known and
checked against the buffer end we can parse it entirely without buffer
overreads.

Fixes playback of
http://streams.videolan.org/streams/mp4/Mr_MrsSmith-h264_aac.mp4

Signed-off-by: Janne Grunau <janne-libav@jannau.net>

libavcodec/h264.c

@@ -3925,7 +3925,14 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
             break;
         case NAL_SPS:
             init_get_bits(&s->gb, ptr, bit_length);
-            ff_h264_decode_seq_parameter_set(h);
+            if (ff_h264_decode_seq_parameter_set(h) < 0 &&
+                h->is_avc && (nalsize != consumed) && nalsize) {
+                av_log(h->s.avctx, AV_LOG_DEBUG, "SPS decoding failure, "
+                       "try parsing the coomplete NAL\n");
+                init_get_bits(&s->gb, buf + buf_index + 1 - consumed,
+                              8 * (nalsize - 1));
+                ff_h264_decode_seq_parameter_set(h);
+            }
 
             if (s->flags& CODEC_FLAG_LOW_DELAY ||
                 (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames))

libavcodec/h264_ps.c

@@ -228,7 +228,6 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){
         get_ue_golomb(&s->gb); /*max_dec_frame_buffering*/
 
         if (get_bits_left(&s->gb) < 0) {
-            av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", -get_bits_left(&s->gb));
             sps->num_reorder_frames=0;
             sps->bitstream_restriction_flag= 0;
         }
@@ -238,6 +237,10 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){
             return -1;
         }
     }
+    if (get_bits_left(&s->gb) < 0) {
+        av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", -get_bits_left(&s->gb));
+        return AVERROR_INVALIDDATA;
+    }
 
     return 0;
 }