Fix updating condition for the probe_size variable in the internal

loop of ff_probe_input_buffer(), making sure that probe_size is always
set to probe_max_size in the last iteration.

Also make the function return an error if we get to the max probe
length and still cannot figure out what the format is.

Patch by Micah Galizia micahgalizia A gmail D com.

Originally committed as revision 22688 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavformat/utils.c

@@ -478,7 +478,8 @@ int ff_probe_input_buffer(ByteIOContext **pb, AVInputFormat **fmt,
         return AVERROR(EINVAL);
     }
 
-    for(probe_size= PROBE_BUF_MIN; probe_size<=max_probe_size && !*fmt && ret >= 0; probe_size<<=1){
+    for(probe_size= PROBE_BUF_MIN; probe_size<=max_probe_size && !*fmt && ret >= 0;
+        probe_size = FFMIN(probe_size<<1, FFMAX(max_probe_size, probe_size+1))) {
         int ret, score = probe_size < max_probe_size ? AVPROBE_SCORE_MAX/4 : 0;
         int buf_offset = (probe_size == PROBE_BUF_MIN) ? 0 : probe_size>>1;
 
@@ -513,6 +514,11 @@ int ff_probe_input_buffer(ByteIOContext **pb, AVInputFormat **fmt,
     }
 
     av_free(buf);
+
+    if (!*fmt) {
+        return AVERROR_INVALIDDATA;
+    }
+
     if (url_fseek(*pb, 0, SEEK_SET) < 0) {
         url_fclose(*pb);
         if (url_fopen(pb, filename, URL_RDONLY) < 0)