avformat/hevc: Fix potential leak in case of ff_hevc_annexb2mp4_buf failure

ff_hevc_annexb2mp4_buf() could indicate an error, yet leave cleaning
after itself to the caller, so that a caller could not simply return the
error, but had to free the buffer first.

(Given that all current callers have set filter_ps = 0, this error can
currently not be triggered.)

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>

libavformat/hevc.c

@@ -25,6 +25,7 @@
 #include "libavutil/intreadwrite.h"
 #include "avc.h"
 #include "avio.h"
+#include "avio_internal.h"
 #include "hevc.h"
 
 #define MAX_SPATIAL_SEGMENTATION 4096 // max. value of u(12) field
@@ -1054,6 +1055,11 @@ int ff_hevc_annexb2mp4_buf(const uint8_t *buf_in, uint8_t **buf_out,
         return ret;
 
     ret   = ff_hevc_annexb2mp4(pb, buf_in, *size, filter_ps, ps_count);
+    if (ret < 0) {
+        ffio_free_dyn_buf(&pb);
+        return ret;
+    }
+
     *size = avio_close_dyn_buf(pb, buf_out);
 
     return ret;

libavformat/hevc.h

@@ -60,13 +60,13 @@ int ff_hevc_annexb2mp4(AVIOContext *pb, const uint8_t *buf_in,
  * If filter_ps is non-zero, any HEVC parameter sets found in the input will be
  * discarded, and *ps_count will be set to the number of discarded PS NAL units.
  *
- * On output, *size holds the size (in bytes) of the output data buffer.
+ * On success, *size holds the size (in bytes) of the output data buffer.
  *
  * @param buf_in address of the buffer holding the input data
  * @param size address of the variable holding the size (in bytes) of the input
- *        buffer (on input) and of the output buffer (on output)
- * @param buf_out address of the variable holding the address of the output
- *        buffer
+ *        buffer (on input) and of the output buffer (on success)
+ * @param buf_out on success, address of the variable holding the address of
+ *        the output buffer
  * @param filter_ps whether to write parameter set NAL units to the output (0)
  *        or to discard them (non-zero)
  * @param ps_count address of the variable where the number of discarded