lavu/mem: fix potential int overflow and crash in av_dynarray_add()

Also extend documentation accordingly.

libavutil/mem.c

@@ -249,15 +249,25 @@ void av_dynarray_add(void *tab_ptr, int *nb_ptr, void *elem)
     nb = *nb_ptr;
     tab = *(intptr_t**)tab_ptr;
     if ((nb & (nb - 1)) == 0) {
-        if (nb == 0)
+        if (nb == 0) {
             nb_alloc = 1;
-        else
+        } else {
+            if (nb > INT_MAX / (2 * sizeof(intptr_t)))
+                goto fail;
             nb_alloc = nb * 2;
+        }
         tab = av_realloc(tab, nb_alloc * sizeof(intptr_t));
+        if (!tab)
+            goto fail;
         *(intptr_t**)tab_ptr = tab;
     }
     tab[nb++] = (intptr_t)elem;
     *nb_ptr = nb;
+    return;
+
+fail:
+    av_freep(tab_ptr);
+    *nb_ptr = 0;
 }
 
 static void fill16(uint8_t *dst, int len)

libavutil/mem.h

@@ -209,6 +209,8 @@ void av_freep(void *ptr);
  * In case of success, the pointer to the array is updated in order to
  * point to the new grown array, and the number pointed to by nb_ptr
  * is incremented.
+ * In case of failure, the array is freed, *tab_ptr is set to NULL and
+ * *nb_ptr is set to 0.
  *
  * @param tab_ptr pointer to the array to grow
  * @param nb_ptr  pointer to the number of elements in the array