vf_format: rework format list parsing

Avoid using AV_PIX_FMT_NB, since that hardcodes the number of pixel
formats into lavfi and will break when a shared lavu is updated, adding
new pixel formats.

libavfilter/vf_format.c

@@ -38,66 +38,104 @@
 typedef struct FormatContext {
     const AVClass *class;
     char *pix_fmts;
+
     /**
-     * List of flags telling if a given image format has been listed
-     * as argument to the filter.
+     * pix_fmts parsed into AVPixelFormats and terminated with
+     * AV_PIX_FMT_NONE
      */
-    int listed_pix_fmt_flags[AV_PIX_FMT_NB];
+    enum AVPixelFormat *formats;
 } FormatContext;
 
-#define AV_PIX_FMT_NAME_MAXSIZE 32
+static av_cold void uninit(AVFilterContext *ctx)
+{
+    FormatContext *s = ctx->priv;
+    av_freep(&s->formats);
+}
 
 static av_cold int init(AVFilterContext *ctx)
 {
     FormatContext *s = ctx->priv;
-    const char *cur, *sep;
-    char             pix_fmt_name[AV_PIX_FMT_NAME_MAXSIZE];
-    int              pix_fmt_name_len;
-    enum AVPixelFormat pix_fmt;
+    char *cur, *sep;
+    int nb_formats = 1;
+    int i;
+
+    /* count the formats */
+    cur = s->pix_fmts;
+    while ((cur = strchr(cur, '|'))) {
+        nb_formats++;
+        if (*cur)
+            cur++;
+    }
+
+    s->formats = av_malloc_array(nb_formats + 1, sizeof(*s->formats));
+    if (!s->formats)
+        return AVERROR(ENOMEM);
 
     /* parse the list of formats */
-    for (cur = s->pix_fmts; cur; cur = sep ? sep + 1 : NULL) {
-        if (!(sep = strchr(cur, '|')))
-            pix_fmt_name_len = strlen(cur);
-        else
-            pix_fmt_name_len = sep - cur;
-        if (pix_fmt_name_len >= AV_PIX_FMT_NAME_MAXSIZE) {
-            av_log(ctx, AV_LOG_ERROR, "Format name too long\n");
-            return -1;
+    cur = s->pix_fmts;
+    for (i = 0; i < nb_formats; i++) {
+        sep = strchr(cur, '|');
+        if (sep)
+            *sep++ = 0;
+
+        s->formats[i] = av_get_pix_fmt(cur);
+        if (s->formats[i] == AV_PIX_FMT_NONE) {
+            av_log(ctx, AV_LOG_ERROR, "Unknown pixel format: %s\n", cur);
+            return AVERROR(EINVAL);
         }
 
-        memcpy(pix_fmt_name, cur, pix_fmt_name_len);
-        pix_fmt_name[pix_fmt_name_len] = 0;
-        pix_fmt = av_get_pix_fmt(pix_fmt_name);
+        cur = sep;
+    }
+    s->formats[nb_formats] = AV_PIX_FMT_NONE;
+
+    if (!strcmp(ctx->filter->name, "noformat")) {
+        const AVPixFmtDescriptor *desc = NULL;
+        enum AVPixelFormat *formats_allowed;
+        int nb_formats_lavu = 0, nb_formats_allowed = 0;;
 
-        if (pix_fmt == AV_PIX_FMT_NONE) {
-            av_log(ctx, AV_LOG_ERROR, "Unknown pixel format: %s\n", pix_fmt_name);
-            return -1;
-        }
+        /* count the formats known to lavu */
+        while ((desc = av_pix_fmt_desc_next(desc)))
+            nb_formats_lavu++;
+
+        formats_allowed = av_malloc_array(nb_formats_lavu + 1, sizeof(*formats_allowed));
+        if (!formats_allowed)
+            return AVERROR(ENOMEM);
+
+        /* for each format known to lavu, check if it's in the list of
+         * forbidden formats */
+        while ((desc = av_pix_fmt_desc_next(desc))) {
+            enum AVPixelFormat pix_fmt = av_pix_fmt_desc_get_id(desc);
 
-        s->listed_pix_fmt_flags[pix_fmt] = 1;
+            for (i = 0; i < nb_formats; i++) {
+                if (s->formats[i] == pix_fmt)
+                    break;
+            }
+            if (i < nb_formats)
+                continue;
+
+            formats_allowed[nb_formats_allowed++] = pix_fmt;
+        }
+        formats_allowed[nb_formats_allowed] = AV_PIX_FMT_NONE;
+        av_freep(&s->formats);
+        s->formats = formats_allowed;
     }
 
     return 0;
 }
 
-static AVFilterFormats *make_format_list(FormatContext *s, int flag)
+static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    enum AVPixelFormat pix_fmt;
-
-    for (pix_fmt = 0; pix_fmt < AV_PIX_FMT_NB; pix_fmt++)
-        if (s->listed_pix_fmt_flags[pix_fmt] == flag) {
-            int ret = ff_add_format(&formats, pix_fmt);
-            if (ret < 0) {
-                ff_formats_unref(&formats);
-                return NULL;
-            }
-        }
+    FormatContext *s = ctx->priv;
+    AVFilterFormats *formats = ff_make_format_list(s->formats);
+
+    if (!formats)
+        return AVERROR(ENOMEM);
 
-    return formats;
+    ff_set_common_formats(ctx, formats);
+    return 0;
 }
 
+
 #define OFFSET(x) offsetof(FormatContext, x)
 static const AVOption options[] = {
     { "pix_fmts", "A '|'-separated list of pixel formats", OFFSET(pix_fmts), AV_OPT_TYPE_STRING, .flags = AV_OPT_FLAG_VIDEO_PARAM },
@@ -105,12 +143,6 @@ static const AVOption options[] = {
 };
 
 #if CONFIG_FORMAT_FILTER
-static int query_formats_format(AVFilterContext *ctx)
-{
-    ff_set_common_formats(ctx, make_format_list(ctx->priv, 1));
-    return 0;
-}
-
 static const AVClass format_class = {
     .class_name = "format",
     .item_name  = av_default_item_name,
@@ -140,8 +172,9 @@ AVFilter ff_vf_format = {
     .description = NULL_IF_CONFIG_SMALL("Convert the input video to one of the specified pixel formats."),
 
     .init      = init,
+    .uninit    = uninit,
 
-    .query_formats = query_formats_format,
+    .query_formats = query_formats,
 
     .priv_size = sizeof(FormatContext),
     .priv_class = &format_class,
@@ -152,12 +185,6 @@ AVFilter ff_vf_format = {
 #endif /* CONFIG_FORMAT_FILTER */
 
 #if CONFIG_NOFORMAT_FILTER
-static int query_formats_noformat(AVFilterContext *ctx)
-{
-    ff_set_common_formats(ctx, make_format_list(ctx->priv, 0));
-    return 0;
-}
-
 static const AVClass noformat_class = {
     .class_name = "noformat",
     .item_name  = av_default_item_name,
@@ -187,8 +214,9 @@ AVFilter ff_vf_noformat = {
     .description = NULL_IF_CONFIG_SMALL("Force libavfilter not to use any of the specified pixel formats for the input to the next filter."),
 
     .init      = init,
+    .uninit    = uninit,
 
-    .query_formats = query_formats_noformat,
+    .query_formats = query_formats,
 
     .priv_size = sizeof(FormatContext),
     .priv_class = &noformat_class,