avcodec/htmlsubtitles: Fixes denial of service due to use of sscanf in inner loop for handling braces

Fixes: [Semmle Security Reports #19439]
Fixes: dos_sscanf2.mkv

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 894995c41e)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/htmlsubtitles.c

@@ -22,6 +22,7 @@
 #include "libavutil/common.h"
 #include "libavutil/parseutils.h"
 #include "htmlsubtitles.h"
+#include <ctype.h>
 
 static int html_color_parse(void *log_ctx, const char *str)
 {
@@ -51,6 +52,25 @@ static void rstrip_spaces_buf(AVBPrint *buf)
             buf->str[--buf->len] = 0;
 }
 
+/*
+ * Fast code for scanning text enclosed in braces. Functionally
+ * equivalent to this sscanf call:
+ *
+ * sscanf(in, "{\\an%*1u}%n", &len) >= 0 && len > 0
+ */
+static int scanbraces(const char* in) {
+    if (strncmp(in, "{\\an", 4) != 0) {
+        return 0;
+    }
+    if (!isdigit(in[4])) {
+        return 0;
+    }
+    if (in[5] != '}') {
+        return 0;
+    }
+    return 1;
+}
+
 /*
  * Fast code for scanning the rest of a tag. Functionally equivalent to
  * this sscanf call:
@@ -110,9 +130,7 @@ int ff_htmlmarkup_to_ass(void *log_ctx, AVBPrint *dst, const char *in)
             break;
         case '{':    /* skip all {\xxx} substrings except for {\an%d}
                         and all microdvd like styles such as {Y:xxx} */
-            len = 0;
-            an += sscanf(in, "{\\an%*1u}%n", &len) >= 0 && len > 0;
-
+            an += scanbraces(in);
             if (!closing_brace_missing) {
                 if (   (an != 1 && in[1] == '\\')
                     || (in[1] && strchr("CcFfoPSsYy", in[1]) && in[2] == ':')) {