avcodec/vmnc: Check location before use

Fixes: runtime error: signed integer overflow: 65535 * 64256 cannot be represented in type 'int'
Fixes: 1717/clusterfuzz-testcase-minimized-5491696676634624

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/vmnc.c

@@ -381,6 +381,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         w   = bytestream2_get_be16(gb);
         h   = bytestream2_get_be16(gb);
         enc = bytestream2_get_be32(gb);
+        if ((dx + w > c->width) || (dy + h > c->height)) {
+            av_log(avctx, AV_LOG_ERROR,
+                    "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
+                    w, h, dx, dy, c->width, c->height);
+            return AVERROR_INVALIDDATA;
+        }
         outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
         size_left = bytestream2_get_bytes_left(gb);
         switch (enc) {
@@ -458,12 +464,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             bytestream2_skip(gb, 2);
             break;
         case 0x00000000: // raw rectangle data
-            if ((dx + w > c->width) || (dy + h > c->height)) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
-                       w, h, dx, dy, c->width, c->height);
-                return AVERROR_INVALIDDATA;
-            }
             if (size_left < w * h * c->bpp2) {
                 av_log(avctx, AV_LOG_ERROR,
                        "Premature end of data! (need %i got %i)\n",
@@ -474,12 +474,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                       c->pic->linesize[0]);
             break;
         case 0x00000005: // HexTile encoded rectangle
-            if ((dx + w > c->width) || (dy + h > c->height)) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
-                       w, h, dx, dy, c->width, c->height);
-                return AVERROR_INVALIDDATA;
-            }
             res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]);
             if (res < 0)
                 return res;