qtrle: return error on decode_init() failure.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit e54ae60e46)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

libavcodec/qtrle.c

@@ -407,7 +407,7 @@ static av_cold int qtrle_decode_init(AVCodecContext *avctx)
     default:
         av_log (avctx, AV_LOG_ERROR, "Unsupported colorspace: %d bits/sample?\n",
             avctx->bits_per_coded_sample);
-        break;
+        return AVERROR_INVALIDDATA;
     }
 
     s->frame.data[0] = NULL;