libavformat/avienc: Check bits per sample for PAL8

Fixes: assertion failure
Fixes: Ticket 8172

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 3595878281)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/avienc.c

@@ -459,6 +459,14 @@ static int avi_write_header(AVFormatContext *s)
                     && par->format != AV_PIX_FMT_NONE)
                     av_log(s, AV_LOG_ERROR, "%s rawvideo cannot be written to avi, output file will be unreadable\n",
                           av_get_pix_fmt_name(par->format));
+
+                if (par->format == AV_PIX_FMT_PAL8) {
+                    if (par->bits_per_coded_sample < 0 || par->bits_per_coded_sample > 8) {
+                        av_log(s, AV_LOG_ERROR, "PAL8 with %d bps is not allowed\n", par->bits_per_coded_sample);
+                        return AVERROR(EINVAL);
+                    }
+                }
+
                 break;
             case AVMEDIA_TYPE_AUDIO:
                 flags = (avi->write_channel_mask == 0) ? FF_PUT_WAV_HEADER_SKIP_CHANNELMASK : 0;