Merge commit '777885983533235ccda5145f96317fc8cd0a18ab'

* commit '777885983533235ccda5145f96317fc8cd0a18ab':
  dcadec: set channel layout in a separate function

Merged-by: Hendrik Leppkes <h.leppkes@gmail.com>

libavcodec/dcadec.c

@@ -1561,121 +1561,12 @@ static int scan_for_extensions(AVCodecContext *avctx)
     return ret;
 }
 
-/**
- * Main frame decoding function
- * FIXME add arguments
- */
-static int dca_decode_frame(AVCodecContext *avctx, void *data,
-                            int *got_frame_ptr, AVPacket *avpkt)
+static int set_channel_layout(AVCodecContext *avctx, int *channels, int num_core_channels)
 {
-    AVFrame *frame     = data;
-    const uint8_t *buf = avpkt->data;
-    int buf_size       = avpkt->size;
-    int channel_mask;
-    int channel_layout;
-    int lfe_samples;
-    int num_core_channels = 0;
-    int i, ret;
-    float **samples_flt;
-    float *src_chan;
-    float *dst_chan;
     DCAContext *s = avctx->priv_data;
-    int channels, full_channels;
-    float scale;
-    int achan;
-    int chset;
-    int mask;
-    int lavc;
-    int posn;
-    int j, k;
-    int endch;
-    int upsample = 0;
-
-    s->exss_ext_mask = 0;
-    s->xch_present   = 0;
-
-    s->dca_buffer_size = AVERROR_INVALIDDATA;
-    for (i = 0; i < buf_size - 3 && s->dca_buffer_size == AVERROR_INVALIDDATA; i++)
-        s->dca_buffer_size = avpriv_dca_convert_bitstream(buf + i, buf_size - i, s->dca_buffer,
-                                                          DCA_MAX_FRAME_SIZE + DCA_MAX_EXSS_HEADER_SIZE);
-
-    if (s->dca_buffer_size == AVERROR_INVALIDDATA) {
-        av_log(avctx, AV_LOG_ERROR, "Not a valid DCA frame\n");
-        return AVERROR_INVALIDDATA;
-    }
-
-    if ((ret = dca_parse_frame_header(s)) < 0) {
-        // seems like the frame is corrupt, try with the next one
-        return ret;
-    }
-    // set AVCodec values with parsed data
-    avctx->sample_rate = s->sample_rate;
-
-    s->profile = FF_PROFILE_DTS;
-
-    for (i = 0; i < (s->sample_blocks / 8); i++) {
-        if ((ret = dca_decode_block(s, 0, i))) {
-            av_log(avctx, AV_LOG_ERROR, "error decoding block\n");
-            return ret;
-        }
-    }
-
-    /* record number of core channels incase less than max channels are requested */
-    num_core_channels = s->prim_channels;
-
-    if (s->prim_channels + !!s->lfe > 2 &&
-        avctx->request_channel_layout == AV_CH_LAYOUT_STEREO) {
-            /* Stereo downmix coefficients
-             *
-             * The decoder can only downmix to 2-channel, so we need to ensure
-             * embedded downmix coefficients are actually targeting 2-channel.
-             */
-            if (s->core_downmix && (s->core_downmix_amode == DCA_STEREO ||
-                                    s->core_downmix_amode == DCA_STEREO_TOTAL)) {
-                for (i = 0; i < num_core_channels + !!s->lfe; i++) {
-                    /* Range checked earlier */
-                    s->downmix_coef[i][0] = dca_dmix_code(s->core_downmix_codes[i][0]);
-                    s->downmix_coef[i][1] = dca_dmix_code(s->core_downmix_codes[i][1]);
-                }
-                s->output = s->core_downmix_amode;
-            } else {
-                int am = s->amode & DCA_CHANNEL_MASK;
-                if (am >= FF_ARRAY_ELEMS(ff_dca_default_coeffs)) {
-                    av_log(s->avctx, AV_LOG_ERROR,
-                           "Invalid channel mode %d\n", am);
-                    return AVERROR_INVALIDDATA;
-                }
-                if (num_core_channels + !!s->lfe >
-                    FF_ARRAY_ELEMS(ff_dca_default_coeffs[0])) {
-                    avpriv_request_sample(s->avctx, "Downmixing %d channels",
-                                          s->prim_channels + !!s->lfe);
-                    return AVERROR_PATCHWELCOME;
-                }
-                for (i = 0; i < num_core_channels + !!s->lfe; i++) {
-                    s->downmix_coef[i][0] = ff_dca_default_coeffs[am][i][0];
-                    s->downmix_coef[i][1] = ff_dca_default_coeffs[am][i][1];
-                }
-            }
-            ff_dlog(s->avctx, "Stereo downmix coeffs:\n");
-            for (i = 0; i < num_core_channels + !!s->lfe; i++) {
-                ff_dlog(s->avctx, "L, input channel %d = %f\n", i,
-                        s->downmix_coef[i][0]);
-                ff_dlog(s->avctx, "R, input channel %d = %f\n", i,
-                        s->downmix_coef[i][1]);
-            }
-            ff_dlog(s->avctx, "\n");
-    }
-
-    if (s->ext_coding)
-        s->core_ext_mask = ff_dca_ext_audio_descr_mask[s->ext_descr];
-    else
-        s->core_ext_mask = 0;
-
-    ret = scan_for_extensions(avctx);
-
-    avctx->profile = s->profile;
-
-    full_channels = channels = s->prim_channels + !!s->lfe;
+    int i, j, chset, mask;
+    int channel_layout, channel_mask;
+    int posn, lavc;
 
     /* If we have XXCH then the channel layout is managed differently */
     /* note that XLL will also have another way to do things */
@@ -1709,7 +1600,7 @@ static int dca_decode_frame(AVCodecContext *avctx, void *data,
                 if (s->channel_order_tab[s->xch_base_channel] < 0)
                     return AVERROR_INVALIDDATA;
             } else {
-                channels       = num_core_channels + !!s->lfe;
+                *channels       = num_core_channels + !!s->lfe;
                 s->xch_present = 0; /* disable further xch processing */
                 if (s->lfe) {
                     avctx->channel_layout |= AV_CH_LOW_FREQUENCY;
@@ -1718,18 +1609,18 @@ static int dca_decode_frame(AVCodecContext *avctx, void *data,
                     s->channel_order_tab = ff_dca_channel_reorder_nolfe[s->amode];
             }
 
-            if (channels > !!s->lfe &&
-                s->channel_order_tab[channels - 1 - !!s->lfe] < 0)
+            if (*channels > !!s->lfe &&
+                s->channel_order_tab[*channels - 1 - !!s->lfe] < 0)
                 return AVERROR_INVALIDDATA;
 
-            if (av_get_channel_layout_nb_channels(avctx->channel_layout) != channels) {
-                av_log(avctx, AV_LOG_ERROR, "Number of channels %d mismatches layout %d\n", channels, av_get_channel_layout_nb_channels(avctx->channel_layout));
+            if (av_get_channel_layout_nb_channels(avctx->channel_layout) != *channels) {
+                av_log(avctx, AV_LOG_ERROR, "Number of channels %d mismatches layout %d\n", *channels, av_get_channel_layout_nb_channels(avctx->channel_layout));
                 return AVERROR_INVALIDDATA;
             }
 
             if (num_core_channels + !!s->lfe > 2 &&
                 avctx->request_channel_layout == AV_CH_LAYOUT_STEREO) {
-                channels              = 2;
+                *channels              = 2;
                 s->output             = s->prim_channels == 2 ? s->amode : DCA_STEREO;
                 avctx->channel_layout = AV_CH_LAYOUT_STEREO;
             }
@@ -1750,7 +1641,7 @@ static int dca_decode_frame(AVCodecContext *avctx, void *data,
         channel_mask = s->xxch_core_spkmask;
 
         {
-            channels = s->prim_channels + !!s->lfe;
+            *channels = s->prim_channels + !!s->lfe;
             for (i = 0; i < s->xxch_chset; i++) {
                 channel_mask |= s->xxch_spk_masks[i];
             }
@@ -1791,15 +1682,134 @@ static int dca_decode_frame(AVCodecContext *avctx, void *data,
 
             s->lfe_index = av_popcount(channel_layout & (AV_CH_LOW_FREQUENCY-1));
         } else { /* native ordering */
-            for (i = 0; i < channels; i++)
+            for (i = 0; i < *channels; i++)
                 s->xxch_order_tab[i] = i;
 
-            s->lfe_index = channels - 1;
+            s->lfe_index = *channels - 1;
         }
 
         s->channel_order_tab = s->xxch_order_tab;
     }
 
+    return 0;
+}
+
+/**
+ * Main frame decoding function
+ * FIXME add arguments
+ */
+static int dca_decode_frame(AVCodecContext *avctx, void *data,
+                            int *got_frame_ptr, AVPacket *avpkt)
+{
+    AVFrame *frame     = data;
+    const uint8_t *buf = avpkt->data;
+    int buf_size       = avpkt->size;
+    int lfe_samples;
+    int num_core_channels = 0;
+    int i, ret;
+    float **samples_flt;
+    float *src_chan;
+    float *dst_chan;
+    DCAContext *s = avctx->priv_data;
+    int channels, full_channels;
+    float scale;
+    int achan;
+    int chset;
+    int mask;
+    int j, k;
+    int endch;
+    int upsample = 0;
+
+    s->exss_ext_mask = 0;
+    s->xch_present   = 0;
+
+    s->dca_buffer_size = AVERROR_INVALIDDATA;
+    for (i = 0; i < buf_size - 3 && s->dca_buffer_size == AVERROR_INVALIDDATA; i++)
+        s->dca_buffer_size = avpriv_dca_convert_bitstream(buf + i, buf_size - i, s->dca_buffer,
+                                                          DCA_MAX_FRAME_SIZE + DCA_MAX_EXSS_HEADER_SIZE);
+
+    if (s->dca_buffer_size == AVERROR_INVALIDDATA) {
+        av_log(avctx, AV_LOG_ERROR, "Not a valid DCA frame\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    if ((ret = dca_parse_frame_header(s)) < 0) {
+        // seems like the frame is corrupt, try with the next one
+        return ret;
+    }
+    // set AVCodec values with parsed data
+    avctx->sample_rate = s->sample_rate;
+
+    s->profile = FF_PROFILE_DTS;
+
+    for (i = 0; i < (s->sample_blocks / 8); i++) {
+        if ((ret = dca_decode_block(s, 0, i))) {
+            av_log(avctx, AV_LOG_ERROR, "error decoding block\n");
+            return ret;
+        }
+    }
+
+    /* record number of core channels incase less than max channels are requested */
+    num_core_channels = s->prim_channels;
+
+    if (s->prim_channels + !!s->lfe > 2 &&
+        avctx->request_channel_layout == AV_CH_LAYOUT_STEREO) {
+            /* Stereo downmix coefficients
+             *
+             * The decoder can only downmix to 2-channel, so we need to ensure
+             * embedded downmix coefficients are actually targeting 2-channel.
+             */
+            if (s->core_downmix && (s->core_downmix_amode == DCA_STEREO ||
+                                    s->core_downmix_amode == DCA_STEREO_TOTAL)) {
+                for (i = 0; i < num_core_channels + !!s->lfe; i++) {
+                    /* Range checked earlier */
+                    s->downmix_coef[i][0] = dca_dmix_code(s->core_downmix_codes[i][0]);
+                    s->downmix_coef[i][1] = dca_dmix_code(s->core_downmix_codes[i][1]);
+                }
+                s->output = s->core_downmix_amode;
+            } else {
+                int am = s->amode & DCA_CHANNEL_MASK;
+                if (am >= FF_ARRAY_ELEMS(ff_dca_default_coeffs)) {
+                    av_log(s->avctx, AV_LOG_ERROR,
+                           "Invalid channel mode %d\n", am);
+                    return AVERROR_INVALIDDATA;
+                }
+                if (num_core_channels + !!s->lfe >
+                    FF_ARRAY_ELEMS(ff_dca_default_coeffs[0])) {
+                    avpriv_request_sample(s->avctx, "Downmixing %d channels",
+                                          s->prim_channels + !!s->lfe);
+                    return AVERROR_PATCHWELCOME;
+                }
+                for (i = 0; i < num_core_channels + !!s->lfe; i++) {
+                    s->downmix_coef[i][0] = ff_dca_default_coeffs[am][i][0];
+                    s->downmix_coef[i][1] = ff_dca_default_coeffs[am][i][1];
+                }
+            }
+            ff_dlog(s->avctx, "Stereo downmix coeffs:\n");
+            for (i = 0; i < num_core_channels + !!s->lfe; i++) {
+                ff_dlog(s->avctx, "L, input channel %d = %f\n", i,
+                        s->downmix_coef[i][0]);
+                ff_dlog(s->avctx, "R, input channel %d = %f\n", i,
+                        s->downmix_coef[i][1]);
+            }
+            ff_dlog(s->avctx, "\n");
+    }
+
+    if (s->ext_coding)
+        s->core_ext_mask = ff_dca_ext_audio_descr_mask[s->ext_descr];
+    else
+        s->core_ext_mask = 0;
+
+    ret = scan_for_extensions(avctx);
+
+    avctx->profile = s->profile;
+
+    full_channels = channels = s->prim_channels + !!s->lfe;
+
+    ret = set_channel_layout(avctx, &channels, num_core_channels);
+    if (ret < 0)
+        return ret;
+
     /* get output buffer */
     frame->nb_samples = 256 * (s->sample_blocks / 8);
     if (s->exss_ext_mask & DCA_EXT_EXSS_XLL) {