lavr: fix handling of custom mix matrices

Adds some validation for changing parameters after setting the matrix and
fixes a bug in the conversion path setup.

libavresample/audio_mix.c

@@ -314,7 +314,15 @@ int ff_audio_mix_init(AVAudioResampleContext *avr)
     }
 
     /* build matrix if the user did not already set one */
-    if (!avr->am->matrix) {
+    if (avr->am->matrix) {
+        if (avr->am->coeff_type != avr->mix_coeff_type      ||
+            avr->am->in_layout  != avr->in_channel_layout   ||
+            avr->am->out_layout != avr->out_channel_layout) {
+            av_log(avr, AV_LOG_ERROR,
+                   "Custom matrix does not match current parameters\n");
+            return AVERROR(EINVAL);
+        }
+    } else {
         int i, j;
         char in_layout_name[128];
         char out_layout_name[128];

libavresample/audio_mix_matrix.c

@@ -294,8 +294,8 @@ int avresample_get_matrix(AVAudioResampleContext *avr, double *matrix,
     in_channels  = av_get_channel_layout_nb_channels(avr->in_channel_layout);
     out_channels = av_get_channel_layout_nb_channels(avr->out_channel_layout);
 
-    if ( in_channels < 0 ||  in_channels > AVRESAMPLE_MAX_CHANNELS ||
-        out_channels < 0 || out_channels > AVRESAMPLE_MAX_CHANNELS) {
+    if ( in_channels <= 0 ||  in_channels > AVRESAMPLE_MAX_CHANNELS ||
+        out_channels <= 0 || out_channels > AVRESAMPLE_MAX_CHANNELS) {
         av_log(avr, AV_LOG_ERROR, "Invalid channel layouts\n");
         return AVERROR(EINVAL);
     }
@@ -332,6 +332,7 @@ int avresample_get_matrix(AVAudioResampleContext *avr, double *matrix,
         av_log(avr, AV_LOG_ERROR, "Invalid mix coeff type\n");
         return AVERROR(EINVAL);
     }
+
     return 0;
 }
 
@@ -343,14 +344,16 @@ int avresample_set_matrix(AVAudioResampleContext *avr, const double *matrix,
     in_channels  = av_get_channel_layout_nb_channels(avr->in_channel_layout);
     out_channels = av_get_channel_layout_nb_channels(avr->out_channel_layout);
 
-    if ( in_channels < 0 ||  in_channels > AVRESAMPLE_MAX_CHANNELS ||
-        out_channels < 0 || out_channels > AVRESAMPLE_MAX_CHANNELS) {
+    if ( in_channels <= 0 ||  in_channels > AVRESAMPLE_MAX_CHANNELS ||
+        out_channels <= 0 || out_channels > AVRESAMPLE_MAX_CHANNELS) {
         av_log(avr, AV_LOG_ERROR, "Invalid channel layouts\n");
         return AVERROR(EINVAL);
     }
 
-    if (avr->am->matrix)
-        av_freep(avr->am->matrix);
+    if (avr->am->matrix) {
+        av_free(avr->am->matrix[0]);
+        avr->am->matrix = NULL;
+    }
 
 #define CONVERT_MATRIX(type, expr)                                          \
     avr->am->matrix_## type[0] = av_mallocz(out_channels * in_channels *    \
@@ -386,5 +389,11 @@ int avresample_set_matrix(AVAudioResampleContext *avr, const double *matrix,
     /* TODO: detect situations where we can just swap around pointers
              instead of doing matrix multiplications with 0.0 and 1.0 */
 
+    /* set AudioMix params */
+    avr->am->in_layout    = avr->in_channel_layout;
+    avr->am->out_layout   = avr->out_channel_layout;
+    avr->am->in_channels  = in_channels;
+    avr->am->out_channels = out_channels;
+
     return 0;
 }

libavresample/utils.c

@@ -48,9 +48,8 @@ int avresample_open(AVAudioResampleContext *avr)
     avr->resample_channels = FFMIN(avr->in_channels, avr->out_channels);
     avr->downmix_needed    = avr->in_channels  > avr->out_channels;
     avr->upmix_needed      = avr->out_channels > avr->in_channels ||
-                             avr->am->matrix                      ||
-                             (avr->out_channels == avr->in_channels &&
-                              avr->in_channel_layout != avr->out_channel_layout);
+                             (!avr->downmix_needed && (avr->am->matrix ||
+                              avr->in_channel_layout != avr->out_channel_layout));
     avr->mixing_needed     = avr->downmix_needed || avr->upmix_needed;
 
     /* set resampling parameters */