swr: fix the noise sample type for noise shaping

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libswresample/dither.c

@@ -23,15 +23,12 @@
 
 #include "noise_shaping_data.c"
 
-void swri_get_dither(SwrContext *s, void *dst, int len, unsigned seed, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt) {
+void swri_get_dither(SwrContext *s, void *dst, int len, unsigned seed, enum AVSampleFormat noise_fmt) {
     double scale = s->dither.ns_scale;
 #define TMP_EXTRA 2
     double *tmp = av_malloc((len + TMP_EXTRA) * sizeof(double));
     int i;
 
-    out_fmt = av_get_packed_sample_fmt(out_fmt);
-    in_fmt  = av_get_packed_sample_fmt( in_fmt);
-
     for(i=0; i<len + TMP_EXTRA; i++){
         double v;
         seed = seed* 1664525 + 1013904223;
@@ -63,11 +60,11 @@ void swri_get_dither(SwrContext *s, void *dst, int len, unsigned seed, enum AVSa
 
         v*= scale;
 
-        switch(in_fmt){
-            case AV_SAMPLE_FMT_S16: ((int16_t*)dst)[i] = v; break;
-            case AV_SAMPLE_FMT_S32: ((int32_t*)dst)[i] = v; break;
-            case AV_SAMPLE_FMT_FLT: ((float  *)dst)[i] = v; break;
-            case AV_SAMPLE_FMT_DBL: ((double *)dst)[i] = v; break;
+        switch(noise_fmt){
+            case AV_SAMPLE_FMT_S16P: ((int16_t*)dst)[i] = v; break;
+            case AV_SAMPLE_FMT_S32P: ((int32_t*)dst)[i] = v; break;
+            case AV_SAMPLE_FMT_FLTP: ((float  *)dst)[i] = v; break;
+            case AV_SAMPLE_FMT_DBLP: ((double *)dst)[i] = v; break;
             default: av_assert0(0);
         }
     }
@@ -117,6 +114,10 @@ int swri_dither_init(SwrContext *s, enum AVSampleFormat out_fmt, enum AVSampleFo
     }
 
     s->dither.noise = s->preout;
+    if (s->dither.method > SWR_DITHER_NS) {
+        s->dither.noise.bps = 4;
+        s->dither.noise.fmt = AV_SAMPLE_FMT_FLTP;
+    }
 
     return 0;
 }

libswresample/swresample.c

@@ -654,7 +654,7 @@ static int swr_convert_internal(struct SwrContext *s, AudioData *out, int out_co
                 return ret;
             if(ret)
                 for(ch=0; ch<s->dither.noise.ch_count; ch++)
-                    swri_get_dither(s, s->dither.noise.ch[ch], s->dither.noise.count, 12345678913579<<ch, s->out_sample_fmt, s->int_sample_fmt);
+                    swri_get_dither(s, s->dither.noise.ch[ch], s->dither.noise.count, 12345678913579<<ch, s->dither.noise.fmt);
             av_assert0(s->dither.noise.ch_count == preout->ch_count);
 
             if(s->dither.dither_pos + out_count > s->dither.noise.count)

libswresample/swresample_internal.h

@@ -176,7 +176,7 @@ void swri_rematrix_free(SwrContext *s);
 int swri_rematrix(SwrContext *s, AudioData *out, AudioData *in, int len, int mustcopy);
 void swri_rematrix_init_x86(struct SwrContext *s);
 
-void swri_get_dither(SwrContext *s, void *dst, int len, unsigned seed, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt);
+void swri_get_dither(SwrContext *s, void *dst, int len, unsigned seed, enum AVSampleFormat noise_fmt);
 int swri_dither_init(SwrContext *s, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt);
 
 void swri_audio_convert_init_arm(struct AudioConvert *ac,