You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

495 lines
16KB

  1. /*
  2. * Copyright (c) 2016 Paul B Mahol
  3. *
  4. * This file is part of FFmpeg.
  5. *
  6. * FFmpeg is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU Lesser General Public License as published
  8. * by the Free Software Foundation; either version 2.1 of the License,
  9. * or (at your option) any later version.
  10. *
  11. * FFmpeg is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with FFmpeg; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19. */
  20. #include "libavutil/audio_fifo.h"
  21. #include "libavutil/avstring.h"
  22. #include "libavfilter/internal.h"
  23. #include "libavutil/common.h"
  24. #include "libavutil/opt.h"
  25. #include "libavcodec/avfft.h"
  26. #include "libavutil/eval.h"
  27. #include "audio.h"
  28. #include "filters.h"
  29. #include "window_func.h"
  30. typedef struct AFFTFiltContext {
  31. const AVClass *class;
  32. char *real_str;
  33. char *img_str;
  34. int fft_bits;
  35. FFTContext *fft, *ifft;
  36. FFTComplex **fft_data;
  37. FFTComplex **fft_temp;
  38. int nb_exprs;
  39. int window_size;
  40. AVExpr **real;
  41. AVExpr **imag;
  42. AVAudioFifo *fifo;
  43. int64_t pts;
  44. int hop_size;
  45. float overlap;
  46. AVFrame *buffer;
  47. int eof;
  48. int win_func;
  49. float *window_func_lut;
  50. } AFFTFiltContext;
  51. static const char *const var_names[] = { "sr", "b", "nb", "ch", "chs", "pts", "re", "im", NULL };
  52. enum { VAR_SAMPLE_RATE, VAR_BIN, VAR_NBBINS, VAR_CHANNEL, VAR_CHANNELS, VAR_PTS, VAR_REAL, VAR_IMAG, VAR_VARS_NB };
  53. #define OFFSET(x) offsetof(AFFTFiltContext, x)
  54. #define A AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
  55. static const AVOption afftfilt_options[] = {
  56. { "real", "set channels real expressions", OFFSET(real_str), AV_OPT_TYPE_STRING, {.str = "re" }, 0, 0, A },
  57. { "imag", "set channels imaginary expressions", OFFSET(img_str), AV_OPT_TYPE_STRING, {.str = "im" }, 0, 0, A },
  58. { "win_size", "set window size", OFFSET(fft_bits), AV_OPT_TYPE_INT, {.i64=12}, 4, 17, A, "fft" },
  59. { "w16", 0, 0, AV_OPT_TYPE_CONST, {.i64=4}, 0, 0, A, "fft" },
  60. { "w32", 0, 0, AV_OPT_TYPE_CONST, {.i64=5}, 0, 0, A, "fft" },
  61. { "w64", 0, 0, AV_OPT_TYPE_CONST, {.i64=6}, 0, 0, A, "fft" },
  62. { "w128", 0, 0, AV_OPT_TYPE_CONST, {.i64=7}, 0, 0, A, "fft" },
  63. { "w256", 0, 0, AV_OPT_TYPE_CONST, {.i64=8}, 0, 0, A, "fft" },
  64. { "w512", 0, 0, AV_OPT_TYPE_CONST, {.i64=9}, 0, 0, A, "fft" },
  65. { "w1024", 0, 0, AV_OPT_TYPE_CONST, {.i64=10}, 0, 0, A, "fft" },
  66. { "w2048", 0, 0, AV_OPT_TYPE_CONST, {.i64=11}, 0, 0, A, "fft" },
  67. { "w4096", 0, 0, AV_OPT_TYPE_CONST, {.i64=12}, 0, 0, A, "fft" },
  68. { "w8192", 0, 0, AV_OPT_TYPE_CONST, {.i64=13}, 0, 0, A, "fft" },
  69. { "w16384", 0, 0, AV_OPT_TYPE_CONST, {.i64=14}, 0, 0, A, "fft" },
  70. { "w32768", 0, 0, AV_OPT_TYPE_CONST, {.i64=15}, 0, 0, A, "fft" },
  71. { "w65536", 0, 0, AV_OPT_TYPE_CONST, {.i64=16}, 0, 0, A, "fft" },
  72. { "w131072",0, 0, AV_OPT_TYPE_CONST, {.i64=17}, 0, 0, A, "fft" },
  73. { "win_func", "set window function", OFFSET(win_func), AV_OPT_TYPE_INT, {.i64 = WFUNC_HANNING}, 0, NB_WFUNC-1, A, "win_func" },
  74. { "rect", "Rectangular", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_RECT}, 0, 0, A, "win_func" },
  75. { "bartlett", "Bartlett", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BARTLETT}, 0, 0, A, "win_func" },
  76. { "hann", "Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, A, "win_func" },
  77. { "hanning", "Hanning", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, A, "win_func" },
  78. { "hamming", "Hamming", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HAMMING}, 0, 0, A, "win_func" },
  79. { "blackman", "Blackman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BLACKMAN}, 0, 0, A, "win_func" },
  80. { "welch", "Welch", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_WELCH}, 0, 0, A, "win_func" },
  81. { "flattop", "Flat-top", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_FLATTOP}, 0, 0, A, "win_func" },
  82. { "bharris", "Blackman-Harris", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHARRIS}, 0, 0, A, "win_func" },
  83. { "bnuttall", "Blackman-Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BNUTTALL}, 0, 0, A, "win_func" },
  84. { "bhann", "Bartlett-Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHANN}, 0, 0, A, "win_func" },
  85. { "sine", "Sine", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_SINE}, 0, 0, A, "win_func" },
  86. { "nuttall", "Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_NUTTALL}, 0, 0, A, "win_func" },
  87. { "lanczos", "Lanczos", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_LANCZOS}, 0, 0, A, "win_func" },
  88. { "gauss", "Gauss", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_GAUSS}, 0, 0, A, "win_func" },
  89. { "tukey", "Tukey", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_TUKEY}, 0, 0, A, "win_func" },
  90. { "dolph", "Dolph-Chebyshev", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_DOLPH}, 0, 0, A, "win_func" },
  91. { "cauchy", "Cauchy", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_CAUCHY}, 0, 0, A, "win_func" },
  92. { "parzen", "Parzen", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_PARZEN}, 0, 0, A, "win_func" },
  93. { "poisson", "Poisson", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_POISSON}, 0, 0, A, "win_func" },
  94. { "bohman", "Bohman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BOHMAN}, 0, 0, A, "win_func" },
  95. { "overlap", "set window overlap", OFFSET(overlap), AV_OPT_TYPE_FLOAT, {.dbl=0.75}, 0, 1, A },
  96. { NULL },
  97. };
  98. AVFILTER_DEFINE_CLASS(afftfilt);
  99. static inline double getreal(void *priv, double x, double ch)
  100. {
  101. AFFTFiltContext *s = priv;
  102. int ich, ix;
  103. ich = av_clip(ch, 0, s->nb_exprs - 1);
  104. ix = av_clip(x, 0, s->window_size / 2);
  105. return s->fft_data[ich][ix].re;
  106. }
  107. static inline double getimag(void *priv, double x, double ch)
  108. {
  109. AFFTFiltContext *s = priv;
  110. int ich, ix;
  111. ich = av_clip(ch, 0, s->nb_exprs - 1);
  112. ix = av_clip(x, 0, s->window_size / 2);
  113. return s->fft_data[ich][ix].im;
  114. }
  115. static double realf(void *priv, double x, double ch) { return getreal(priv, x, ch); }
  116. static double imagf(void *priv, double x, double ch) { return getimag(priv, x, ch); }
  117. static const char *const func2_names[] = { "real", "imag", NULL };
  118. double (*func2[])(void *, double, double) = { realf, imagf, NULL };
  119. static int config_input(AVFilterLink *inlink)
  120. {
  121. AVFilterContext *ctx = inlink->dst;
  122. AFFTFiltContext *s = ctx->priv;
  123. char *saveptr = NULL;
  124. int ret = 0, ch;
  125. float overlap;
  126. char *args;
  127. const char *last_expr = "1";
  128. s->pts = AV_NOPTS_VALUE;
  129. s->fft = av_fft_init(s->fft_bits, 0);
  130. s->ifft = av_fft_init(s->fft_bits, 1);
  131. if (!s->fft || !s->ifft)
  132. return AVERROR(ENOMEM);
  133. s->window_size = 1 << s->fft_bits;
  134. s->fft_data = av_calloc(inlink->channels, sizeof(*s->fft_data));
  135. if (!s->fft_data)
  136. return AVERROR(ENOMEM);
  137. s->fft_temp = av_calloc(inlink->channels, sizeof(*s->fft_temp));
  138. if (!s->fft_temp)
  139. return AVERROR(ENOMEM);
  140. for (ch = 0; ch < inlink->channels; ch++) {
  141. s->fft_data[ch] = av_calloc(s->window_size, sizeof(**s->fft_data));
  142. if (!s->fft_data[ch])
  143. return AVERROR(ENOMEM);
  144. }
  145. for (ch = 0; ch < inlink->channels; ch++) {
  146. s->fft_temp[ch] = av_calloc(s->window_size, sizeof(**s->fft_temp));
  147. if (!s->fft_temp[ch])
  148. return AVERROR(ENOMEM);
  149. }
  150. s->real = av_calloc(inlink->channels, sizeof(*s->real));
  151. if (!s->real)
  152. return AVERROR(ENOMEM);
  153. s->imag = av_calloc(inlink->channels, sizeof(*s->imag));
  154. if (!s->imag)
  155. return AVERROR(ENOMEM);
  156. args = av_strdup(s->real_str);
  157. if (!args)
  158. return AVERROR(ENOMEM);
  159. for (ch = 0; ch < inlink->channels; ch++) {
  160. char *arg = av_strtok(ch == 0 ? args : NULL, "|", &saveptr);
  161. ret = av_expr_parse(&s->real[ch], arg ? arg : last_expr, var_names,
  162. NULL, NULL, func2_names, func2, 0, ctx);
  163. if (ret < 0)
  164. break;
  165. if (arg)
  166. last_expr = arg;
  167. s->nb_exprs++;
  168. }
  169. av_free(args);
  170. args = av_strdup(s->img_str ? s->img_str : s->real_str);
  171. if (!args)
  172. return AVERROR(ENOMEM);
  173. for (ch = 0; ch < inlink->channels; ch++) {
  174. char *arg = av_strtok(ch == 0 ? args : NULL, "|", &saveptr);
  175. ret = av_expr_parse(&s->imag[ch], arg ? arg : last_expr, var_names,
  176. NULL, NULL, func2_names, func2, 0, ctx);
  177. if (ret < 0)
  178. break;
  179. if (arg)
  180. last_expr = arg;
  181. }
  182. av_free(args);
  183. s->fifo = av_audio_fifo_alloc(inlink->format, inlink->channels, s->window_size);
  184. if (!s->fifo)
  185. return AVERROR(ENOMEM);
  186. s->window_func_lut = av_realloc_f(s->window_func_lut, s->window_size,
  187. sizeof(*s->window_func_lut));
  188. if (!s->window_func_lut)
  189. return AVERROR(ENOMEM);
  190. generate_window_func(s->window_func_lut, s->window_size, s->win_func, &overlap);
  191. if (s->overlap == 1)
  192. s->overlap = overlap;
  193. s->hop_size = s->window_size * (1 - s->overlap);
  194. if (s->hop_size <= 0)
  195. return AVERROR(EINVAL);
  196. s->buffer = ff_get_audio_buffer(inlink, s->window_size * 2);
  197. if (!s->buffer)
  198. return AVERROR(ENOMEM);
  199. return ret;
  200. }
  201. static int filter_frame(AVFilterLink *inlink)
  202. {
  203. AVFilterContext *ctx = inlink->dst;
  204. AVFilterLink *outlink = ctx->outputs[0];
  205. AFFTFiltContext *s = ctx->priv;
  206. const int window_size = s->window_size;
  207. const float f = 1. / (s->window_size / 2);
  208. double values[VAR_VARS_NB];
  209. AVFrame *out, *in = NULL;
  210. int ch, n, ret, i;
  211. if (!in) {
  212. in = ff_get_audio_buffer(outlink, window_size);
  213. if (!in)
  214. return AVERROR(ENOMEM);
  215. }
  216. ret = av_audio_fifo_peek(s->fifo, (void **)in->extended_data, window_size);
  217. if (ret < 0)
  218. goto fail;
  219. for (ch = 0; ch < inlink->channels; ch++) {
  220. const float *src = (float *)in->extended_data[ch];
  221. FFTComplex *fft_data = s->fft_data[ch];
  222. for (n = 0; n < in->nb_samples; n++) {
  223. fft_data[n].re = src[n] * s->window_func_lut[n];
  224. fft_data[n].im = 0;
  225. }
  226. for (; n < window_size; n++) {
  227. fft_data[n].re = 0;
  228. fft_data[n].im = 0;
  229. }
  230. }
  231. values[VAR_PTS] = s->pts;
  232. values[VAR_SAMPLE_RATE] = inlink->sample_rate;
  233. values[VAR_NBBINS] = window_size / 2;
  234. values[VAR_CHANNELS] = inlink->channels;
  235. for (ch = 0; ch < inlink->channels; ch++) {
  236. FFTComplex *fft_data = s->fft_data[ch];
  237. av_fft_permute(s->fft, fft_data);
  238. av_fft_calc(s->fft, fft_data);
  239. }
  240. for (ch = 0; ch < inlink->channels; ch++) {
  241. FFTComplex *fft_data = s->fft_data[ch];
  242. FFTComplex *fft_temp = s->fft_temp[ch];
  243. float *buf = (float *)s->buffer->extended_data[ch];
  244. int x;
  245. values[VAR_CHANNEL] = ch;
  246. for (n = 0; n <= window_size / 2; n++) {
  247. float fr, fi;
  248. values[VAR_BIN] = n;
  249. values[VAR_REAL] = fft_data[n].re;
  250. values[VAR_IMAG] = fft_data[n].im;
  251. fr = av_expr_eval(s->real[ch], values, s);
  252. fi = av_expr_eval(s->imag[ch], values, s);
  253. fft_temp[n].re = fr;
  254. fft_temp[n].im = fi;
  255. }
  256. for (n = window_size / 2 + 1, x = window_size / 2 - 1; n < window_size; n++, x--) {
  257. fft_temp[n].re = fft_temp[x].re;
  258. fft_temp[n].im = -fft_temp[x].im;
  259. }
  260. av_fft_permute(s->ifft, fft_temp);
  261. av_fft_calc(s->ifft, fft_temp);
  262. for (i = 0; i < window_size; i++) {
  263. buf[i] += s->fft_temp[ch][i].re * f;
  264. }
  265. }
  266. out = ff_get_audio_buffer(outlink, s->hop_size);
  267. if (!out) {
  268. ret = AVERROR(ENOMEM);
  269. goto fail;
  270. }
  271. out->pts = s->pts;
  272. s->pts += s->hop_size;
  273. for (ch = 0; ch < inlink->channels; ch++) {
  274. float *dst = (float *)out->extended_data[ch];
  275. float *buf = (float *)s->buffer->extended_data[ch];
  276. for (n = 0; n < s->hop_size; n++)
  277. dst[n] = buf[n] * (1.f - s->overlap);
  278. memmove(buf, buf + s->hop_size, window_size * 4);
  279. }
  280. ret = ff_filter_frame(outlink, out);
  281. if (ret < 0)
  282. goto fail;
  283. av_audio_fifo_drain(s->fifo, s->hop_size);
  284. fail:
  285. av_frame_free(&in);
  286. return ret < 0 ? ret : 0;
  287. }
  288. static int activate(AVFilterContext *ctx)
  289. {
  290. AVFilterLink *inlink = ctx->inputs[0];
  291. AVFilterLink *outlink = ctx->outputs[0];
  292. AFFTFiltContext *s = ctx->priv;
  293. AVFrame *in = NULL;
  294. int ret = 0, status;
  295. int64_t pts;
  296. FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
  297. if (!s->eof && av_audio_fifo_size(s->fifo) < s->window_size) {
  298. ret = ff_inlink_consume_frame(inlink, &in);
  299. if (ret < 0)
  300. return ret;
  301. if (ret > 0) {
  302. ret = av_audio_fifo_write(s->fifo, (void **)in->extended_data,
  303. in->nb_samples);
  304. if (ret >= 0 && s->pts == AV_NOPTS_VALUE)
  305. s->pts = in->pts;
  306. av_frame_free(&in);
  307. if (ret < 0)
  308. return ret;
  309. }
  310. }
  311. if ((av_audio_fifo_size(s->fifo) >= s->window_size) ||
  312. (av_audio_fifo_size(s->fifo) > 0 && s->eof)) {
  313. ret = filter_frame(inlink);
  314. if (av_audio_fifo_size(s->fifo) >= s->window_size)
  315. ff_filter_set_ready(ctx, 100);
  316. return ret;
  317. }
  318. if (!s->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) {
  319. if (status == AVERROR_EOF) {
  320. s->eof = 1;
  321. if (av_audio_fifo_size(s->fifo) >= 0) {
  322. ff_filter_set_ready(ctx, 100);
  323. return 0;
  324. }
  325. }
  326. }
  327. if (s->eof && av_audio_fifo_size(s->fifo) <= 0) {
  328. ff_outlink_set_status(outlink, AVERROR_EOF, s->pts);
  329. return 0;
  330. }
  331. if (!s->eof)
  332. FF_FILTER_FORWARD_WANTED(outlink, inlink);
  333. return FFERROR_NOT_READY;
  334. }
  335. static int query_formats(AVFilterContext *ctx)
  336. {
  337. AVFilterFormats *formats;
  338. AVFilterChannelLayouts *layouts;
  339. static const enum AVSampleFormat sample_fmts[] = {
  340. AV_SAMPLE_FMT_FLTP,
  341. AV_SAMPLE_FMT_NONE
  342. };
  343. int ret;
  344. layouts = ff_all_channel_counts();
  345. if (!layouts)
  346. return AVERROR(ENOMEM);
  347. ret = ff_set_common_channel_layouts(ctx, layouts);
  348. if (ret < 0)
  349. return ret;
  350. formats = ff_make_format_list(sample_fmts);
  351. if (!formats)
  352. return AVERROR(ENOMEM);
  353. ret = ff_set_common_formats(ctx, formats);
  354. if (ret < 0)
  355. return ret;
  356. formats = ff_all_samplerates();
  357. if (!formats)
  358. return AVERROR(ENOMEM);
  359. return ff_set_common_samplerates(ctx, formats);
  360. }
  361. static av_cold void uninit(AVFilterContext *ctx)
  362. {
  363. AFFTFiltContext *s = ctx->priv;
  364. int i;
  365. av_fft_end(s->fft);
  366. av_fft_end(s->ifft);
  367. for (i = 0; i < s->nb_exprs; i++) {
  368. if (s->fft_data)
  369. av_freep(&s->fft_data[i]);
  370. if (s->fft_temp)
  371. av_freep(&s->fft_temp[i]);
  372. }
  373. av_freep(&s->fft_data);
  374. av_freep(&s->fft_temp);
  375. for (i = 0; i < s->nb_exprs; i++) {
  376. av_expr_free(s->real[i]);
  377. av_expr_free(s->imag[i]);
  378. }
  379. av_freep(&s->real);
  380. av_freep(&s->imag);
  381. av_frame_free(&s->buffer);
  382. av_freep(&s->window_func_lut);
  383. av_audio_fifo_free(s->fifo);
  384. }
  385. static const AVFilterPad inputs[] = {
  386. {
  387. .name = "default",
  388. .type = AVMEDIA_TYPE_AUDIO,
  389. .config_props = config_input,
  390. },
  391. { NULL }
  392. };
  393. static const AVFilterPad outputs[] = {
  394. {
  395. .name = "default",
  396. .type = AVMEDIA_TYPE_AUDIO,
  397. },
  398. { NULL }
  399. };
  400. AVFilter ff_af_afftfilt = {
  401. .name = "afftfilt",
  402. .description = NULL_IF_CONFIG_SMALL("Apply arbitrary expressions to samples in frequency domain."),
  403. .priv_size = sizeof(AFFTFiltContext),
  404. .priv_class = &afftfilt_class,
  405. .inputs = inputs,
  406. .outputs = outputs,
  407. .activate = activate,
  408. .query_formats = query_formats,
  409. .uninit = uninit,
  410. };