You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

996 lines
36KB

  1. /*
  2. * Copyright (c) 2010 Stefano Sabatini
  3. * Copyright (c) 2010 Baptiste Coudurier
  4. * Copyright (c) 2007 Bobby Bingham
  5. *
  6. * This file is part of FFmpeg.
  7. *
  8. * FFmpeg is free software; you can redistribute it and/or
  9. * modify it under the terms of the GNU Lesser General Public
  10. * License as published by the Free Software Foundation; either
  11. * version 2.1 of the License, or (at your option) any later version.
  12. *
  13. * FFmpeg is distributed in the hope that it will be useful,
  14. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16. * Lesser General Public License for more details.
  17. *
  18. * You should have received a copy of the GNU Lesser General Public
  19. * License along with FFmpeg; if not, write to the Free Software
  20. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21. */
  22. /**
  23. * @file
  24. * overlay one video on top of another
  25. */
  26. #include "avfilter.h"
  27. #include "formats.h"
  28. #include "libavutil/common.h"
  29. #include "libavutil/eval.h"
  30. #include "libavutil/avstring.h"
  31. #include "libavutil/pixdesc.h"
  32. #include "libavutil/imgutils.h"
  33. #include "libavutil/mathematics.h"
  34. #include "libavutil/opt.h"
  35. #include "libavutil/timestamp.h"
  36. #include "internal.h"
  37. #include "drawutils.h"
  38. #include "framesync.h"
  39. #include "video.h"
  40. static const char *const var_names[] = {
  41. "main_w", "W", ///< width of the main video
  42. "main_h", "H", ///< height of the main video
  43. "overlay_w", "w", ///< width of the overlay video
  44. "overlay_h", "h", ///< height of the overlay video
  45. "hsub",
  46. "vsub",
  47. "x",
  48. "y",
  49. "n", ///< number of frame
  50. "pos", ///< position in the file
  51. "t", ///< timestamp expressed in seconds
  52. NULL
  53. };
  54. enum var_name {
  55. VAR_MAIN_W, VAR_MW,
  56. VAR_MAIN_H, VAR_MH,
  57. VAR_OVERLAY_W, VAR_OW,
  58. VAR_OVERLAY_H, VAR_OH,
  59. VAR_HSUB,
  60. VAR_VSUB,
  61. VAR_X,
  62. VAR_Y,
  63. VAR_N,
  64. VAR_POS,
  65. VAR_T,
  66. VAR_VARS_NB
  67. };
  68. #define MAIN 0
  69. #define OVERLAY 1
  70. #define R 0
  71. #define G 1
  72. #define B 2
  73. #define A 3
  74. #define Y 0
  75. #define U 1
  76. #define V 2
  77. enum EvalMode {
  78. EVAL_MODE_INIT,
  79. EVAL_MODE_FRAME,
  80. EVAL_MODE_NB
  81. };
  82. enum OverlayFormat {
  83. OVERLAY_FORMAT_YUV420,
  84. OVERLAY_FORMAT_YUV422,
  85. OVERLAY_FORMAT_YUV444,
  86. OVERLAY_FORMAT_RGB,
  87. OVERLAY_FORMAT_GBRP,
  88. OVERLAY_FORMAT_AUTO,
  89. OVERLAY_FORMAT_NB
  90. };
  91. typedef struct OverlayContext {
  92. const AVClass *class;
  93. int x, y; ///< position of overlaid picture
  94. uint8_t main_is_packed_rgb;
  95. uint8_t main_rgba_map[4];
  96. uint8_t main_has_alpha;
  97. uint8_t overlay_is_packed_rgb;
  98. uint8_t overlay_rgba_map[4];
  99. uint8_t overlay_has_alpha;
  100. int format; ///< OverlayFormat
  101. int alpha_format;
  102. int eval_mode; ///< EvalMode
  103. FFFrameSync fs;
  104. int main_pix_step[4]; ///< steps per pixel for each plane of the main output
  105. int overlay_pix_step[4]; ///< steps per pixel for each plane of the overlay
  106. int hsub, vsub; ///< chroma subsampling values
  107. const AVPixFmtDescriptor *main_desc; ///< format descriptor for main input
  108. double var_values[VAR_VARS_NB];
  109. char *x_expr, *y_expr;
  110. AVExpr *x_pexpr, *y_pexpr;
  111. void (*blend_image)(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y);
  112. } OverlayContext;
  113. static av_cold void uninit(AVFilterContext *ctx)
  114. {
  115. OverlayContext *s = ctx->priv;
  116. ff_framesync_uninit(&s->fs);
  117. av_expr_free(s->x_pexpr); s->x_pexpr = NULL;
  118. av_expr_free(s->y_pexpr); s->y_pexpr = NULL;
  119. }
  120. static inline int normalize_xy(double d, int chroma_sub)
  121. {
  122. if (isnan(d))
  123. return INT_MAX;
  124. return (int)d & ~((1 << chroma_sub) - 1);
  125. }
  126. static void eval_expr(AVFilterContext *ctx)
  127. {
  128. OverlayContext *s = ctx->priv;
  129. s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values, NULL);
  130. s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr, s->var_values, NULL);
  131. s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values, NULL);
  132. s->x = normalize_xy(s->var_values[VAR_X], s->hsub);
  133. s->y = normalize_xy(s->var_values[VAR_Y], s->vsub);
  134. }
  135. static int set_expr(AVExpr **pexpr, const char *expr, const char *option, void *log_ctx)
  136. {
  137. int ret;
  138. AVExpr *old = NULL;
  139. if (*pexpr)
  140. old = *pexpr;
  141. ret = av_expr_parse(pexpr, expr, var_names,
  142. NULL, NULL, NULL, NULL, 0, log_ctx);
  143. if (ret < 0) {
  144. av_log(log_ctx, AV_LOG_ERROR,
  145. "Error when evaluating the expression '%s' for %s\n",
  146. expr, option);
  147. *pexpr = old;
  148. return ret;
  149. }
  150. av_expr_free(old);
  151. return 0;
  152. }
  153. static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
  154. char *res, int res_len, int flags)
  155. {
  156. OverlayContext *s = ctx->priv;
  157. int ret;
  158. if (!strcmp(cmd, "x"))
  159. ret = set_expr(&s->x_pexpr, args, cmd, ctx);
  160. else if (!strcmp(cmd, "y"))
  161. ret = set_expr(&s->y_pexpr, args, cmd, ctx);
  162. else
  163. ret = AVERROR(ENOSYS);
  164. if (ret < 0)
  165. return ret;
  166. if (s->eval_mode == EVAL_MODE_INIT) {
  167. eval_expr(ctx);
  168. av_log(ctx, AV_LOG_VERBOSE, "x:%f xi:%d y:%f yi:%d\n",
  169. s->var_values[VAR_X], s->x,
  170. s->var_values[VAR_Y], s->y);
  171. }
  172. return ret;
  173. }
  174. static const enum AVPixelFormat alpha_pix_fmts[] = {
  175. AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
  176. AV_PIX_FMT_ARGB, AV_PIX_FMT_ABGR, AV_PIX_FMT_RGBA,
  177. AV_PIX_FMT_BGRA, AV_PIX_FMT_GBRAP, AV_PIX_FMT_NONE
  178. };
  179. static int query_formats(AVFilterContext *ctx)
  180. {
  181. OverlayContext *s = ctx->priv;
  182. /* overlay formats contains alpha, for avoiding conversion with alpha information loss */
  183. static const enum AVPixelFormat main_pix_fmts_yuv420[] = {
  184. AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVA420P,
  185. AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,
  186. AV_PIX_FMT_NONE
  187. };
  188. static const enum AVPixelFormat overlay_pix_fmts_yuv420[] = {
  189. AV_PIX_FMT_YUVA420P, AV_PIX_FMT_NONE
  190. };
  191. static const enum AVPixelFormat main_pix_fmts_yuv422[] = {
  192. AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_NONE
  193. };
  194. static const enum AVPixelFormat overlay_pix_fmts_yuv422[] = {
  195. AV_PIX_FMT_YUVA422P, AV_PIX_FMT_NONE
  196. };
  197. static const enum AVPixelFormat main_pix_fmts_yuv444[] = {
  198. AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE
  199. };
  200. static const enum AVPixelFormat overlay_pix_fmts_yuv444[] = {
  201. AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE
  202. };
  203. static const enum AVPixelFormat main_pix_fmts_gbrp[] = {
  204. AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRAP, AV_PIX_FMT_NONE
  205. };
  206. static const enum AVPixelFormat overlay_pix_fmts_gbrp[] = {
  207. AV_PIX_FMT_GBRAP, AV_PIX_FMT_NONE
  208. };
  209. static const enum AVPixelFormat main_pix_fmts_rgb[] = {
  210. AV_PIX_FMT_ARGB, AV_PIX_FMT_RGBA,
  211. AV_PIX_FMT_ABGR, AV_PIX_FMT_BGRA,
  212. AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
  213. AV_PIX_FMT_NONE
  214. };
  215. static const enum AVPixelFormat overlay_pix_fmts_rgb[] = {
  216. AV_PIX_FMT_ARGB, AV_PIX_FMT_RGBA,
  217. AV_PIX_FMT_ABGR, AV_PIX_FMT_BGRA,
  218. AV_PIX_FMT_NONE
  219. };
  220. AVFilterFormats *main_formats = NULL;
  221. AVFilterFormats *overlay_formats = NULL;
  222. int ret;
  223. switch (s->format) {
  224. case OVERLAY_FORMAT_YUV420:
  225. if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv420)) ||
  226. !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420))) {
  227. ret = AVERROR(ENOMEM);
  228. goto fail;
  229. }
  230. break;
  231. case OVERLAY_FORMAT_YUV422:
  232. if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv422)) ||
  233. !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422))) {
  234. ret = AVERROR(ENOMEM);
  235. goto fail;
  236. }
  237. break;
  238. case OVERLAY_FORMAT_YUV444:
  239. if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv444)) ||
  240. !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444))) {
  241. ret = AVERROR(ENOMEM);
  242. goto fail;
  243. }
  244. break;
  245. case OVERLAY_FORMAT_RGB:
  246. if (!(main_formats = ff_make_format_list(main_pix_fmts_rgb)) ||
  247. !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb))) {
  248. ret = AVERROR(ENOMEM);
  249. goto fail;
  250. }
  251. break;
  252. case OVERLAY_FORMAT_GBRP:
  253. if (!(main_formats = ff_make_format_list(main_pix_fmts_gbrp)) ||
  254. !(overlay_formats = ff_make_format_list(overlay_pix_fmts_gbrp))) {
  255. ret = AVERROR(ENOMEM);
  256. goto fail;
  257. }
  258. break;
  259. case OVERLAY_FORMAT_AUTO:
  260. if (!(main_formats = ff_make_format_list(alpha_pix_fmts))) {
  261. ret = AVERROR(ENOMEM);
  262. goto fail;
  263. }
  264. break;
  265. default:
  266. av_assert0(0);
  267. }
  268. if (s->format == OVERLAY_FORMAT_AUTO) {
  269. ret = ff_set_common_formats(ctx, main_formats);
  270. if (ret < 0)
  271. goto fail;
  272. } else {
  273. if ((ret = ff_formats_ref(main_formats , &ctx->inputs[MAIN]->out_formats )) < 0 ||
  274. (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
  275. (ret = ff_formats_ref(main_formats , &ctx->outputs[MAIN]->in_formats )) < 0)
  276. goto fail;
  277. }
  278. return 0;
  279. fail:
  280. if (main_formats)
  281. av_freep(&main_formats->formats);
  282. av_freep(&main_formats);
  283. if (overlay_formats)
  284. av_freep(&overlay_formats->formats);
  285. av_freep(&overlay_formats);
  286. return ret;
  287. }
  288. static int config_input_overlay(AVFilterLink *inlink)
  289. {
  290. AVFilterContext *ctx = inlink->dst;
  291. OverlayContext *s = inlink->dst->priv;
  292. int ret;
  293. const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(inlink->format);
  294. av_image_fill_max_pixsteps(s->overlay_pix_step, NULL, pix_desc);
  295. /* Finish the configuration by evaluating the expressions
  296. now when both inputs are configured. */
  297. s->var_values[VAR_MAIN_W ] = s->var_values[VAR_MW] = ctx->inputs[MAIN ]->w;
  298. s->var_values[VAR_MAIN_H ] = s->var_values[VAR_MH] = ctx->inputs[MAIN ]->h;
  299. s->var_values[VAR_OVERLAY_W] = s->var_values[VAR_OW] = ctx->inputs[OVERLAY]->w;
  300. s->var_values[VAR_OVERLAY_H] = s->var_values[VAR_OH] = ctx->inputs[OVERLAY]->h;
  301. s->var_values[VAR_HSUB] = 1<<pix_desc->log2_chroma_w;
  302. s->var_values[VAR_VSUB] = 1<<pix_desc->log2_chroma_h;
  303. s->var_values[VAR_X] = NAN;
  304. s->var_values[VAR_Y] = NAN;
  305. s->var_values[VAR_N] = 0;
  306. s->var_values[VAR_T] = NAN;
  307. s->var_values[VAR_POS] = NAN;
  308. if ((ret = set_expr(&s->x_pexpr, s->x_expr, "x", ctx)) < 0 ||
  309. (ret = set_expr(&s->y_pexpr, s->y_expr, "y", ctx)) < 0)
  310. return ret;
  311. s->overlay_is_packed_rgb =
  312. ff_fill_rgba_map(s->overlay_rgba_map, inlink->format) >= 0;
  313. s->overlay_has_alpha = ff_fmt_is_in(inlink->format, alpha_pix_fmts);
  314. if (s->eval_mode == EVAL_MODE_INIT) {
  315. eval_expr(ctx);
  316. av_log(ctx, AV_LOG_VERBOSE, "x:%f xi:%d y:%f yi:%d\n",
  317. s->var_values[VAR_X], s->x,
  318. s->var_values[VAR_Y], s->y);
  319. }
  320. av_log(ctx, AV_LOG_VERBOSE,
  321. "main w:%d h:%d fmt:%s overlay w:%d h:%d fmt:%s\n",
  322. ctx->inputs[MAIN]->w, ctx->inputs[MAIN]->h,
  323. av_get_pix_fmt_name(ctx->inputs[MAIN]->format),
  324. ctx->inputs[OVERLAY]->w, ctx->inputs[OVERLAY]->h,
  325. av_get_pix_fmt_name(ctx->inputs[OVERLAY]->format));
  326. return 0;
  327. }
  328. static int config_output(AVFilterLink *outlink)
  329. {
  330. AVFilterContext *ctx = outlink->src;
  331. OverlayContext *s = ctx->priv;
  332. int ret;
  333. if ((ret = ff_framesync_init_dualinput(&s->fs, ctx)) < 0)
  334. return ret;
  335. outlink->w = ctx->inputs[MAIN]->w;
  336. outlink->h = ctx->inputs[MAIN]->h;
  337. outlink->time_base = ctx->inputs[MAIN]->time_base;
  338. return ff_framesync_configure(&s->fs);
  339. }
  340. // divide by 255 and round to nearest
  341. // apply a fast variant: (X+127)/255 = ((X+127)*257+257)>>16 = ((X+128)*257)>>16
  342. #define FAST_DIV255(x) ((((x) + 128) * 257) >> 16)
  343. // calculate the unpremultiplied alpha, applying the general equation:
  344. // alpha = alpha_overlay / ( (alpha_main + alpha_overlay) - (alpha_main * alpha_overlay) )
  345. // (((x) << 16) - ((x) << 9) + (x)) is a faster version of: 255 * 255 * x
  346. // ((((x) + (y)) << 8) - ((x) + (y)) - (y) * (x)) is a faster version of: 255 * (x + y)
  347. #define UNPREMULTIPLY_ALPHA(x, y) ((((x) << 16) - ((x) << 9) + (x)) / ((((x) + (y)) << 8) - ((x) + (y)) - (y) * (x)))
  348. /**
  349. * Blend image in src to destination buffer dst at position (x, y).
  350. */
  351. static av_always_inline void blend_image_packed_rgb(AVFilterContext *ctx,
  352. AVFrame *dst, const AVFrame *src,
  353. int main_has_alpha, int x, int y,
  354. int is_straight)
  355. {
  356. OverlayContext *s = ctx->priv;
  357. int i, imax, j, jmax;
  358. const int src_w = src->width;
  359. const int src_h = src->height;
  360. const int dst_w = dst->width;
  361. const int dst_h = dst->height;
  362. uint8_t alpha; ///< the amount of overlay to blend on to main
  363. const int dr = s->main_rgba_map[R];
  364. const int dg = s->main_rgba_map[G];
  365. const int db = s->main_rgba_map[B];
  366. const int da = s->main_rgba_map[A];
  367. const int dstep = s->main_pix_step[0];
  368. const int sr = s->overlay_rgba_map[R];
  369. const int sg = s->overlay_rgba_map[G];
  370. const int sb = s->overlay_rgba_map[B];
  371. const int sa = s->overlay_rgba_map[A];
  372. const int sstep = s->overlay_pix_step[0];
  373. uint8_t *S, *sp, *d, *dp;
  374. i = FFMAX(-y, 0);
  375. sp = src->data[0] + i * src->linesize[0];
  376. dp = dst->data[0] + (y+i) * dst->linesize[0];
  377. for (imax = FFMIN(-y + dst_h, src_h); i < imax; i++) {
  378. j = FFMAX(-x, 0);
  379. S = sp + j * sstep;
  380. d = dp + (x+j) * dstep;
  381. for (jmax = FFMIN(-x + dst_w, src_w); j < jmax; j++) {
  382. alpha = S[sa];
  383. // if the main channel has an alpha channel, alpha has to be calculated
  384. // to create an un-premultiplied (straight) alpha value
  385. if (main_has_alpha && alpha != 0 && alpha != 255) {
  386. uint8_t alpha_d = d[da];
  387. alpha = UNPREMULTIPLY_ALPHA(alpha, alpha_d);
  388. }
  389. switch (alpha) {
  390. case 0:
  391. break;
  392. case 255:
  393. d[dr] = S[sr];
  394. d[dg] = S[sg];
  395. d[db] = S[sb];
  396. break;
  397. default:
  398. // main_value = main_value * (1 - alpha) + overlay_value * alpha
  399. // since alpha is in the range 0-255, the result must divided by 255
  400. d[dr] = is_straight ? FAST_DIV255(d[dr] * (255 - alpha) + S[sr] * alpha) :
  401. FFMIN(FAST_DIV255(d[dr] * (255 - alpha)) + S[sr], 255);
  402. d[dg] = is_straight ? FAST_DIV255(d[dg] * (255 - alpha) + S[sg] * alpha) :
  403. FFMIN(FAST_DIV255(d[dg] * (255 - alpha)) + S[sg], 255);
  404. d[db] = is_straight ? FAST_DIV255(d[db] * (255 - alpha) + S[sb] * alpha) :
  405. FFMIN(FAST_DIV255(d[db] * (255 - alpha)) + S[sb], 255);
  406. }
  407. if (main_has_alpha) {
  408. switch (alpha) {
  409. case 0:
  410. break;
  411. case 255:
  412. d[da] = S[sa];
  413. break;
  414. default:
  415. // apply alpha compositing: main_alpha += (1-main_alpha) * overlay_alpha
  416. d[da] += FAST_DIV255((255 - d[da]) * S[sa]);
  417. }
  418. }
  419. d += dstep;
  420. S += sstep;
  421. }
  422. dp += dst->linesize[0];
  423. sp += src->linesize[0];
  424. }
  425. }
  426. static av_always_inline void blend_plane(AVFilterContext *ctx,
  427. AVFrame *dst, const AVFrame *src,
  428. int src_w, int src_h,
  429. int dst_w, int dst_h,
  430. int i, int hsub, int vsub,
  431. int x, int y,
  432. int main_has_alpha,
  433. int dst_plane,
  434. int dst_offset,
  435. int dst_step,
  436. int straight,
  437. int yuv)
  438. {
  439. int src_wp = AV_CEIL_RSHIFT(src_w, hsub);
  440. int src_hp = AV_CEIL_RSHIFT(src_h, vsub);
  441. int dst_wp = AV_CEIL_RSHIFT(dst_w, hsub);
  442. int dst_hp = AV_CEIL_RSHIFT(dst_h, vsub);
  443. int yp = y>>vsub;
  444. int xp = x>>hsub;
  445. uint8_t *s, *sp, *d, *dp, *dap, *a, *da, *ap;
  446. int jmax, j, k, kmax;
  447. j = FFMAX(-yp, 0);
  448. sp = src->data[i] + j * src->linesize[i];
  449. dp = dst->data[dst_plane]
  450. + (yp+j) * dst->linesize[dst_plane]
  451. + dst_offset;
  452. ap = src->data[3] + (j<<vsub) * src->linesize[3];
  453. dap = dst->data[3] + ((yp+j) << vsub) * dst->linesize[3];
  454. for (jmax = FFMIN(-yp + dst_hp, src_hp); j < jmax; j++) {
  455. k = FFMAX(-xp, 0);
  456. d = dp + (xp+k) * dst_step;
  457. s = sp + k;
  458. a = ap + (k<<hsub);
  459. da = dap + ((xp+k) << hsub);
  460. for (kmax = FFMIN(-xp + dst_wp, src_wp); k < kmax; k++) {
  461. int alpha_v, alpha_h, alpha;
  462. // average alpha for color components, improve quality
  463. if (hsub && vsub && j+1 < src_hp && k+1 < src_wp) {
  464. alpha = (a[0] + a[src->linesize[3]] +
  465. a[1] + a[src->linesize[3]+1]) >> 2;
  466. } else if (hsub || vsub) {
  467. alpha_h = hsub && k+1 < src_wp ?
  468. (a[0] + a[1]) >> 1 : a[0];
  469. alpha_v = vsub && j+1 < src_hp ?
  470. (a[0] + a[src->linesize[3]]) >> 1 : a[0];
  471. alpha = (alpha_v + alpha_h) >> 1;
  472. } else
  473. alpha = a[0];
  474. // if the main channel has an alpha channel, alpha has to be calculated
  475. // to create an un-premultiplied (straight) alpha value
  476. if (main_has_alpha && alpha != 0 && alpha != 255) {
  477. // average alpha for color components, improve quality
  478. uint8_t alpha_d;
  479. if (hsub && vsub && j+1 < src_hp && k+1 < src_wp) {
  480. alpha_d = (da[0] + da[dst->linesize[3]] +
  481. da[1] + da[dst->linesize[3]+1]) >> 2;
  482. } else if (hsub || vsub) {
  483. alpha_h = hsub && k+1 < src_wp ?
  484. (da[0] + da[1]) >> 1 : da[0];
  485. alpha_v = vsub && j+1 < src_hp ?
  486. (da[0] + da[dst->linesize[3]]) >> 1 : da[0];
  487. alpha_d = (alpha_v + alpha_h) >> 1;
  488. } else
  489. alpha_d = da[0];
  490. alpha = UNPREMULTIPLY_ALPHA(alpha, alpha_d);
  491. }
  492. if (straight) {
  493. *d = FAST_DIV255(*d * (255 - alpha) + *s * alpha);
  494. } else {
  495. if (i && yuv)
  496. *d = av_clip(FAST_DIV255((*d - 128) * (255 - alpha)) + *s - 128, -128, 128) + 128;
  497. else
  498. *d = FFMIN(FAST_DIV255(*d * (255 - alpha)) + *s, 255);
  499. }
  500. s++;
  501. d += dst_step;
  502. da += 1 << hsub;
  503. a += 1 << hsub;
  504. }
  505. dp += dst->linesize[dst_plane];
  506. sp += src->linesize[i];
  507. ap += (1 << vsub) * src->linesize[3];
  508. dap += (1 << vsub) * dst->linesize[3];
  509. }
  510. }
  511. static inline void alpha_composite(const AVFrame *src, const AVFrame *dst,
  512. int src_w, int src_h,
  513. int dst_w, int dst_h,
  514. int x, int y)
  515. {
  516. uint8_t alpha; ///< the amount of overlay to blend on to main
  517. uint8_t *s, *sa, *d, *da;
  518. int i, imax, j, jmax;
  519. i = FFMAX(-y, 0);
  520. sa = src->data[3] + i * src->linesize[3];
  521. da = dst->data[3] + (y+i) * dst->linesize[3];
  522. for (imax = FFMIN(-y + dst_h, src_h); i < imax; i++) {
  523. j = FFMAX(-x, 0);
  524. s = sa + j;
  525. d = da + x+j;
  526. for (jmax = FFMIN(-x + dst_w, src_w); j < jmax; j++) {
  527. alpha = *s;
  528. if (alpha != 0 && alpha != 255) {
  529. uint8_t alpha_d = *d;
  530. alpha = UNPREMULTIPLY_ALPHA(alpha, alpha_d);
  531. }
  532. switch (alpha) {
  533. case 0:
  534. break;
  535. case 255:
  536. *d = *s;
  537. break;
  538. default:
  539. // apply alpha compositing: main_alpha += (1-main_alpha) * overlay_alpha
  540. *d += FAST_DIV255((255 - *d) * *s);
  541. }
  542. d += 1;
  543. s += 1;
  544. }
  545. da += dst->linesize[3];
  546. sa += src->linesize[3];
  547. }
  548. }
  549. static av_always_inline void blend_image_yuv(AVFilterContext *ctx,
  550. AVFrame *dst, const AVFrame *src,
  551. int hsub, int vsub,
  552. int main_has_alpha,
  553. int x, int y,
  554. int is_straight)
  555. {
  556. OverlayContext *s = ctx->priv;
  557. const int src_w = src->width;
  558. const int src_h = src->height;
  559. const int dst_w = dst->width;
  560. const int dst_h = dst->height;
  561. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 0, 0, 0, x, y, main_has_alpha,
  562. s->main_desc->comp[0].plane, s->main_desc->comp[0].offset, s->main_desc->comp[0].step, is_straight, 1);
  563. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 1, hsub, vsub, x, y, main_has_alpha,
  564. s->main_desc->comp[1].plane, s->main_desc->comp[1].offset, s->main_desc->comp[1].step, is_straight, 1);
  565. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 2, hsub, vsub, x, y, main_has_alpha,
  566. s->main_desc->comp[2].plane, s->main_desc->comp[2].offset, s->main_desc->comp[2].step, is_straight, 1);
  567. if (main_has_alpha)
  568. alpha_composite(src, dst, src_w, src_h, dst_w, dst_h, x, y);
  569. }
  570. static av_always_inline void blend_image_planar_rgb(AVFilterContext *ctx,
  571. AVFrame *dst, const AVFrame *src,
  572. int hsub, int vsub,
  573. int main_has_alpha,
  574. int x, int y,
  575. int is_straight)
  576. {
  577. OverlayContext *s = ctx->priv;
  578. const int src_w = src->width;
  579. const int src_h = src->height;
  580. const int dst_w = dst->width;
  581. const int dst_h = dst->height;
  582. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 0, 0, 0, x, y, main_has_alpha,
  583. s->main_desc->comp[1].plane, s->main_desc->comp[1].offset, s->main_desc->comp[1].step, is_straight, 0);
  584. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 1, hsub, vsub, x, y, main_has_alpha,
  585. s->main_desc->comp[2].plane, s->main_desc->comp[2].offset, s->main_desc->comp[2].step, is_straight, 0);
  586. blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 2, hsub, vsub, x, y, main_has_alpha,
  587. s->main_desc->comp[0].plane, s->main_desc->comp[0].offset, s->main_desc->comp[0].step, is_straight, 0);
  588. if (main_has_alpha)
  589. alpha_composite(src, dst, src_w, src_h, dst_w, dst_h, x, y);
  590. }
  591. static void blend_image_yuv420(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  592. {
  593. blend_image_yuv(ctx, dst, src, 1, 1, 0, x, y, 1);
  594. }
  595. static void blend_image_yuva420(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  596. {
  597. blend_image_yuv(ctx, dst, src, 1, 1, 1, x, y, 1);
  598. }
  599. static void blend_image_yuv422(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  600. {
  601. blend_image_yuv(ctx, dst, src, 1, 0, 0, x, y, 1);
  602. }
  603. static void blend_image_yuva422(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  604. {
  605. blend_image_yuv(ctx, dst, src, 1, 0, 1, x, y, 1);
  606. }
  607. static void blend_image_yuv444(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  608. {
  609. blend_image_yuv(ctx, dst, src, 0, 0, 0, x, y, 1);
  610. }
  611. static void blend_image_yuva444(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  612. {
  613. blend_image_yuv(ctx, dst, src, 0, 0, 1, x, y, 1);
  614. }
  615. static void blend_image_gbrp(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  616. {
  617. blend_image_planar_rgb(ctx, dst, src, 0, 0, 0, x, y, 1);
  618. }
  619. static void blend_image_gbrap(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  620. {
  621. blend_image_planar_rgb(ctx, dst, src, 0, 0, 1, x, y, 1);
  622. }
  623. static void blend_image_yuv420_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  624. {
  625. blend_image_yuv(ctx, dst, src, 1, 1, 0, x, y, 0);
  626. }
  627. static void blend_image_yuva420_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  628. {
  629. blend_image_yuv(ctx, dst, src, 1, 1, 1, x, y, 0);
  630. }
  631. static void blend_image_yuv422_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  632. {
  633. blend_image_yuv(ctx, dst, src, 1, 0, 0, x, y, 0);
  634. }
  635. static void blend_image_yuva422_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  636. {
  637. blend_image_yuv(ctx, dst, src, 1, 0, 1, x, y, 0);
  638. }
  639. static void blend_image_yuv444_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  640. {
  641. blend_image_yuv(ctx, dst, src, 0, 0, 0, x, y, 0);
  642. }
  643. static void blend_image_yuva444_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  644. {
  645. blend_image_yuv(ctx, dst, src, 0, 0, 1, x, y, 0);
  646. }
  647. static void blend_image_gbrp_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  648. {
  649. blend_image_planar_rgb(ctx, dst, src, 0, 0, 0, x, y, 0);
  650. }
  651. static void blend_image_gbrap_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  652. {
  653. blend_image_planar_rgb(ctx, dst, src, 0, 0, 1, x, y, 0);
  654. }
  655. static void blend_image_rgb(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  656. {
  657. blend_image_packed_rgb(ctx, dst, src, 0, x, y, 1);
  658. }
  659. static void blend_image_rgba(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  660. {
  661. blend_image_packed_rgb(ctx, dst, src, 1, x, y, 1);
  662. }
  663. static void blend_image_rgb_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  664. {
  665. blend_image_packed_rgb(ctx, dst, src, 0, x, y, 0);
  666. }
  667. static void blend_image_rgba_pm(AVFilterContext *ctx, AVFrame *dst, const AVFrame *src, int x, int y)
  668. {
  669. blend_image_packed_rgb(ctx, dst, src, 1, x, y, 0);
  670. }
  671. static int config_input_main(AVFilterLink *inlink)
  672. {
  673. OverlayContext *s = inlink->dst->priv;
  674. const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(inlink->format);
  675. av_image_fill_max_pixsteps(s->main_pix_step, NULL, pix_desc);
  676. s->hsub = pix_desc->log2_chroma_w;
  677. s->vsub = pix_desc->log2_chroma_h;
  678. s->main_desc = pix_desc;
  679. s->main_is_packed_rgb =
  680. ff_fill_rgba_map(s->main_rgba_map, inlink->format) >= 0;
  681. s->main_has_alpha = ff_fmt_is_in(inlink->format, alpha_pix_fmts);
  682. switch (s->format) {
  683. case OVERLAY_FORMAT_YUV420:
  684. s->blend_image = s->main_has_alpha ? blend_image_yuva420 : blend_image_yuv420;
  685. break;
  686. case OVERLAY_FORMAT_YUV422:
  687. s->blend_image = s->main_has_alpha ? blend_image_yuva422 : blend_image_yuv422;
  688. break;
  689. case OVERLAY_FORMAT_YUV444:
  690. s->blend_image = s->main_has_alpha ? blend_image_yuva444 : blend_image_yuv444;
  691. break;
  692. case OVERLAY_FORMAT_RGB:
  693. s->blend_image = s->main_has_alpha ? blend_image_rgba : blend_image_rgb;
  694. break;
  695. case OVERLAY_FORMAT_GBRP:
  696. s->blend_image = s->main_has_alpha ? blend_image_gbrap : blend_image_gbrp;
  697. break;
  698. case OVERLAY_FORMAT_AUTO:
  699. switch (inlink->format) {
  700. case AV_PIX_FMT_YUVA420P:
  701. s->blend_image = blend_image_yuva420;
  702. break;
  703. case AV_PIX_FMT_YUVA422P:
  704. s->blend_image = blend_image_yuva422;
  705. break;
  706. case AV_PIX_FMT_YUVA444P:
  707. s->blend_image = blend_image_yuva444;
  708. break;
  709. case AV_PIX_FMT_ARGB:
  710. case AV_PIX_FMT_RGBA:
  711. case AV_PIX_FMT_BGRA:
  712. case AV_PIX_FMT_ABGR:
  713. s->blend_image = blend_image_rgba;
  714. break;
  715. case AV_PIX_FMT_GBRAP:
  716. s->blend_image = blend_image_gbrap;
  717. break;
  718. default:
  719. av_assert0(0);
  720. break;
  721. }
  722. break;
  723. }
  724. if (!s->alpha_format)
  725. return 0;
  726. switch (s->format) {
  727. case OVERLAY_FORMAT_YUV420:
  728. s->blend_image = s->main_has_alpha ? blend_image_yuva420_pm : blend_image_yuv420_pm;
  729. break;
  730. case OVERLAY_FORMAT_YUV422:
  731. s->blend_image = s->main_has_alpha ? blend_image_yuva422_pm : blend_image_yuv422_pm;
  732. break;
  733. case OVERLAY_FORMAT_YUV444:
  734. s->blend_image = s->main_has_alpha ? blend_image_yuva444_pm : blend_image_yuv444_pm;
  735. break;
  736. case OVERLAY_FORMAT_RGB:
  737. s->blend_image = s->main_has_alpha ? blend_image_rgba_pm : blend_image_rgb_pm;
  738. break;
  739. case OVERLAY_FORMAT_GBRP:
  740. s->blend_image = s->main_has_alpha ? blend_image_gbrap_pm : blend_image_gbrp_pm;
  741. break;
  742. case OVERLAY_FORMAT_AUTO:
  743. switch (inlink->format) {
  744. case AV_PIX_FMT_YUVA420P:
  745. s->blend_image = blend_image_yuva420_pm;
  746. break;
  747. case AV_PIX_FMT_YUVA422P:
  748. s->blend_image = blend_image_yuva422_pm;
  749. break;
  750. case AV_PIX_FMT_YUVA444P:
  751. s->blend_image = blend_image_yuva444_pm;
  752. break;
  753. case AV_PIX_FMT_ARGB:
  754. case AV_PIX_FMT_RGBA:
  755. case AV_PIX_FMT_BGRA:
  756. case AV_PIX_FMT_ABGR:
  757. s->blend_image = blend_image_rgba_pm;
  758. break;
  759. case AV_PIX_FMT_GBRAP:
  760. s->blend_image = blend_image_gbrap_pm;
  761. break;
  762. default:
  763. av_assert0(0);
  764. break;
  765. }
  766. break;
  767. }
  768. return 0;
  769. }
  770. static int do_blend(FFFrameSync *fs)
  771. {
  772. AVFilterContext *ctx = fs->parent;
  773. AVFrame *mainpic, *second;
  774. OverlayContext *s = ctx->priv;
  775. AVFilterLink *inlink = ctx->inputs[0];
  776. int ret;
  777. ret = ff_framesync_dualinput_get_writable(fs, &mainpic, &second);
  778. if (ret < 0)
  779. return ret;
  780. if (!second)
  781. return ff_filter_frame(ctx->outputs[0], mainpic);
  782. if (s->eval_mode == EVAL_MODE_FRAME) {
  783. int64_t pos = mainpic->pkt_pos;
  784. s->var_values[VAR_N] = inlink->frame_count_out;
  785. s->var_values[VAR_T] = mainpic->pts == AV_NOPTS_VALUE ?
  786. NAN : mainpic->pts * av_q2d(inlink->time_base);
  787. s->var_values[VAR_POS] = pos == -1 ? NAN : pos;
  788. s->var_values[VAR_OVERLAY_W] = s->var_values[VAR_OW] = second->width;
  789. s->var_values[VAR_OVERLAY_H] = s->var_values[VAR_OH] = second->height;
  790. s->var_values[VAR_MAIN_W ] = s->var_values[VAR_MW] = mainpic->width;
  791. s->var_values[VAR_MAIN_H ] = s->var_values[VAR_MH] = mainpic->height;
  792. eval_expr(ctx);
  793. av_log(ctx, AV_LOG_DEBUG, "n:%f t:%f pos:%f x:%f xi:%d y:%f yi:%d\n",
  794. s->var_values[VAR_N], s->var_values[VAR_T], s->var_values[VAR_POS],
  795. s->var_values[VAR_X], s->x,
  796. s->var_values[VAR_Y], s->y);
  797. }
  798. if (s->x < mainpic->width && s->x + second->width >= 0 ||
  799. s->y < mainpic->height && s->y + second->height >= 0)
  800. s->blend_image(ctx, mainpic, second, s->x, s->y);
  801. return ff_filter_frame(ctx->outputs[0], mainpic);
  802. }
  803. static av_cold int init(AVFilterContext *ctx)
  804. {
  805. OverlayContext *s = ctx->priv;
  806. s->fs.on_event = do_blend;
  807. return 0;
  808. }
  809. static int activate(AVFilterContext *ctx)
  810. {
  811. OverlayContext *s = ctx->priv;
  812. return ff_framesync_activate(&s->fs);
  813. }
  814. #define OFFSET(x) offsetof(OverlayContext, x)
  815. #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
  816. static const AVOption overlay_options[] = {
  817. { "x", "set the x expression", OFFSET(x_expr), AV_OPT_TYPE_STRING, {.str = "0"}, CHAR_MIN, CHAR_MAX, FLAGS },
  818. { "y", "set the y expression", OFFSET(y_expr), AV_OPT_TYPE_STRING, {.str = "0"}, CHAR_MIN, CHAR_MAX, FLAGS },
  819. { "eof_action", "Action to take when encountering EOF from secondary input ",
  820. OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
  821. EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" },
  822. { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
  823. { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
  824. { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" },
  825. { "eval", "specify when to evaluate expressions", OFFSET(eval_mode), AV_OPT_TYPE_INT, {.i64 = EVAL_MODE_FRAME}, 0, EVAL_MODE_NB-1, FLAGS, "eval" },
  826. { "init", "eval expressions once during initialization", 0, AV_OPT_TYPE_CONST, {.i64=EVAL_MODE_INIT}, .flags = FLAGS, .unit = "eval" },
  827. { "frame", "eval expressions per-frame", 0, AV_OPT_TYPE_CONST, {.i64=EVAL_MODE_FRAME}, .flags = FLAGS, .unit = "eval" },
  828. { "shortest", "force termination when the shortest input terminates", OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
  829. { "format", "set output format", OFFSET(format), AV_OPT_TYPE_INT, {.i64=OVERLAY_FORMAT_YUV420}, 0, OVERLAY_FORMAT_NB-1, FLAGS, "format" },
  830. { "yuv420", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_YUV420}, .flags = FLAGS, .unit = "format" },
  831. { "yuv422", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_YUV422}, .flags = FLAGS, .unit = "format" },
  832. { "yuv444", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_YUV444}, .flags = FLAGS, .unit = "format" },
  833. { "rgb", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_RGB}, .flags = FLAGS, .unit = "format" },
  834. { "gbrp", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_GBRP}, .flags = FLAGS, .unit = "format" },
  835. { "auto", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_AUTO}, .flags = FLAGS, .unit = "format" },
  836. { "repeatlast", "repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
  837. { "alpha", "alpha format", OFFSET(alpha_format), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, FLAGS, "alpha_format" },
  838. { "straight", "", 0, AV_OPT_TYPE_CONST, {.i64=0}, .flags = FLAGS, .unit = "alpha_format" },
  839. { "premultiplied", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, .flags = FLAGS, .unit = "alpha_format" },
  840. { NULL }
  841. };
  842. FRAMESYNC_DEFINE_CLASS(overlay, OverlayContext, fs);
  843. static const AVFilterPad avfilter_vf_overlay_inputs[] = {
  844. {
  845. .name = "main",
  846. .type = AVMEDIA_TYPE_VIDEO,
  847. .config_props = config_input_main,
  848. },
  849. {
  850. .name = "overlay",
  851. .type = AVMEDIA_TYPE_VIDEO,
  852. .config_props = config_input_overlay,
  853. },
  854. { NULL }
  855. };
  856. static const AVFilterPad avfilter_vf_overlay_outputs[] = {
  857. {
  858. .name = "default",
  859. .type = AVMEDIA_TYPE_VIDEO,
  860. .config_props = config_output,
  861. },
  862. { NULL }
  863. };
  864. AVFilter ff_vf_overlay = {
  865. .name = "overlay",
  866. .description = NULL_IF_CONFIG_SMALL("Overlay a video source on top of the input."),
  867. .preinit = overlay_framesync_preinit,
  868. .init = init,
  869. .uninit = uninit,
  870. .priv_size = sizeof(OverlayContext),
  871. .priv_class = &overlay_class,
  872. .query_formats = query_formats,
  873. .activate = activate,
  874. .process_command = process_command,
  875. .inputs = avfilter_vf_overlay_inputs,
  876. .outputs = avfilter_vf_overlay_outputs,
  877. .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL,
  878. };