falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
44689 Commits
32 Branches
283MB
Tree: c5714b51aa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c5714b51aa'
${ noResults }
FFmpeg/libavcodec/hevcdsp_template.c

1441 lines
60KB
Raw Blame History 

  1. /*

  2.  * HEVC video decoder

  3.  *

  4.  * Copyright (C) 2012 - 2013 Guillaume Martres

  5.  *

  6.  * This file is part of Libav.

  7.  *

  8.  * Libav is free software; you can redistribute it and/or

  9.  * modify it under the terms of the GNU Lesser General Public

  10.  * License as published by the Free Software Foundation; either

  11.  * version 2.1 of the License, or (at your option) any later version.

  12.  *

  13.  * Libav is distributed in the hope that it will be useful,

  14.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  15.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  16.  * Lesser General Public License for more details.

  17.  *

  18.  * You should have received a copy of the GNU Lesser General Public

  19.  * License along with Libav; if not, write to the Free Software

  20.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  21.  */

  22. 

  23. #include "get_bits.h"

  24. #include "hevcdec.h"

  25. 

  26. #include "bit_depth_template.c"

  27. 

  28. static void FUNC(put_pcm)(uint8_t *_dst, ptrdiff_t stride, int size,

  29.                           GetBitContext *gb, int pcm_bit_depth)

  30. {

  31.     int x, y;

  32.     pixel *dst = (pixel *)_dst;

  33. 

  34.     stride /= sizeof(pixel);

  35. 

  36.     for (y = 0; y < size; y++) {

  37.         for (x = 0; x < size; x++)

  38.             dst[x] = get_bits(gb, pcm_bit_depth) << (BIT_DEPTH - pcm_bit_depth);

  39.         dst += stride;

  40.     }

  41. }

  42. 

  43. static av_always_inline void FUNC(add_residual)(uint8_t *_dst, int16_t *res,

  44.                                                 ptrdiff_t stride, int size)

  45. {

  46.     int x, y;

  47.     pixel *dst = (pixel *)_dst;

  48. 

  49.     stride /= sizeof(pixel);

  50. 

  51.     for (y = 0; y < size; y++) {

  52.         for (x = 0; x < size; x++) {

  53.             dst[x] = av_clip_pixel(dst[x] + *res);

  54.             res++;

  55.         }

  56.         dst += stride;

  57.     }

  58. }

  59. 

  60. static void FUNC(add_residual4x4)(uint8_t *_dst, int16_t *res,

  61.                                   ptrdiff_t stride)

  62. {

  63.     FUNC(add_residual)(_dst, res, stride, 4);

  64. }

  65. 

  66. static void FUNC(add_residual8x8)(uint8_t *_dst, int16_t *res,

  67.                                   ptrdiff_t stride)

  68. {

  69.     FUNC(add_residual)(_dst, res, stride, 8);

  70. }

  71. 

  72. static void FUNC(add_residual16x16)(uint8_t *_dst, int16_t *res,

  73.                                     ptrdiff_t stride)

  74. {

  75.     FUNC(add_residual)(_dst, res, stride, 16);

  76. }

  77. 

  78. static void FUNC(add_residual32x32)(uint8_t *_dst, int16_t *res,

  79.                                     ptrdiff_t stride)

  80. {

  81.     FUNC(add_residual)(_dst, res, stride, 32);

  82. }

  83. 

  84. static void FUNC(dequant)(int16_t *coeffs)

  85. {

  86.     int shift  = 13 - BIT_DEPTH;

  87. #if BIT_DEPTH <= 13

  88.     int offset = 1 << (shift - 1);

  89. #else

  90.     int offset = 0;

  91. #endif

  92.     int x, y;

  93. 

  94.     for (y = 0; y < 4 * 4; y += 4) {

  95.         for (x = 0; x < 4; x++)

  96.             coeffs[y + x] = (coeffs[y + x] + offset) >> shift;

  97.     }

  98. }

  99. 

  100. #define SET(dst, x)   (dst) = (x)

  101. #define SCALE(dst, x) (dst) = av_clip_int16(((x) + add) >> shift)

  102. 

  103. #define TR_4x4_LUMA(dst, src, step, assign)                             \

  104.     do {                                                                \

  105.         int c0 = src[0 * step] + src[2 * step];                         \

  106.         int c1 = src[2 * step] + src[3 * step];                         \

  107.         int c2 = src[0 * step] - src[3 * step];                         \

  108.         int c3 = 74 * src[1 * step];                                    \

  109.                                                                         \

  110.         assign(dst[2 * step], 74 * (src[0 * step] -                     \

  111.                                     src[2 * step] +                     \

  112.                                     src[3 * step]));                    \

  113.         assign(dst[0 * step], 29 * c0 + 55 * c1 + c3);                  \

  114.         assign(dst[1 * step], 55 * c2 - 29 * c1 + c3);                  \

  115.         assign(dst[3 * step], 55 * c0 + 29 * c2 - c3);                  \

  116.     } while (0)

  117. 

  118. static void FUNC(transform_4x4_luma)(int16_t *coeffs)

  119. {

  120.     int i;

  121.     int shift    = 7;

  122.     int add      = 1 << (shift - 1);

  123.     int16_t *src = coeffs;

  124. 

  125.     for (i = 0; i < 4; i++) {

  126.         TR_4x4_LUMA(src, src, 4, SCALE);

  127.         src++;

  128.     }

  129. 

  130.     shift = 20 - BIT_DEPTH;

  131.     add   = 1 << (shift - 1);

  132.     for (i = 0; i < 4; i++) {

  133.         TR_4x4_LUMA(coeffs, coeffs, 1, SCALE);

  134.         coeffs += 4;

  135.     }

  136. }

  137. 

  138. #undef TR_4x4_LUMA

  139. 

  140. #define TR_4(dst, src, dstep, sstep, assign, end)                       \

  141.     do {                                                                \

  142.         const int e0 = transform[8 * 0][0] * src[0 * sstep] +           \

  143.                        transform[8 * 2][0] * src[2 * sstep];            \

  144.         const int e1 = transform[8 * 0][1] * src[0 * sstep] +           \

  145.                        transform[8 * 2][1] * src[2 * sstep];            \

  146.         const int o0 = transform[8 * 1][0] * src[1 * sstep] +           \

  147.                        transform[8 * 3][0] * src[3 * sstep];            \

  148.         const int o1 = transform[8 * 1][1] * src[1 * sstep] +           \

  149.                        transform[8 * 3][1] * src[3 * sstep];            \

  150.                                                                         \

  151.         assign(dst[0 * dstep], e0 + o0);                                \

  152.         assign(dst[1 * dstep], e1 + o1);                                \

  153.         assign(dst[2 * dstep], e1 - o1);                                \

  154.         assign(dst[3 * dstep], e0 - o0);                                \

  155.     } while (0)

  156. 

  157. #define TR_8(dst, src, dstep, sstep, assign, end)                 \

  158.     do {                                                          \

  159.         int i, j;                                                 \

  160.         int e_8[4];                                               \

  161.         int o_8[4] = { 0 };                                       \

  162.         for (i = 0; i < 4; i++)                                   \

  163.             for (j = 1; j < end; j += 2)                          \

  164.                 o_8[i] += transform[4 * j][i] * src[j * sstep];   \

  165.         TR_4(e_8, src, 1, 2 * sstep, SET, 4);                     \

  166.                                                                   \

  167.         for (i = 0; i < 4; i++) {                                 \

  168.             assign(dst[i * dstep], e_8[i] + o_8[i]);              \

  169.             assign(dst[(7 - i) * dstep], e_8[i] - o_8[i]);        \

  170.         }                                                         \

  171.     } while (0)

  172. 

  173. #define TR_16(dst, src, dstep, sstep, assign, end)                \

  174.     do {                                                          \

  175.         int i, j;                                                 \

  176.         int e_16[8];                                              \

  177.         int o_16[8] = { 0 };                                      \

  178.         for (i = 0; i < 8; i++)                                   \

  179.             for (j = 1; j < end; j += 2)                          \

  180.                 o_16[i] += transform[2 * j][i] * src[j * sstep];  \

  181.         TR_8(e_16, src, 1, 2 * sstep, SET, 8);                    \

  182.                                                                   \

  183.         for (i = 0; i < 8; i++) {                                 \

  184.             assign(dst[i * dstep], e_16[i] + o_16[i]);            \

  185.             assign(dst[(15 - i) * dstep], e_16[i] - o_16[i]);     \

  186.         }                                                         \

  187.     } while (0)

  188. 

  189. #define TR_32(dst, src, dstep, sstep, assign, end)                \

  190.     do {                                                          \

  191.         int i, j;                                                 \

  192.         int e_32[16];                                             \

  193.         int o_32[16] = { 0 };                                     \

  194.         for (i = 0; i < 16; i++)                                  \

  195.             for (j = 1; j < end; j += 2)                          \

  196.                 o_32[i] += transform[j][i] * src[j * sstep];      \

  197.         TR_16(e_32, src, 1, 2 * sstep, SET, end / 2);             \

  198.                                                                   \

  199.         for (i = 0; i < 16; i++) {                                \

  200.             assign(dst[i * dstep], e_32[i] + o_32[i]);            \

  201.             assign(dst[(31 - i) * dstep], e_32[i] - o_32[i]);     \

  202.         }                                                         \

  203.     } while (0)

  204. 

  205. #define IDCT_VAR4(H)                                              \

  206.     int limit2 = FFMIN(col_limit + 4, H)

  207. #define IDCT_VAR8(H)                                              \

  208.     int limit  = FFMIN(col_limit, H);                             \

  209.     int limit2 = FFMIN(col_limit + 4, H)

  210. #define IDCT_VAR16(H)   IDCT_VAR8(H)

  211. #define IDCT_VAR32(H)   IDCT_VAR8(H)

  212. 

  213. #define IDCT(H)                                                   \

  214. static void FUNC(idct_ ## H ## x ## H )(int16_t *coeffs,          \

  215.                                         int col_limit)            \

  216. {                                                                 \

  217.     int i;                                                        \

  218.     int      shift = 7;                                           \

  219.     int      add   = 1 << (shift - 1);                            \

  220.     int16_t *src   = coeffs;                                      \

  221.     IDCT_VAR ## H(H);                                             \

  222.                                                                   \

  223.     for (i = 0; i < H; i++) {                                     \

  224.         TR_ ## H(src, src, H, H, SCALE, limit2);                  \

  225.         if (limit2 < H && i%4 == 0 && !!i)                        \

  226.             limit2 -= 4;                                          \

  227.         src++;                                                    \

  228.     }                                                             \

  229.                                                                   \

  230.     shift = 20 - BIT_DEPTH;                                       \

  231.     add   = 1 << (shift - 1);                                     \

  232.     for (i = 0; i < H; i++) {                                     \

  233.         TR_ ## H(coeffs, coeffs, 1, 1, SCALE, limit);             \

  234.         coeffs += H;                                              \

  235.     }                                                             \

  236. }

  237. 

  238. #define IDCT_DC(H)                                                \

  239. static void FUNC(idct_ ## H ## x ## H ## _dc)(int16_t *coeffs)    \

  240. {                                                                 \

  241.     int i, j;                                                     \

  242.     int shift = 14 - BIT_DEPTH;                                   \

  243.     int add   = 1 << (shift - 1);                                 \

  244.     int coeff = (((coeffs[0] + 1) >> 1) + add) >> shift;          \

  245.                                                                   \

  246.     for (j = 0; j < H; j++) {                                     \

  247.         for (i = 0; i < H; i++) {                                 \

  248.             coeffs[i + j * H] = coeff;                            \

  249.         }                                                         \

  250.     }                                                             \

  251. }

  252. 

  253. IDCT( 4)

  254. IDCT( 8)

  255. IDCT(16)

  256. IDCT(32)

  257. IDCT_DC( 4)

  258. IDCT_DC( 8)

  259. IDCT_DC(16)

  260. IDCT_DC(32)

  261. #undef TR_4

  262. #undef TR_8

  263. #undef TR_16

  264. #undef TR_32

  265. 

  266. #undef SET

  267. #undef SCALE

  268. #undef ADD_AND_SCALE

  269. 

  270. static void FUNC(sao_band_filter)(uint8_t *_dst, uint8_t *_src,

  271.                                   ptrdiff_t stride, SAOParams *sao,

  272.                                   int *borders, int width, int height,

  273.                                   int c_idx, int class)

  274. {

  275.     pixel *dst = (pixel *)_dst;

  276.     pixel *src = (pixel *)_src;

  277.     int offset_table[32] = { 0 };

  278.     int k, y, x;

  279.     int chroma = !!c_idx;

  280.     int shift  = BIT_DEPTH - 5;

  281.     int *sao_offset_val = sao->offset_val[c_idx];

  282.     int sao_left_class  = sao->band_position[c_idx];

  283.     int init_y = 0, init_x = 0;

  284. 

  285.     stride /= sizeof(pixel);

  286. 

  287.     switch (class) {

  288.     case 0:

  289.         if (!borders[2])

  290.             width -= (8 >> chroma) + 2;

  291.         if (!borders[3])

  292.             height -= (4 >> chroma) + 2;

  293.         break;

  294.     case 1:

  295.         init_y = -(4 >> chroma) - 2;

  296.         if (!borders[2])

  297.             width -= (8 >> chroma) + 2;

  298.         height = (4 >> chroma) + 2;

  299.         break;

  300.     case 2:

  301.         init_x = -(8 >> chroma) - 2;

  302.         width  =  (8 >> chroma) + 2;

  303.         if (!borders[3])

  304.             height -= (4 >> chroma) + 2;

  305.         break;

  306.     case 3:

  307.         init_y = -(4 >> chroma) - 2;

  308.         init_x = -(8 >> chroma) - 2;

  309.         width  =  (8 >> chroma) + 2;

  310.         height =  (4 >> chroma) + 2;

  311.         break;

  312.     }

  313. 

  314.     dst = dst + (init_y * stride + init_x);

  315.     src = src + (init_y * stride + init_x);

  316.     for (k = 0; k < 4; k++)

  317.         offset_table[(k + sao_left_class) & 31] = sao_offset_val[k + 1];

  318.     for (y = 0; y < height; y++) {

  319.         for (x = 0; x < width; x++)

  320.             dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]);

  321.         dst += stride;

  322.         src += stride;

  323.     }

  324. }

  325. 

  326. static void FUNC(sao_band_filter_0)(uint8_t *dst, uint8_t *src,

  327.                                     ptrdiff_t stride, SAOParams *sao,

  328.                                     int *borders, int width, int height,

  329.                                     int c_idx)

  330. {

  331.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  332.                           width, height, c_idx, 0);

  333. }

  334. 

  335. static void FUNC(sao_band_filter_1)(uint8_t *dst, uint8_t *src,

  336.                                     ptrdiff_t stride, SAOParams *sao,

  337.                                     int *borders, int width, int height,

  338.                                     int c_idx)

  339. {

  340.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  341.                           width, height, c_idx, 1);

  342. }

  343. 

  344. static void FUNC(sao_band_filter_2)(uint8_t *dst, uint8_t *src,

  345.                                     ptrdiff_t stride, SAOParams *sao,

  346.                                     int *borders, int width, int height,

  347.                                     int c_idx)

  348. {

  349.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  350.                           width, height, c_idx, 2);

  351. }

  352. 

  353. static void FUNC(sao_band_filter_3)(uint8_t *_dst, uint8_t *_src,

  354.                                     ptrdiff_t stride, SAOParams *sao,

  355.                                     int *borders, int width, int height,

  356.                                     int c_idx)

  357. {

  358.     FUNC(sao_band_filter)(_dst, _src, stride, sao, borders,

  359.                           width, height, c_idx, 3);

  360. }

  361. 

  362. static void FUNC(sao_edge_filter_0)(uint8_t *_dst, uint8_t *_src,

  363.                                     ptrdiff_t stride, SAOParams *sao,

  364.                                     int *borders, int _width, int _height,

  365.                                     int c_idx, uint8_t vert_edge,

  366.                                     uint8_t horiz_edge, uint8_t diag_edge)

  367. {

  368.     int x, y;

  369.     pixel *dst = (pixel *)_dst;

  370.     pixel *src = (pixel *)_src;

  371.     int chroma = !!c_idx;

  372.     int *sao_offset_val = sao->offset_val[c_idx];

  373.     int sao_eo_class    = sao->eo_class[c_idx];

  374.     int init_x = 0, init_y = 0, width = _width, height = _height;

  375. 

  376.     static const int8_t pos[4][2][2] = {

  377.         { { -1,  0 }, {  1, 0 } }, // horizontal

  378.         { {  0, -1 }, {  0, 1 } }, // vertical

  379.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  380.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  381.     };

  382.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  383. 

  384. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  385. 

  386.     stride /= sizeof(pixel);

  387. 

  388.     if (!borders[2])

  389.         width -= (8 >> chroma) + 2;

  390.     if (!borders[3])

  391.         height -= (4 >> chroma) + 2;

  392. 

  393.     dst = dst + (init_y * stride + init_x);

  394.     src = src + (init_y * stride + init_x);

  395.     init_y = init_x = 0;

  396.     if (sao_eo_class != SAO_EO_VERT) {

  397.         if (borders[0]) {

  398.             int offset_val = sao_offset_val[0];

  399.             ptrdiff_t y_stride   = 0;

  400.             for (y = 0; y < height; y++) {

  401.                 dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);

  402.                 y_stride     += stride;

  403.             }

  404.             init_x = 1;

  405.         }

  406.         if (borders[2]) {

  407.             int offset_val = sao_offset_val[0];

  408.             ptrdiff_t x_stride = width - 1;

  409.             for (x = 0; x < height; x++) {

  410.                 dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);

  411.                 x_stride     += stride;

  412.             }

  413.             width--;

  414.         }

  415.     }

  416.     if (sao_eo_class != SAO_EO_HORIZ) {

  417.         if (borders[1]) {

  418.             int offset_val = sao_offset_val[0];

  419.             for (x = init_x; x < width; x++)

  420.                 dst[x] = av_clip_pixel(src[x] + offset_val);

  421.             init_y = 1;

  422.         }

  423.         if (borders[3]) {

  424.             int offset_val = sao_offset_val[0];

  425.             ptrdiff_t y_stride = stride * (height - 1);

  426.             for (x = init_x; x < width; x++)

  427.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);

  428.             height--;

  429.         }

  430.     }

  431.     {

  432.         ptrdiff_t y_stride = init_y * stride;

  433.         int pos_0_0  = pos[sao_eo_class][0][0];

  434.         int pos_0_1  = pos[sao_eo_class][0][1];

  435.         int pos_1_0  = pos[sao_eo_class][1][0];

  436.         int pos_1_1  = pos[sao_eo_class][1][1];

  437. 

  438.         ptrdiff_t y_stride_0_1 = (init_y + pos_0_1) * stride;

  439.         ptrdiff_t y_stride_1_1 = (init_y + pos_1_1) * stride;

  440.         for (y = init_y; y < height; y++) {

  441.             for (x = init_x; x < width; x++) {

  442.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  443.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  444.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  445.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  446.             }

  447.             y_stride     += stride;

  448.             y_stride_0_1 += stride;

  449.             y_stride_1_1 += stride;

  450.         }

  451.     }

  452. 

  453.     {

  454.         // Restore pixels that can't be modified

  455.         int save_upper_left = !diag_edge && sao_eo_class == SAO_EO_135D && !borders[0] && !borders[1];

  456.         if (vert_edge && sao_eo_class != SAO_EO_VERT)

  457.             for (y = init_y+save_upper_left; y< height; y++)

  458.                 dst[y*stride] = src[y*stride];

  459.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  460.             for(x = init_x+save_upper_left; x<width; x++)

  461.                 dst[x] = src[x];

  462.         if(diag_edge && sao_eo_class == SAO_EO_135D)

  463.             dst[0] = src[0];

  464.     }

  465. 

  466. #undef CMP

  467. }

  468. 

  469. static void FUNC(sao_edge_filter_1)(uint8_t *_dst, uint8_t *_src,

  470.                                     ptrdiff_t stride, SAOParams *sao,

  471.                                     int *borders, int _width, int _height,

  472.                                     int c_idx, uint8_t vert_edge,

  473.                                     uint8_t horiz_edge, uint8_t diag_edge)

  474. {

  475.     int x, y;

  476.     pixel *dst = (pixel *)_dst;

  477.     pixel *src = (pixel *)_src;

  478.     int chroma = !!c_idx;

  479.     int *sao_offset_val = sao->offset_val[c_idx];

  480.     int sao_eo_class    = sao->eo_class[c_idx];

  481.     int init_x = 0, init_y = 0, width = _width, height = _height;

  482. 

  483.     static const int8_t pos[4][2][2] = {

  484.         { { -1, 0  }, { 1,  0 } }, // horizontal

  485.         { { 0,  -1 }, { 0,  1 } }, // vertical

  486.         { { -1, -1 }, { 1,  1 } }, // 45 degree

  487.         { { 1,  -1 }, { -1, 1 } }, // 135 degree

  488.     };

  489.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  490. 

  491. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  492. 

  493.     stride /= sizeof(pixel);

  494. 

  495.     init_y = -(4 >> chroma) - 2;

  496.     if (!borders[2])

  497.         width -= (8 >> chroma) + 2;

  498.     height = (4 >> chroma) + 2;

  499. 

  500.     dst = dst + (init_y * stride + init_x);

  501.     src = src + (init_y * stride + init_x);

  502.     init_y = init_x = 0;

  503.     if (sao_eo_class != SAO_EO_VERT) {

  504.         if (borders[0]) {

  505.             int offset_val = sao_offset_val[0];

  506.             ptrdiff_t y_stride = 0;

  507.             for (y = 0; y < height; y++) {

  508.                 dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);

  509.                 y_stride     += stride;

  510.             }

  511.             init_x = 1;

  512.         }

  513.         if (borders[2]) {

  514.             int offset_val = sao_offset_val[0];

  515.             ptrdiff_t x_stride = width - 1;

  516.             for (x = 0; x < height; x++) {

  517.                 dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);

  518.                 x_stride     += stride;

  519.             }

  520.             width--;

  521.         }

  522.     }

  523.     {

  524.         ptrdiff_t y_stride = init_y * stride;

  525.         int pos_0_0  = pos[sao_eo_class][0][0];

  526.         int pos_0_1  = pos[sao_eo_class][0][1];

  527.         int pos_1_0  = pos[sao_eo_class][1][0];

  528.         int pos_1_1  = pos[sao_eo_class][1][1];

  529. 

  530.         ptrdiff_t y_stride_0_1 = (init_y + pos_0_1) * stride;

  531.         ptrdiff_t y_stride_1_1 = (init_y + pos_1_1) * stride;

  532.         for (y = init_y; y < height; y++) {

  533.             for (x = init_x; x < width; x++) {

  534.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  535.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  536.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  537.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  538.             }

  539.             y_stride     += stride;

  540.             y_stride_0_1 += stride;

  541.             y_stride_1_1 += stride;

  542.         }

  543.     }

  544. 

  545.     {

  546.         // Restore pixels that can't be modified

  547.         int save_lower_left = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[0];

  548.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  549.             for(y = init_y; y< height-save_lower_left; y++)

  550.                 dst[y*stride] = src[y*stride];

  551.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  552.             for(x = init_x+save_lower_left; x<width; x++)

  553.                 dst[(height-1)*stride+x] = src[(height-1)*stride+x];

  554.         if(diag_edge && sao_eo_class == SAO_EO_45D)

  555.             dst[stride*(height-1)] = src[stride*(height-1)];

  556.     }

  557. 

  558. #undef CMP

  559. }

  560. 

  561. static void FUNC(sao_edge_filter_2)(uint8_t *_dst, uint8_t *_src,

  562.                                     ptrdiff_t stride, SAOParams *sao,

  563.                                     int *borders, int _width, int _height,

  564.                                     int c_idx, uint8_t vert_edge,

  565.                                     uint8_t horiz_edge, uint8_t diag_edge)

  566. {

  567.     int x, y;

  568.     pixel *dst = (pixel *)_dst;

  569.     pixel *src = (pixel *)_src;

  570.     int chroma = !!c_idx;

  571.     int *sao_offset_val = sao->offset_val[c_idx];

  572.     int sao_eo_class    = sao->eo_class[c_idx];

  573.     int init_x = 0, init_y = 0, width = _width, height = _height;

  574. 

  575.     static const int8_t pos[4][2][2] = {

  576.         { { -1,  0 }, {  1, 0 } }, // horizontal

  577.         { {  0, -1 }, {  0, 1 } }, // vertical

  578.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  579.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  580.     };

  581.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  582. 

  583. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  584. 

  585.     stride /= sizeof(pixel);

  586. 

  587.     init_x = -(8 >> chroma) - 2;

  588.     width  =  (8 >> chroma) + 2;

  589.     if (!borders[3])

  590.         height -= (4 >> chroma) + 2;

  591. 

  592.     dst = dst + (init_y * stride + init_x);

  593.     src = src + (init_y * stride + init_x);

  594.     init_y = init_x = 0;

  595.     if (sao_eo_class != SAO_EO_HORIZ) {

  596.         if (borders[1]) {

  597.             int offset_val = sao_offset_val[0];

  598.             for (x = init_x; x < width; x++)

  599.                 dst[x] = av_clip_pixel(src[x] + offset_val);

  600.             init_y = 1;

  601.         }

  602.         if (borders[3]) {

  603.             int offset_val = sao_offset_val[0];

  604.             ptrdiff_t y_stride = stride * (height - 1);

  605.             for (x = init_x; x < width; x++)

  606.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);

  607.             height--;

  608.         }

  609.     }

  610.     {

  611.         ptrdiff_t y_stride = init_y * stride;

  612.         int pos_0_0  = pos[sao_eo_class][0][0];

  613.         int pos_0_1  = pos[sao_eo_class][0][1];

  614.         int pos_1_0  = pos[sao_eo_class][1][0];

  615.         int pos_1_1  = pos[sao_eo_class][1][1];

  616. 

  617.         ptrdiff_t y_stride_0_1 = (init_y + pos_0_1) * stride;

  618.         ptrdiff_t y_stride_1_1 = (init_y + pos_1_1) * stride;

  619.         for (y = init_y; y < height; y++) {

  620.             for (x = init_x; x < width; x++) {

  621.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  622.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  623.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  624.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  625.             }

  626.             y_stride     += stride;

  627.             y_stride_0_1 += stride;

  628.             y_stride_1_1 += stride;

  629.         }

  630.     }

  631. 

  632.     {

  633.         // Restore pixels that can't be modified

  634.         int save_upper_right = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[1];

  635.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  636.             for(y = init_y+save_upper_right; y< height; y++)

  637.                 dst[y*stride+width-1] = src[y*stride+width-1];

  638.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  639.             for(x = init_x; x<width-save_upper_right; x++)

  640.                 dst[x] = src[x];

  641.         if(diag_edge && sao_eo_class == SAO_EO_45D)

  642.             dst[width-1] = src[width-1];

  643.     }

  644. #undef CMP

  645. }

  646. 

  647. static void FUNC(sao_edge_filter_3)(uint8_t *_dst, uint8_t *_src,

  648.                                     ptrdiff_t stride, SAOParams *sao,

  649.                                     int *borders, int _width, int _height,

  650.                                     int c_idx, uint8_t vert_edge,

  651.                                     uint8_t horiz_edge, uint8_t diag_edge)

  652. {

  653.     int x, y;

  654.     pixel *dst = (pixel *)_dst;

  655.     pixel *src = (pixel *)_src;

  656.     int chroma = !!c_idx;

  657.     int *sao_offset_val = sao->offset_val[c_idx];

  658.     int sao_eo_class    = sao->eo_class[c_idx];

  659.     int init_x = 0, init_y = 0, width = _width, height = _height;

  660. 

  661.     static const int8_t pos[4][2][2] = {

  662.         { { -1,  0 }, {  1, 0 } }, // horizontal

  663.         { {  0, -1 }, {  0, 1 } }, // vertical

  664.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  665.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  666.     };

  667.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  668. 

  669. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  670. 

  671.     stride /= sizeof(pixel);

  672. 

  673.     init_y = -(4 >> chroma) - 2;

  674.     init_x = -(8 >> chroma) - 2;

  675.     width  =  (8 >> chroma) + 2;

  676.     height =  (4 >> chroma) + 2;

  677. 

  678. 

  679.     dst    = dst + (init_y * stride + init_x);

  680.     src    = src + (init_y * stride + init_x);

  681.     init_y = init_x = 0;

  682. 

  683.     {

  684.         ptrdiff_t y_stride = init_y * stride;

  685.         int pos_0_0  = pos[sao_eo_class][0][0];

  686.         int pos_0_1  = pos[sao_eo_class][0][1];

  687.         int pos_1_0  = pos[sao_eo_class][1][0];

  688.         int pos_1_1  = pos[sao_eo_class][1][1];

  689. 

  690.         ptrdiff_t y_stride_0_1 = (init_y + pos_0_1) * stride;

  691.         ptrdiff_t y_stride_1_1 = (init_y + pos_1_1) * stride;

  692. 

  693.         for (y = init_y; y < height; y++) {

  694.             for (x = init_x; x < width; x++) {

  695.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  696.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  697.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  698.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  699.             }

  700.             y_stride     += stride;

  701.             y_stride_0_1 += stride;

  702.             y_stride_1_1 += stride;

  703.         }

  704.     }

  705. 

  706.     {

  707.         // Restore pixels that can't be modified

  708.         int save_lower_right = !diag_edge && sao_eo_class == SAO_EO_135D;

  709.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  710.             for(y = init_y; y< height-save_lower_right; y++)

  711.                 dst[y*stride+width-1] = src[y*stride+width-1];

  712.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  713.             for(x = init_x; x<width-save_lower_right; x++)

  714.                 dst[(height-1)*stride+x] = src[(height-1)*stride+x];

  715.         if(diag_edge && sao_eo_class == SAO_EO_135D)

  716.             dst[stride*(height-1)+width-1] = src[stride*(height-1)+width-1];

  717.     }

  718. #undef CMP

  719. }

  720. 

  721. #undef SET

  722. #undef SCALE

  723. #undef TR_4

  724. #undef TR_8

  725. #undef TR_16

  726. #undef TR_32

  727. 

  728. static av_always_inline void

  729. FUNC(put_hevc_qpel_pixels)(int16_t *dst, ptrdiff_t dststride,

  730.                            uint8_t *_src, ptrdiff_t _srcstride,

  731.                            int width, int height, int mx, int my,

  732.                            int16_t* mcbuffer)

  733. {

  734.     int x, y;

  735.     pixel *src          = (pixel *)_src;

  736.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  737. 

  738.     dststride /= sizeof(*dst);

  739.     for (y = 0; y < height; y++) {

  740.         for (x = 0; x < width; x++)

  741.             dst[x] = src[x] << (14 - BIT_DEPTH);

  742.         src += srcstride;

  743.         dst += dststride;

  744.     }

  745. }

  746. 

  747. #define QPEL_FILTER_1(src, stride)      \

  748.     (1 * -src[x - 3 * stride] +         \

  749.      4 *  src[x - 2 * stride] -         \

  750.     10 *  src[x -     stride] +         \

  751.     58 *  src[x]              +         \

  752.     17 *  src[x +     stride] -         \

  753.      5 *  src[x + 2 * stride] +         \

  754.      1 *  src[x + 3 * stride])

  755. 

  756. #define QPEL_FILTER_2(src, stride)      \

  757.     (1  * -src[x - 3 * stride] +        \

  758.      4  *  src[x - 2 * stride] -        \

  759.     11  *  src[x -     stride] +        \

  760.     40  *  src[x]              +        \

  761.     40  *  src[x +     stride] -        \

  762.     11  *  src[x + 2 * stride] +        \

  763.      4  *  src[x + 3 * stride] -        \

  764.      1  *  src[x + 4 * stride])

  765. 

  766. #define QPEL_FILTER_3(src, stride)      \

  767.     (1  * src[x - 2 * stride] -         \

  768.      5  * src[x -     stride] +         \

  769.     17  * src[x]              +         \

  770.     58  * src[x + stride]     -         \

  771.     10  * src[x + 2 * stride] +         \

  772.      4  * src[x + 3 * stride] -         \

  773.      1  * src[x + 4 * stride])

  774. 

  775. 

  776. #define PUT_HEVC_QPEL_H(H)                                                     \

  777. static void FUNC(put_hevc_qpel_h ## H)(int16_t *dst,  ptrdiff_t dststride,     \

  778.                                        uint8_t *_src, ptrdiff_t _srcstride,    \

  779.                                        int width, int height,                  \

  780.                                        int16_t* mcbuffer)                      \

  781. {                                                                              \

  782.     int x, y;                                                                  \

  783.     pixel *src = (pixel*)_src;                                                 \

  784.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  785.                                                                                \

  786.     dststride /= sizeof(*dst);                                                 \

  787.     for (y = 0; y < height; y++) {                                             \

  788.         for (x = 0; x < width; x++)                                            \

  789.             dst[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \

  790.         src += srcstride;                                                      \

  791.         dst += dststride;                                                      \

  792.     }                                                                          \

  793. }

  794. 

  795. #define PUT_HEVC_QPEL_V(V)                                                     \

  796. static void FUNC(put_hevc_qpel_v ## V)(int16_t *dst,  ptrdiff_t dststride,     \

  797.                                        uint8_t *_src, ptrdiff_t _srcstride,    \

  798.                                        int width, int height,                  \

  799.                                        int16_t* mcbuffer)                      \

  800. {                                                                              \

  801.     int x, y;                                                                  \

  802.     pixel *src = (pixel*)_src;                                                 \

  803.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  804.                                                                                \

  805.     dststride /= sizeof(*dst);                                                 \

  806.     for (y = 0; y < height; y++)  {                                            \

  807.         for (x = 0; x < width; x++)                                            \

  808.             dst[x] = QPEL_FILTER_ ## V(src, srcstride) >> (BIT_DEPTH - 8);     \

  809.         src += srcstride;                                                      \

  810.         dst += dststride;                                                      \

  811.     }                                                                          \

  812. }

  813. 

  814. #define PUT_HEVC_QPEL_HV(H, V)                                                 \

  815. static void FUNC(put_hevc_qpel_h ## H ## v ## V)(int16_t *dst,                 \

  816.                                                  ptrdiff_t dststride,          \

  817.                                                  uint8_t *_src,                \

  818.                                                  ptrdiff_t _srcstride,         \

  819.                                                  int width, int height,        \

  820.                                                  int16_t* mcbuffer)            \

  821. {                                                                              \

  822.     int x, y;                                                                  \

  823.     pixel *src = (pixel*)_src;                                                 \

  824.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  825.                                                                                \

  826.     int16_t tmp_array[(MAX_PB_SIZE + 7) * MAX_PB_SIZE];                        \

  827.     int16_t *tmp = tmp_array;                                                  \

  828.                                                                                \

  829.     dststride /= sizeof(*dst);                                                 \

  830.     src -= ff_hevc_qpel_extra_before[V] * srcstride;                           \

  831.                                                                                \

  832.     for (y = 0; y < height + ff_hevc_qpel_extra[V]; y++) {                     \

  833.         for (x = 0; x < width; x++)                                            \

  834.             tmp[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \

  835.         src += srcstride;                                                      \

  836.         tmp += MAX_PB_SIZE;                                                    \

  837.     }                                                                          \

  838.                                                                                \

  839.     tmp = tmp_array + ff_hevc_qpel_extra_before[V] * MAX_PB_SIZE;              \

  840.                                                                                \

  841.     for (y = 0; y < height; y++) {                                             \

  842.         for (x = 0; x < width; x++)                                            \

  843.             dst[x] = QPEL_FILTER_ ## V(tmp, MAX_PB_SIZE) >> 6;                 \

  844.         tmp += MAX_PB_SIZE;                                                    \

  845.         dst += dststride;                                                      \

  846.     }                                                                          \

  847. }

  848. 

  849. PUT_HEVC_QPEL_H(1)

  850. PUT_HEVC_QPEL_H(2)

  851. PUT_HEVC_QPEL_H(3)

  852. PUT_HEVC_QPEL_V(1)

  853. PUT_HEVC_QPEL_V(2)

  854. PUT_HEVC_QPEL_V(3)

  855. PUT_HEVC_QPEL_HV(1, 1)

  856. PUT_HEVC_QPEL_HV(1, 2)

  857. PUT_HEVC_QPEL_HV(1, 3)

  858. PUT_HEVC_QPEL_HV(2, 1)

  859. PUT_HEVC_QPEL_HV(2, 2)

  860. PUT_HEVC_QPEL_HV(2, 3)

  861. PUT_HEVC_QPEL_HV(3, 1)

  862. PUT_HEVC_QPEL_HV(3, 2)

  863. PUT_HEVC_QPEL_HV(3, 3)

  864. 

  865. #define QPEL(W)                                                                             \

  866. static void FUNC(put_hevc_qpel_pixels_ ## W)(int16_t *dst, ptrdiff_t dststride,             \

  867.                                              uint8_t *src, ptrdiff_t srcstride,             \

  868.                                              int height, int mx, int my,                    \

  869.                                              int16_t *mcbuffer)                             \

  870. {                                                                                           \

  871.     FUNC(put_hevc_qpel_pixels)(dst, dststride, src, srcstride, W, height,                   \

  872.                                mx, my, mcbuffer);                                           \

  873. }                                                                                           \

  874.                                                                                             \

  875. static void FUNC(put_hevc_qpel_h_ ## W)(int16_t *dst, ptrdiff_t dststride,                  \

  876.                                         uint8_t *src, ptrdiff_t srcstride,                  \

  877.                                         int height, int mx, int my,                         \

  878.                                         int16_t *mcbuffer)                                  \

  879. {                                                                                           \

  880.     if (mx == 1)                                                                            \

  881.         FUNC(put_hevc_qpel_h1)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  882.     else if (mx == 2)                                                                       \

  883.         FUNC(put_hevc_qpel_h2)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  884.     else                                                                                    \

  885.         FUNC(put_hevc_qpel_h3)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  886. }                                                                                           \

  887.                                                                                             \

  888. static void FUNC(put_hevc_qpel_v_ ## W)(int16_t *dst, ptrdiff_t dststride,                  \

  889.                                              uint8_t *src, ptrdiff_t srcstride,             \

  890.                                              int height, int mx, int my,                    \

  891.                                              int16_t *mcbuffer)                             \

  892. {                                                                                           \

  893.     if (my == 1)                                                                            \

  894.         FUNC(put_hevc_qpel_v1)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  895.     else if (my == 2)                                                                       \

  896.         FUNC(put_hevc_qpel_v2)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  897.     else                                                                                    \

  898.         FUNC(put_hevc_qpel_v3)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  899. }                                                                                           \

  900.                                                                                             \

  901. static void FUNC(put_hevc_qpel_hv_ ## W)(int16_t *dst, ptrdiff_t dststride,                 \

  902.                                              uint8_t *src, ptrdiff_t srcstride,             \

  903.                                              int height, int mx, int my,                    \

  904.                                              int16_t *mcbuffer)                             \

  905. {                                                                                           \

  906.     if (my == 1) {                                                                          \

  907.         if (mx == 1)                                                                        \

  908.             FUNC(put_hevc_qpel_h1v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  909.         else if (mx == 2)                                                                   \

  910.             FUNC(put_hevc_qpel_h2v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  911.         else                                                                                \

  912.             FUNC(put_hevc_qpel_h3v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  913.     } else if (my == 2) {                                                                   \

  914.         if (mx == 1)                                                                        \

  915.             FUNC(put_hevc_qpel_h1v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  916.         else if (mx == 2)                                                                   \

  917.             FUNC(put_hevc_qpel_h2v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  918.         else                                                                                \

  919.             FUNC(put_hevc_qpel_h3v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  920.     } else {                                                                                \

  921.         if (mx == 1)                                                                        \

  922.             FUNC(put_hevc_qpel_h1v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  923.         else if (mx == 2)                                                                   \

  924.             FUNC(put_hevc_qpel_h2v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  925.         else                                                                                \

  926.             FUNC(put_hevc_qpel_h3v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  927.     }                                                                                       \

  928. }

  929. 

  930. QPEL(64)

  931. QPEL(48)

  932. QPEL(32)

  933. QPEL(24)

  934. QPEL(16)

  935. QPEL(12)

  936. QPEL(8)

  937. QPEL(4)

  938. 

  939. static inline void FUNC(put_hevc_epel_pixels)(int16_t *dst, ptrdiff_t dststride,

  940.                                               uint8_t *_src, ptrdiff_t _srcstride,

  941.                                               int width, int height, int mx, int my,

  942.                                               int16_t* mcbuffer)

  943. {

  944.     int x, y;

  945.     pixel *src          = (pixel *)_src;

  946.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  947. 

  948.     dststride /= sizeof(*dst);

  949.     for (y = 0; y < height; y++) {

  950.         for (x = 0; x < width; x++)

  951.             dst[x] = src[x] << (14 - BIT_DEPTH);

  952.         src += srcstride;

  953.         dst += dststride;

  954.     }

  955. }

  956. 

  957. #define EPEL_FILTER(src, stride)                \

  958.     (filter_0 * src[x - stride] +               \

  959.      filter_1 * src[x]          +               \

  960.      filter_2 * src[x + stride] +               \

  961.      filter_3 * src[x + 2 * stride])

  962. 

  963. static inline void FUNC(put_hevc_epel_h)(int16_t *dst, ptrdiff_t dststride,

  964.                                          uint8_t *_src, ptrdiff_t _srcstride,

  965.                                          int width, int height, int mx, int my,

  966.                                          int16_t* mcbuffer)

  967. {

  968.     int x, y;

  969.     pixel *src = (pixel *)_src;

  970.     ptrdiff_t srcstride  = _srcstride / sizeof(pixel);

  971.     const int16_t *filter = ff_hevc_epel_coeffs[mx - 1];

  972.     int8_t filter_0 = filter[0];

  973.     int8_t filter_1 = filter[1];

  974.     int8_t filter_2 = filter[2];

  975.     int8_t filter_3 = filter[3];

  976.     dststride /= sizeof(*dst);

  977.     for (y = 0; y < height; y++) {

  978.         for (x = 0; x < width; x++)

  979.             dst[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);

  980.         src += srcstride;

  981.         dst += dststride;

  982.     }

  983. }

  984. 

  985. static inline void FUNC(put_hevc_epel_v)(int16_t *dst, ptrdiff_t dststride,

  986.                                          uint8_t *_src, ptrdiff_t _srcstride,

  987.                                          int width, int height, int mx, int my,

  988.                                          int16_t* mcbuffer)

  989. {

  990.     int x, y;

  991.     pixel *src = (pixel *)_src;

  992.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  993.     const int16_t *filter = ff_hevc_epel_coeffs[my - 1];

  994.     int8_t filter_0 = filter[0];

  995.     int8_t filter_1 = filter[1];

  996.     int8_t filter_2 = filter[2];

  997.     int8_t filter_3 = filter[3];

  998. 

  999.     dststride /= sizeof(*dst);

  1000.     for (y = 0; y < height; y++) {

  1001.         for (x = 0; x < width; x++)

  1002.             dst[x] = EPEL_FILTER(src, srcstride) >> (BIT_DEPTH - 8);

  1003.         src += srcstride;

  1004.         dst += dststride;

  1005.     }

  1006. }

  1007. 

  1008. static inline void FUNC(put_hevc_epel_hv)(int16_t *dst, ptrdiff_t dststride,

  1009.                                           uint8_t *_src, ptrdiff_t _srcstride,

  1010.                                           int width, int height, int mx, int my,

  1011.                                           int16_t* mcbuffer)

  1012. {

  1013.     int x, y;

  1014.     pixel *src = (pixel *)_src;

  1015.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  1016.     const int16_t *filter_h = ff_hevc_epel_coeffs[mx - 1];

  1017.     const int16_t *filter_v = ff_hevc_epel_coeffs[my - 1];

  1018.     int8_t filter_0 = filter_h[0];

  1019.     int8_t filter_1 = filter_h[1];

  1020.     int8_t filter_2 = filter_h[2];

  1021.     int8_t filter_3 = filter_h[3];

  1022.     int16_t tmp_array[(MAX_PB_SIZE + 3) * MAX_PB_SIZE];

  1023.     int16_t *tmp = tmp_array;

  1024. 

  1025.     dststride /= sizeof(*dst);

  1026.     src -= EPEL_EXTRA_BEFORE * srcstride;

  1027. 

  1028.     for (y = 0; y < height + EPEL_EXTRA; y++) {

  1029.         for (x = 0; x < width; x++)

  1030.             tmp[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);

  1031.         src += srcstride;

  1032.         tmp += MAX_PB_SIZE;

  1033.     }

  1034. 

  1035.     tmp      = tmp_array + EPEL_EXTRA_BEFORE * MAX_PB_SIZE;

  1036.     filter_0 = filter_v[0];

  1037.     filter_1 = filter_v[1];

  1038.     filter_2 = filter_v[2];

  1039.     filter_3 = filter_v[3];

  1040.     for (y = 0; y < height; y++) {

  1041.         for (x = 0; x < width; x++)

  1042.             dst[x] = EPEL_FILTER(tmp, MAX_PB_SIZE) >> 6;

  1043.         tmp += MAX_PB_SIZE;

  1044.         dst += dststride;

  1045.     }

  1046. }

  1047. 

  1048. #define EPEL(W)                                                                 \

  1049. static void FUNC(put_hevc_epel_pixels_ ## W)(int16_t *dst, ptrdiff_t dststride, \

  1050.                                              uint8_t *src, ptrdiff_t srcstride, \

  1051.                                              int height, int mx, int my,        \

  1052.                                              int16_t *mcbuffer)                 \

  1053. {                                                                               \

  1054.     FUNC(put_hevc_epel_pixels)(dst, dststride, src, srcstride,                  \

  1055.                                W, height, mx, my, mcbuffer);                    \

  1056. }                                                                               \

  1057. static void FUNC(put_hevc_epel_h_ ## W)(int16_t *dst, ptrdiff_t dststride,      \

  1058.                                         uint8_t *src, ptrdiff_t srcstride,      \

  1059.                                         int height, int mx, int my,             \

  1060.                                         int16_t *mcbuffer)                      \

  1061. {                                                                               \

  1062.     FUNC(put_hevc_epel_h)(dst, dststride, src, srcstride,                       \

  1063.                           W, height, mx, my, mcbuffer);                         \

  1064. }                                                                               \

  1065. static void FUNC(put_hevc_epel_v_ ## W)(int16_t *dst, ptrdiff_t dststride,      \

  1066.                                         uint8_t *src, ptrdiff_t srcstride,      \

  1067.                                         int height, int mx, int my,             \

  1068.                                         int16_t *mcbuffer)                      \

  1069. {                                                                               \

  1070.     FUNC(put_hevc_epel_v)(dst, dststride, src, srcstride,                       \

  1071.                           W, height, mx, my, mcbuffer);                         \

  1072. }                                                                               \

  1073. static void FUNC(put_hevc_epel_hv_ ## W)(int16_t *dst, ptrdiff_t dststride,     \

  1074.                                          uint8_t *src, ptrdiff_t srcstride,     \

  1075.                                          int height, int mx, int my,            \

  1076.                                          int16_t *mcbuffer)                     \

  1077. {                                                                               \

  1078.     FUNC(put_hevc_epel_hv)(dst, dststride, src, srcstride,                      \

  1079.                            W, height, mx, my, mcbuffer);                        \

  1080. }

  1081. 

  1082. EPEL(32)

  1083. EPEL(24)

  1084. EPEL(16)

  1085. EPEL(12)

  1086. EPEL(8)

  1087. EPEL(6)

  1088. EPEL(4)

  1089. EPEL(2)

  1090. 

  1091. static av_always_inline void

  1092. FUNC(put_unweighted_pred)(uint8_t *_dst, ptrdiff_t _dststride,

  1093.                           int16_t *src, ptrdiff_t srcstride,

  1094.                           int width, int height)

  1095. {

  1096.     int x, y;

  1097.     pixel *dst          = (pixel *)_dst;

  1098.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1099. 

  1100.     int shift = 14 - BIT_DEPTH;

  1101. #if BIT_DEPTH < 14

  1102.     int offset = 1 << (shift - 1);

  1103. #else

  1104.     int offset = 0;

  1105. #endif

  1106.     srcstride /= sizeof(*src);

  1107.     for (y = 0; y < height; y++) {

  1108.         for (x = 0; x < width; x++)

  1109.             dst[x] = av_clip_pixel((src[x] + offset) >> shift);

  1110.         dst += dststride;

  1111.         src += srcstride;

  1112.     }

  1113. }

  1114. 

  1115. static av_always_inline void

  1116. FUNC(put_unweighted_pred_avg)(uint8_t *_dst, ptrdiff_t _dststride,

  1117.                               int16_t *src1, int16_t *src2,

  1118.                               ptrdiff_t srcstride,

  1119.                               int width, int height)

  1120. {

  1121.     int x, y;

  1122.     pixel *dst          = (pixel *)_dst;

  1123.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1124. 

  1125.     int shift = 14 + 1 - BIT_DEPTH;

  1126. #if BIT_DEPTH < 14

  1127.     int offset = 1 << (shift - 1);

  1128. #else

  1129.     int offset = 0;

  1130. #endif

  1131. 

  1132.     srcstride /= sizeof(*src1);

  1133.     for (y = 0; y < height; y++) {

  1134.         for (x = 0; x < width; x++)

  1135.             dst[x] = av_clip_pixel((src1[x] + src2[x] + offset) >> shift);

  1136.         dst  += dststride;

  1137.         src1 += srcstride;

  1138.         src2 += srcstride;

  1139.     }

  1140. }

  1141. 

  1142. static av_always_inline void

  1143. FUNC(weighted_pred)(uint8_t denom, int16_t wlxFlag, int16_t olxFlag,

  1144.                     uint8_t *_dst, ptrdiff_t _dststride,

  1145.                     int16_t *src, ptrdiff_t srcstride,

  1146.                     int width, int height)

  1147. {

  1148.     int shift, log2Wd, wx, ox, x, y, offset;

  1149.     pixel *dst          = (pixel *)_dst;

  1150.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1151. 

  1152.     shift  = 14 - BIT_DEPTH;

  1153.     log2Wd = denom + shift;

  1154.     offset = 1 << (log2Wd - 1);

  1155.     wx     = wlxFlag;

  1156.     ox     = olxFlag * (1 << (BIT_DEPTH - 8));

  1157. 

  1158.     srcstride /= sizeof(*src);

  1159.     for (y = 0; y < height; y++) {

  1160.         for (x = 0; x < width; x++) {

  1161.             if (log2Wd >= 1) {

  1162.                 dst[x] = av_clip_pixel(((src[x] * wx + offset) >> log2Wd) + ox);

  1163.             } else {

  1164.                 dst[x] = av_clip_pixel(src[x] * wx + ox);

  1165.             }

  1166.         }

  1167.         dst += dststride;

  1168.         src += srcstride;

  1169.     }

  1170. }

  1171. 

  1172. static av_always_inline void

  1173. FUNC(weighted_pred_avg)(uint8_t denom,

  1174.                         int16_t wl0Flag, int16_t wl1Flag,

  1175.                         int16_t ol0Flag, int16_t ol1Flag,

  1176.                         uint8_t *_dst, ptrdiff_t _dststride,

  1177.                         int16_t *src1, int16_t *src2,

  1178.                         ptrdiff_t srcstride,

  1179.                         int width, int height)

  1180. {

  1181.     int shift, log2Wd, w0, w1, o0, o1, x, y;

  1182.     pixel *dst = (pixel *)_dst;

  1183.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1184. 

  1185.     shift  = 14 - BIT_DEPTH;

  1186.     log2Wd = denom + shift;

  1187.     w0     = wl0Flag;

  1188.     w1     = wl1Flag;

  1189.     o0     = ol0Flag * (1 << (BIT_DEPTH - 8));

  1190.     o1     = ol1Flag * (1 << (BIT_DEPTH - 8));

  1191. 

  1192.     srcstride /= sizeof(*src1);

  1193.     for (y = 0; y < height; y++) {

  1194.         for (x = 0; x < width; x++)

  1195.             dst[x] = av_clip_pixel((src1[x] * w0 + src2[x] * w1 +

  1196.                                     ((o0 + o1 + 1) << log2Wd)) >> (log2Wd + 1));

  1197.         dst  += dststride;

  1198.         src1 += srcstride;

  1199.         src2 += srcstride;

  1200.     }

  1201. }

  1202. 

  1203. #define PUT_PRED(w)                                                                            \

  1204. static void FUNC(put_unweighted_pred_ ## w)(uint8_t *dst, ptrdiff_t dststride,                 \

  1205.                                             int16_t *src, ptrdiff_t srcstride,                 \

  1206.                                             int height)                                        \

  1207. {                                                                                              \

  1208.     FUNC(put_unweighted_pred)(dst, dststride, src, srcstride, w, height);                      \

  1209. }                                                                                              \

  1210. static void FUNC(put_unweighted_pred_avg_ ## w)(uint8_t *dst, ptrdiff_t dststride,             \

  1211.                                                 int16_t *src1, int16_t *src2,                  \

  1212.                                                 ptrdiff_t srcstride, int height)               \

  1213. {                                                                                              \

  1214.     FUNC(put_unweighted_pred_avg)(dst, dststride, src1, src2, srcstride, w, height);           \

  1215. }                                                                                              \

  1216. static void FUNC(put_weighted_pred_ ## w)(uint8_t denom, int16_t weight, int16_t offset,       \

  1217.                                           uint8_t *dst, ptrdiff_t dststride,                   \

  1218.                                           int16_t *src, ptrdiff_t srcstride, int height)       \

  1219. {                                                                                              \

  1220.     FUNC(weighted_pred)(denom, weight, offset,                                                 \

  1221.                         dst, dststride, src, srcstride, w, height);                            \

  1222. }                                                                                              \

  1223. static void FUNC(put_weighted_pred_avg_ ## w)(uint8_t denom, int16_t weight0, int16_t weight1, \

  1224.                                               int16_t offset0, int16_t offset1,                \

  1225.                                               uint8_t *dst, ptrdiff_t dststride,               \

  1226.                                               int16_t *src1, int16_t *src2,                    \

  1227.                                               ptrdiff_t srcstride, int height)                 \

  1228. {                                                                                              \

  1229.     FUNC(weighted_pred_avg)(denom, weight0, weight1, offset0, offset1,                         \

  1230.                             dst, dststride, src1, src2, srcstride, w, height);                 \

  1231. }

  1232. 

  1233. PUT_PRED(64)

  1234. PUT_PRED(48)

  1235. PUT_PRED(32)

  1236. PUT_PRED(24)

  1237. PUT_PRED(16)

  1238. PUT_PRED(12)

  1239. PUT_PRED(8)

  1240. PUT_PRED(6)

  1241. PUT_PRED(4)

  1242. PUT_PRED(2)

  1243. 

  1244. // line zero

  1245. #define P3 pix[-4 * xstride]

  1246. #define P2 pix[-3 * xstride]

  1247. #define P1 pix[-2 * xstride]

  1248. #define P0 pix[-1 * xstride]

  1249. #define Q0 pix[0 * xstride]

  1250. #define Q1 pix[1 * xstride]

  1251. #define Q2 pix[2 * xstride]

  1252. #define Q3 pix[3 * xstride]

  1253. 

  1254. // line three. used only for deblocking decision

  1255. #define TP3 pix[-4 * xstride + 3 * ystride]

  1256. #define TP2 pix[-3 * xstride + 3 * ystride]

  1257. #define TP1 pix[-2 * xstride + 3 * ystride]

  1258. #define TP0 pix[-1 * xstride + 3 * ystride]

  1259. #define TQ0 pix[0  * xstride + 3 * ystride]

  1260. #define TQ1 pix[1  * xstride + 3 * ystride]

  1261. #define TQ2 pix[2  * xstride + 3 * ystride]

  1262. #define TQ3 pix[3  * xstride + 3 * ystride]

  1263. 

  1264. static void FUNC(hevc_loop_filter_luma)(uint8_t *_pix,

  1265.                                         ptrdiff_t _xstride, ptrdiff_t _ystride,

  1266.                                         int beta, int *_tc,

  1267.                                         uint8_t *_no_p, uint8_t *_no_q)

  1268. {

  1269.     int d, j;

  1270.     pixel *pix        = (pixel *)_pix;

  1271.     ptrdiff_t xstride = _xstride / sizeof(pixel);

  1272.     ptrdiff_t ystride = _ystride / sizeof(pixel);

  1273. 

  1274.     beta <<= BIT_DEPTH - 8;

  1275. 

  1276.     for (j = 0; j < 2; j++) {

  1277.         const int dp0  = abs(P2  - 2 * P1  + P0);

  1278.         const int dq0  = abs(Q2  - 2 * Q1  + Q0);

  1279.         const int dp3  = abs(TP2 - 2 * TP1 + TP0);

  1280.         const int dq3  = abs(TQ2 - 2 * TQ1 + TQ0);

  1281.         const int d0   = dp0 + dq0;

  1282.         const int d3   = dp3 + dq3;

  1283.         const int tc   = _tc[j]   << (BIT_DEPTH - 8);

  1284.         const int no_p = _no_p[j];

  1285.         const int no_q = _no_q[j];

  1286. 

  1287.         if (d0 + d3 >= beta) {

  1288.             pix += 4 * ystride;

  1289.             continue;

  1290.         } else {

  1291.             const int beta_3 = beta >> 3;

  1292.             const int beta_2 = beta >> 2;

  1293.             const int tc25   = ((tc * 5 + 1) >> 1);

  1294. 

  1295.             if (abs(P3  -  P0) + abs(Q3  -  Q0) < beta_3 && abs(P0  -  Q0) < tc25 &&

  1296.                 abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&

  1297.                                       (d0 << 1) < beta_2 &&      (d3 << 1) < beta_2) {

  1298.                 // strong filtering

  1299.                 const int tc2 = tc << 1;

  1300.                 for (d = 0; d < 4; d++) {

  1301.                     const int p3 = P3;

  1302.                     const int p2 = P2;

  1303.                     const int p1 = P1;

  1304.                     const int p0 = P0;

  1305.                     const int q0 = Q0;

  1306.                     const int q1 = Q1;

  1307.                     const int q2 = Q2;

  1308.                     const int q3 = Q3;

  1309.                     if (!no_p) {

  1310.                         P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc2, tc2);

  1311.                         P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2);

  1312.                         P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc2, tc2);

  1313.                     }

  1314.                     if (!no_q) {

  1315.                         Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc2, tc2);

  1316.                         Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2);

  1317.                         Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc2, tc2);

  1318.                     }

  1319.                     pix += ystride;

  1320.                 }

  1321.             } else { // normal filtering

  1322.                 int nd_p = 1;

  1323.                 int nd_q = 1;

  1324.                 const int tc_2 = tc >> 1;

  1325.                 if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))

  1326.                     nd_p = 2;

  1327.                 if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))

  1328.                     nd_q = 2;

  1329. 

  1330.                 for (d = 0; d < 4; d++) {

  1331.                     const int p2 = P2;

  1332.                     const int p1 = P1;

  1333.                     const int p0 = P0;

  1334.                     const int q0 = Q0;

  1335.                     const int q1 = Q1;

  1336.                     const int q2 = Q2;

  1337.                     int delta0   = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4;

  1338.                     if (abs(delta0) < 10 * tc) {

  1339.                         delta0 = av_clip(delta0, -tc, tc);

  1340.                         if (!no_p)

  1341.                             P0 = av_clip_pixel(p0 + delta0);

  1342.                         if (!no_q)

  1343.                             Q0 = av_clip_pixel(q0 - delta0);

  1344.                         if (!no_p && nd_p > 1) {

  1345.                             const int deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2);

  1346.                             P1 = av_clip_pixel(p1 + deltap1);

  1347.                         }

  1348.                         if (!no_q && nd_q > 1) {

  1349.                             const int deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2);

  1350.                             Q1 = av_clip_pixel(q1 + deltaq1);

  1351.                         }

  1352.                     }

  1353.                     pix += ystride;

  1354.                 }

  1355.             }

  1356.         }

  1357.     }

  1358. }

  1359. 

  1360. static void FUNC(hevc_loop_filter_chroma)(uint8_t *_pix, ptrdiff_t _xstride,

  1361.                                           ptrdiff_t _ystride, int *_tc,

  1362.                                           uint8_t *_no_p, uint8_t *_no_q)

  1363. {

  1364.     int d, j, no_p, no_q;

  1365.     pixel *pix        = (pixel *)_pix;

  1366.     ptrdiff_t xstride = _xstride / sizeof(pixel);

  1367.     ptrdiff_t ystride = _ystride / sizeof(pixel);

  1368. 

  1369.     for (j = 0; j < 2; j++) {

  1370.         const int tc = _tc[j] << (BIT_DEPTH - 8);

  1371.         if (tc <= 0) {

  1372.             pix += 4 * ystride;

  1373.             continue;

  1374.         }

  1375.         no_p = _no_p[j];

  1376.         no_q = _no_q[j];

  1377. 

  1378.         for (d = 0; d < 4; d++) {

  1379.             int delta0;

  1380.             const int p1 = P1;

  1381.             const int p0 = P0;

  1382.             const int q0 = Q0;

  1383.             const int q1 = Q1;

  1384.             delta0 = av_clip((((q0 - p0) * 4) + p1 - q1 + 4) >> 3, -tc, tc);

  1385.             if (!no_p)

  1386.                 P0 = av_clip_pixel(p0 + delta0);

  1387.             if (!no_q)

  1388.                 Q0 = av_clip_pixel(q0 - delta0);

  1389.             pix += ystride;

  1390.         }

  1391.     }

  1392. }

  1393. 

  1394. static void FUNC(hevc_h_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,

  1395.                                             int *tc, uint8_t *no_p,

  1396.                                             uint8_t *no_q)

  1397. {

  1398.     FUNC(hevc_loop_filter_chroma)(pix, stride, sizeof(pixel), tc, no_p, no_q);

  1399. }

  1400. 

  1401. static void FUNC(hevc_v_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,

  1402.                                             int *tc, uint8_t *no_p,

  1403.                                             uint8_t *no_q)

  1404. {

  1405.     FUNC(hevc_loop_filter_chroma)(pix, sizeof(pixel), stride, tc, no_p, no_q);

  1406. }

  1407. 

  1408. static void FUNC(hevc_h_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,

  1409.                                           int beta, int *tc, uint8_t *no_p,

  1410.                                           uint8_t *no_q)

  1411. {

  1412.     FUNC(hevc_loop_filter_luma)(pix, stride, sizeof(pixel),

  1413.                                 beta, tc, no_p, no_q);

  1414. }

  1415. 

  1416. static void FUNC(hevc_v_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,

  1417.                                           int beta, int *tc, uint8_t *no_p,

  1418.                                           uint8_t *no_q)

  1419. {

  1420.     FUNC(hevc_loop_filter_luma)(pix, sizeof(pixel), stride,

  1421.                                 beta, tc, no_p, no_q);

  1422. }

  1423. 

  1424. #undef P3

  1425. #undef P2

  1426. #undef P1

  1427. #undef P0

  1428. #undef Q0

  1429. #undef Q1

  1430. #undef Q2

  1431. #undef Q3

  1432. 

  1433. #undef TP3

  1434. #undef TP2

  1435. #undef TP1

  1436. #undef TP0

  1437. #undef TQ0

  1438. #undef TQ1

  1439. #undef TQ2

  1440. #undef TQ3