falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
43487 Commits
32 Branches
283MB
Tree: a92fd8a062
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a92fd8a062'
${ noResults }
FFmpeg/libavcodec/hevcdsp_template.c

1429 lines
59KB
Raw Blame History 

  1. /*

  2.  * HEVC video decoder

  3.  *

  4.  * Copyright (C) 2012 - 2013 Guillaume Martres

  5.  *

  6.  * This file is part of Libav.

  7.  *

  8.  * Libav is free software; you can redistribute it and/or

  9.  * modify it under the terms of the GNU Lesser General Public

  10.  * License as published by the Free Software Foundation; either

  11.  * version 2.1 of the License, or (at your option) any later version.

  12.  *

  13.  * Libav is distributed in the hope that it will be useful,

  14.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  15.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  16.  * Lesser General Public License for more details.

  17.  *

  18.  * You should have received a copy of the GNU Lesser General Public

  19.  * License along with Libav; if not, write to the Free Software

  20.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  21.  */

  22. 

  23. #include "get_bits.h"

  24. #include "hevc.h"

  25. 

  26. #include "bit_depth_template.c"

  27. 

  28. static void FUNC(put_pcm)(uint8_t *_dst, ptrdiff_t stride, int size,

  29.                           GetBitContext *gb, int pcm_bit_depth)

  30. {

  31.     int x, y;

  32.     pixel *dst = (pixel *)_dst;

  33. 

  34.     stride /= sizeof(pixel);

  35. 

  36.     for (y = 0; y < size; y++) {

  37.         for (x = 0; x < size; x++)

  38.             dst[x] = get_bits(gb, pcm_bit_depth) << (BIT_DEPTH - pcm_bit_depth);

  39.         dst += stride;

  40.     }

  41. }

  42. 

  43. static av_always_inline void FUNC(add_residual)(uint8_t *_dst, int16_t *res,

  44.                                                 ptrdiff_t stride, int size)

  45. {

  46.     int x, y;

  47.     pixel *dst = (pixel *)_dst;

  48. 

  49.     stride /= sizeof(pixel);

  50. 

  51.     for (y = 0; y < size; y++) {

  52.         for (x = 0; x < size; x++) {

  53.             dst[x] = av_clip_pixel(dst[x] + *res);

  54.             res++;

  55.         }

  56.         dst += stride;

  57.     }

  58. }

  59. 

  60. static void FUNC(add_residual4x4)(uint8_t *_dst, int16_t *res,

  61.                                   ptrdiff_t stride)

  62. {

  63.     FUNC(add_residual)(_dst, res, stride, 4);

  64. }

  65. 

  66. static void FUNC(add_residual8x8)(uint8_t *_dst, int16_t *res,

  67.                                   ptrdiff_t stride)

  68. {

  69.     FUNC(add_residual)(_dst, res, stride, 8);

  70. }

  71. 

  72. static void FUNC(add_residual16x16)(uint8_t *_dst, int16_t *res,

  73.                                     ptrdiff_t stride)

  74. {

  75.     FUNC(add_residual)(_dst, res, stride, 16);

  76. }

  77. 

  78. static void FUNC(add_residual32x32)(uint8_t *_dst, int16_t *res,

  79.                                     ptrdiff_t stride)

  80. {

  81.     FUNC(add_residual)(_dst, res, stride, 32);

  82. }

  83. 

  84. static void FUNC(dequant)(int16_t *coeffs)

  85. {

  86.     int shift  = 13 - BIT_DEPTH;

  87. #if BIT_DEPTH <= 13

  88.     int offset = 1 << (shift - 1);

  89. #else

  90.     int offset = 0;

  91. #endif

  92.     int x, y;

  93. 

  94.     for (y = 0; y < 4 * 4; y += 4) {

  95.         for (x = 0; x < 4; x++)

  96.             coeffs[y + x] = (coeffs[y + x] + offset) >> shift;

  97.     }

  98. }

  99. 

  100. #define SET(dst, x)   (dst) = (x)

  101. #define SCALE(dst, x) (dst) = av_clip_int16(((x) + add) >> shift)

  102. 

  103. #define TR_4x4_LUMA(dst, src, step, assign)                             \

  104.     do {                                                                \

  105.         int c0 = src[0 * step] + src[2 * step];                         \

  106.         int c1 = src[2 * step] + src[3 * step];                         \

  107.         int c2 = src[0 * step] - src[3 * step];                         \

  108.         int c3 = 74 * src[1 * step];                                    \

  109.                                                                         \

  110.         assign(dst[2 * step], 74 * (src[0 * step] -                     \

  111.                                     src[2 * step] +                     \

  112.                                     src[3 * step]));                    \

  113.         assign(dst[0 * step], 29 * c0 + 55 * c1 + c3);                  \

  114.         assign(dst[1 * step], 55 * c2 - 29 * c1 + c3);                  \

  115.         assign(dst[3 * step], 55 * c0 + 29 * c2 - c3);                  \

  116.     } while (0)

  117. 

  118. static void FUNC(transform_4x4_luma)(int16_t *coeffs)

  119. {

  120.     int i;

  121.     int shift    = 7;

  122.     int add      = 1 << (shift - 1);

  123.     int16_t *src = coeffs;

  124. 

  125.     for (i = 0; i < 4; i++) {

  126.         TR_4x4_LUMA(src, src, 4, SCALE);

  127.         src++;

  128.     }

  129. 

  130.     shift = 20 - BIT_DEPTH;

  131.     add   = 1 << (shift - 1);

  132.     for (i = 0; i < 4; i++) {

  133.         TR_4x4_LUMA(coeffs, coeffs, 1, SCALE);

  134.         coeffs += 4;

  135.     }

  136. }

  137. 

  138. #undef TR_4x4_LUMA

  139. 

  140. #define TR_4(dst, src, dstep, sstep, assign)                            \

  141.     do {                                                                \

  142.         const int e0 = transform[8 * 0][0] * src[0 * sstep] +           \

  143.                        transform[8 * 2][0] * src[2 * sstep];            \

  144.         const int e1 = transform[8 * 0][1] * src[0 * sstep] +           \

  145.                        transform[8 * 2][1] * src[2 * sstep];            \

  146.         const int o0 = transform[8 * 1][0] * src[1 * sstep] +           \

  147.                        transform[8 * 3][0] * src[3 * sstep];            \

  148.         const int o1 = transform[8 * 1][1] * src[1 * sstep] +           \

  149.                        transform[8 * 3][1] * src[3 * sstep];            \

  150.                                                                         \

  151.         assign(dst[0 * dstep], e0 + o0);                                \

  152.         assign(dst[1 * dstep], e1 + o1);                                \

  153.         assign(dst[2 * dstep], e1 - o1);                                \

  154.         assign(dst[3 * dstep], e0 - o0);                                \

  155.     } while (0)

  156. 

  157. #define TR_8(dst, src, dstep, sstep, assign)                      \

  158.     do {                                                          \

  159.         int i, j;                                                 \

  160.         int e_8[4];                                               \

  161.         int o_8[4] = { 0 };                                       \

  162.         for (i = 0; i < 4; i++)                                   \

  163.             for (j = 1; j < 8; j += 2)                            \

  164.                 o_8[i] += transform[4 * j][i] * src[j * sstep];   \

  165.         TR_4(e_8, src, 1, 2 * sstep, SET);                        \

  166.                                                                   \

  167.         for (i = 0; i < 4; i++) {                                 \

  168.             assign(dst[i * dstep], e_8[i] + o_8[i]);              \

  169.             assign(dst[(7 - i) * dstep], e_8[i] - o_8[i]);        \

  170.         }                                                         \

  171.     } while (0)

  172. 

  173. #define TR_16(dst, src, dstep, sstep, assign)                     \

  174.     do {                                                          \

  175.         int i, j;                                                 \

  176.         int e_16[8];                                              \

  177.         int o_16[8] = { 0 };                                      \

  178.         for (i = 0; i < 8; i++)                                   \

  179.             for (j = 1; j < 16; j += 2)                           \

  180.                 o_16[i] += transform[2 * j][i] * src[j * sstep];  \

  181.         TR_8(e_16, src, 1, 2 * sstep, SET);                       \

  182.                                                                   \

  183.         for (i = 0; i < 8; i++) {                                 \

  184.             assign(dst[i * dstep], e_16[i] + o_16[i]);            \

  185.             assign(dst[(15 - i) * dstep], e_16[i] - o_16[i]);     \

  186.         }                                                         \

  187.     } while (0)

  188. 

  189. #define TR_32(dst, src, dstep, sstep, assign)                     \

  190.     do {                                                          \

  191.         int i, j;                                                 \

  192.         int e_32[16];                                             \

  193.         int o_32[16] = { 0 };                                     \

  194.         for (i = 0; i < 16; i++)                                  \

  195.             for (j = 1; j < 32; j += 2)                           \

  196.                 o_32[i] += transform[j][i] * src[j * sstep];      \

  197.         TR_16(e_32, src, 1, 2 * sstep, SET);                      \

  198.                                                                   \

  199.         for (i = 0; i < 16; i++) {                                \

  200.             assign(dst[i * dstep], e_32[i] + o_32[i]);            \

  201.             assign(dst[(31 - i) * dstep], e_32[i] - o_32[i]);     \

  202.         }                                                         \

  203.     } while (0)

  204. 

  205. #define IDCT(H)                                                   \

  206. static void FUNC(idct_ ## H ## x ## H )(int16_t *coeffs)          \

  207. {                                                                 \

  208.     int i;                                                        \

  209.     int      shift = 7;                                           \

  210.     int      add   = 1 << (shift - 1);                            \

  211.     int16_t *src   = coeffs;                                      \

  212.                                                                   \

  213.     for (i = 0; i < H; i++) {                                     \

  214.         TR_ ## H(src, src, H, H, SCALE);                          \

  215.         src++;                                                    \

  216.     }                                                             \

  217.                                                                   \

  218.     shift = 20 - BIT_DEPTH;                                       \

  219.     add   = 1 << (shift - 1);                                     \

  220.     for (i = 0; i < H; i++) {                                     \

  221.         TR_ ## H(coeffs, coeffs, 1, 1, SCALE);                    \

  222.         coeffs += H;                                              \

  223.     }                                                             \

  224. }

  225. 

  226. #define IDCT_DC(H)                                                \

  227. static void FUNC(idct_ ## H ## x ## H ## _dc)(int16_t *coeffs)    \

  228. {                                                                 \

  229.     int i, j;                                                     \

  230.     int shift = 14 - BIT_DEPTH;                                   \

  231.     int add   = 1 << (shift - 1);                                 \

  232.     int coeff = (((coeffs[0] + 1) >> 1) + add) >> shift;          \

  233.                                                                   \

  234.     for (j = 0; j < H; j++) {                                     \

  235.         for (i = 0; i < H; i++) {                                 \

  236.             coeffs[i + j * H] = coeff;                            \

  237.         }                                                         \

  238.     }                                                             \

  239. }

  240. 

  241. IDCT( 4)

  242. IDCT( 8)

  243. IDCT(16)

  244. IDCT(32)

  245. IDCT_DC( 4)

  246. IDCT_DC( 8)

  247. IDCT_DC(16)

  248. IDCT_DC(32)

  249. #undef TR_4

  250. #undef TR_8

  251. #undef TR_16

  252. #undef TR_32

  253. 

  254. #undef SET

  255. #undef SCALE

  256. #undef ADD_AND_SCALE

  257. 

  258. static void FUNC(sao_band_filter)(uint8_t *_dst, uint8_t *_src,

  259.                                   ptrdiff_t stride, SAOParams *sao,

  260.                                   int *borders, int width, int height,

  261.                                   int c_idx, int class)

  262. {

  263.     pixel *dst = (pixel *)_dst;

  264.     pixel *src = (pixel *)_src;

  265.     int offset_table[32] = { 0 };

  266.     int k, y, x;

  267.     int chroma = !!c_idx;

  268.     int shift  = BIT_DEPTH - 5;

  269.     int *sao_offset_val = sao->offset_val[c_idx];

  270.     int sao_left_class  = sao->band_position[c_idx];

  271.     int init_y = 0, init_x = 0;

  272. 

  273.     stride /= sizeof(pixel);

  274. 

  275.     switch (class) {

  276.     case 0:

  277.         if (!borders[2])

  278.             width -= (8 >> chroma) + 2;

  279.         if (!borders[3])

  280.             height -= (4 >> chroma) + 2;

  281.         break;

  282.     case 1:

  283.         init_y = -(4 >> chroma) - 2;

  284.         if (!borders[2])

  285.             width -= (8 >> chroma) + 2;

  286.         height = (4 >> chroma) + 2;

  287.         break;

  288.     case 2:

  289.         init_x = -(8 >> chroma) - 2;

  290.         width  =  (8 >> chroma) + 2;

  291.         if (!borders[3])

  292.             height -= (4 >> chroma) + 2;

  293.         break;

  294.     case 3:

  295.         init_y = -(4 >> chroma) - 2;

  296.         init_x = -(8 >> chroma) - 2;

  297.         width  =  (8 >> chroma) + 2;

  298.         height =  (4 >> chroma) + 2;

  299.         break;

  300.     }

  301. 

  302.     dst = dst + (init_y * stride + init_x);

  303.     src = src + (init_y * stride + init_x);

  304.     for (k = 0; k < 4; k++)

  305.         offset_table[(k + sao_left_class) & 31] = sao_offset_val[k + 1];

  306.     for (y = 0; y < height; y++) {

  307.         for (x = 0; x < width; x++)

  308.             dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]);

  309.         dst += stride;

  310.         src += stride;

  311.     }

  312. }

  313. 

  314. static void FUNC(sao_band_filter_0)(uint8_t *dst, uint8_t *src,

  315.                                     ptrdiff_t stride, SAOParams *sao,

  316.                                     int *borders, int width, int height,

  317.                                     int c_idx)

  318. {

  319.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  320.                           width, height, c_idx, 0);

  321. }

  322. 

  323. static void FUNC(sao_band_filter_1)(uint8_t *dst, uint8_t *src,

  324.                                     ptrdiff_t stride, SAOParams *sao,

  325.                                     int *borders, int width, int height,

  326.                                     int c_idx)

  327. {

  328.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  329.                           width, height, c_idx, 1);

  330. }

  331. 

  332. static void FUNC(sao_band_filter_2)(uint8_t *dst, uint8_t *src,

  333.                                     ptrdiff_t stride, SAOParams *sao,

  334.                                     int *borders, int width, int height,

  335.                                     int c_idx)

  336. {

  337.     FUNC(sao_band_filter)(dst, src, stride, sao, borders,

  338.                           width, height, c_idx, 2);

  339. }

  340. 

  341. static void FUNC(sao_band_filter_3)(uint8_t *_dst, uint8_t *_src,

  342.                                     ptrdiff_t stride, SAOParams *sao,

  343.                                     int *borders, int width, int height,

  344.                                     int c_idx)

  345. {

  346.     FUNC(sao_band_filter)(_dst, _src, stride, sao, borders,

  347.                           width, height, c_idx, 3);

  348. }

  349. 

  350. static void FUNC(sao_edge_filter_0)(uint8_t *_dst, uint8_t *_src,

  351.                                     ptrdiff_t stride, SAOParams *sao,

  352.                                     int *borders, int _width, int _height,

  353.                                     int c_idx, uint8_t vert_edge,

  354.                                     uint8_t horiz_edge, uint8_t diag_edge)

  355. {

  356.     int x, y;

  357.     pixel *dst = (pixel *)_dst;

  358.     pixel *src = (pixel *)_src;

  359.     int chroma = !!c_idx;

  360.     int *sao_offset_val = sao->offset_val[c_idx];

  361.     int sao_eo_class    = sao->eo_class[c_idx];

  362.     int init_x = 0, init_y = 0, width = _width, height = _height;

  363. 

  364.     static const int8_t pos[4][2][2] = {

  365.         { { -1,  0 }, {  1, 0 } }, // horizontal

  366.         { {  0, -1 }, {  0, 1 } }, // vertical

  367.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  368.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  369.     };

  370.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  371. 

  372. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  373. 

  374.     stride /= sizeof(pixel);

  375. 

  376.     if (!borders[2])

  377.         width -= (8 >> chroma) + 2;

  378.     if (!borders[3])

  379.         height -= (4 >> chroma) + 2;

  380. 

  381.     dst = dst + (init_y * stride + init_x);

  382.     src = src + (init_y * stride + init_x);

  383.     init_y = init_x = 0;

  384.     if (sao_eo_class != SAO_EO_VERT) {

  385.         if (borders[0]) {

  386.             int offset_val = sao_offset_val[0];

  387.             int y_stride   = 0;

  388.             for (y = 0; y < height; y++) {

  389.                 dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);

  390.                 y_stride     += stride;

  391.             }

  392.             init_x = 1;

  393.         }

  394.         if (borders[2]) {

  395.             int offset_val = sao_offset_val[0];

  396.             int x_stride   = width - 1;

  397.             for (x = 0; x < height; x++) {

  398.                 dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);

  399.                 x_stride     += stride;

  400.             }

  401.             width--;

  402.         }

  403.     }

  404.     if (sao_eo_class != SAO_EO_HORIZ) {

  405.         if (borders[1]) {

  406.             int offset_val = sao_offset_val[0];

  407.             for (x = init_x; x < width; x++)

  408.                 dst[x] = av_clip_pixel(src[x] + offset_val);

  409.             init_y = 1;

  410.         }

  411.         if (borders[3]) {

  412.             int offset_val = sao_offset_val[0];

  413.             int y_stride   = stride * (height - 1);

  414.             for (x = init_x; x < width; x++)

  415.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);

  416.             height--;

  417.         }

  418.     }

  419.     {

  420.         int y_stride = init_y * stride;

  421.         int pos_0_0  = pos[sao_eo_class][0][0];

  422.         int pos_0_1  = pos[sao_eo_class][0][1];

  423.         int pos_1_0  = pos[sao_eo_class][1][0];

  424.         int pos_1_1  = pos[sao_eo_class][1][1];

  425. 

  426.         int y_stride_0_1 = (init_y + pos_0_1) * stride;

  427.         int y_stride_1_1 = (init_y + pos_1_1) * stride;

  428.         for (y = init_y; y < height; y++) {

  429.             for (x = init_x; x < width; x++) {

  430.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  431.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  432.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  433.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  434.             }

  435.             y_stride     += stride;

  436.             y_stride_0_1 += stride;

  437.             y_stride_1_1 += stride;

  438.         }

  439.     }

  440. 

  441.     {

  442.         // Restore pixels that can't be modified

  443.         int save_upper_left = !diag_edge && sao_eo_class == SAO_EO_135D && !borders[0] && !borders[1];

  444.         if (vert_edge && sao_eo_class != SAO_EO_VERT)

  445.             for (y = init_y+save_upper_left; y< height; y++)

  446.                 dst[y*stride] = src[y*stride];

  447.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  448.             for(x = init_x+save_upper_left; x<width; x++)

  449.                 dst[x] = src[x];

  450.         if(diag_edge && sao_eo_class == SAO_EO_135D)

  451.             dst[0] = src[0];

  452.     }

  453. 

  454. #undef CMP

  455. }

  456. 

  457. static void FUNC(sao_edge_filter_1)(uint8_t *_dst, uint8_t *_src,

  458.                                     ptrdiff_t stride, SAOParams *sao,

  459.                                     int *borders, int _width, int _height,

  460.                                     int c_idx, uint8_t vert_edge,

  461.                                     uint8_t horiz_edge, uint8_t diag_edge)

  462. {

  463.     int x, y;

  464.     pixel *dst = (pixel *)_dst;

  465.     pixel *src = (pixel *)_src;

  466.     int chroma = !!c_idx;

  467.     int *sao_offset_val = sao->offset_val[c_idx];

  468.     int sao_eo_class    = sao->eo_class[c_idx];

  469.     int init_x = 0, init_y = 0, width = _width, height = _height;

  470. 

  471.     static const int8_t pos[4][2][2] = {

  472.         { { -1, 0  }, { 1,  0 } }, // horizontal

  473.         { { 0,  -1 }, { 0,  1 } }, // vertical

  474.         { { -1, -1 }, { 1,  1 } }, // 45 degree

  475.         { { 1,  -1 }, { -1, 1 } }, // 135 degree

  476.     };

  477.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  478. 

  479. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  480. 

  481.     stride /= sizeof(pixel);

  482. 

  483.     init_y = -(4 >> chroma) - 2;

  484.     if (!borders[2])

  485.         width -= (8 >> chroma) + 2;

  486.     height = (4 >> chroma) + 2;

  487. 

  488.     dst = dst + (init_y * stride + init_x);

  489.     src = src + (init_y * stride + init_x);

  490.     init_y = init_x = 0;

  491.     if (sao_eo_class != SAO_EO_VERT) {

  492.         if (borders[0]) {

  493.             int offset_val = sao_offset_val[0];

  494.             int y_stride   = 0;

  495.             for (y = 0; y < height; y++) {

  496.                 dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);

  497.                 y_stride     += stride;

  498.             }

  499.             init_x = 1;

  500.         }

  501.         if (borders[2]) {

  502.             int offset_val = sao_offset_val[0];

  503.             int x_stride   = width - 1;

  504.             for (x = 0; x < height; x++) {

  505.                 dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);

  506.                 x_stride     += stride;

  507.             }

  508.             width--;

  509.         }

  510.     }

  511.     {

  512.         int y_stride = init_y * stride;

  513.         int pos_0_0  = pos[sao_eo_class][0][0];

  514.         int pos_0_1  = pos[sao_eo_class][0][1];

  515.         int pos_1_0  = pos[sao_eo_class][1][0];

  516.         int pos_1_1  = pos[sao_eo_class][1][1];

  517. 

  518.         int y_stride_0_1 = (init_y + pos_0_1) * stride;

  519.         int y_stride_1_1 = (init_y + pos_1_1) * stride;

  520.         for (y = init_y; y < height; y++) {

  521.             for (x = init_x; x < width; x++) {

  522.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  523.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  524.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  525.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  526.             }

  527.             y_stride     += stride;

  528.             y_stride_0_1 += stride;

  529.             y_stride_1_1 += stride;

  530.         }

  531.     }

  532. 

  533.     {

  534.         // Restore pixels that can't be modified

  535.         int save_lower_left = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[0];

  536.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  537.             for(y = init_y; y< height-save_lower_left; y++)

  538.                 dst[y*stride] = src[y*stride];

  539.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  540.             for(x = init_x+save_lower_left; x<width; x++)

  541.                 dst[(height-1)*stride+x] = src[(height-1)*stride+x];

  542.         if(diag_edge && sao_eo_class == SAO_EO_45D)

  543.             dst[stride*(height-1)] = src[stride*(height-1)];

  544.     }

  545. 

  546. #undef CMP

  547. }

  548. 

  549. static void FUNC(sao_edge_filter_2)(uint8_t *_dst, uint8_t *_src,

  550.                                     ptrdiff_t stride, SAOParams *sao,

  551.                                     int *borders, int _width, int _height,

  552.                                     int c_idx, uint8_t vert_edge,

  553.                                     uint8_t horiz_edge, uint8_t diag_edge)

  554. {

  555.     int x, y;

  556.     pixel *dst = (pixel *)_dst;

  557.     pixel *src = (pixel *)_src;

  558.     int chroma = !!c_idx;

  559.     int *sao_offset_val = sao->offset_val[c_idx];

  560.     int sao_eo_class    = sao->eo_class[c_idx];

  561.     int init_x = 0, init_y = 0, width = _width, height = _height;

  562. 

  563.     static const int8_t pos[4][2][2] = {

  564.         { { -1,  0 }, {  1, 0 } }, // horizontal

  565.         { {  0, -1 }, {  0, 1 } }, // vertical

  566.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  567.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  568.     };

  569.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  570. 

  571. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  572. 

  573.     stride /= sizeof(pixel);

  574. 

  575.     init_x = -(8 >> chroma) - 2;

  576.     width  =  (8 >> chroma) + 2;

  577.     if (!borders[3])

  578.         height -= (4 >> chroma) + 2;

  579. 

  580.     dst = dst + (init_y * stride + init_x);

  581.     src = src + (init_y * stride + init_x);

  582.     init_y = init_x = 0;

  583.     if (sao_eo_class != SAO_EO_HORIZ) {

  584.         if (borders[1]) {

  585.             int offset_val = sao_offset_val[0];

  586.             for (x = init_x; x < width; x++)

  587.                 dst[x] = av_clip_pixel(src[x] + offset_val);

  588.             init_y = 1;

  589.         }

  590.         if (borders[3]) {

  591.             int offset_val = sao_offset_val[0];

  592.             int y_stride   = stride * (height - 1);

  593.             for (x = init_x; x < width; x++)

  594.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);

  595.             height--;

  596.         }

  597.     }

  598.     {

  599.         int y_stride = init_y * stride;

  600.         int pos_0_0  = pos[sao_eo_class][0][0];

  601.         int pos_0_1  = pos[sao_eo_class][0][1];

  602.         int pos_1_0  = pos[sao_eo_class][1][0];

  603.         int pos_1_1  = pos[sao_eo_class][1][1];

  604. 

  605.         int y_stride_0_1 = (init_y + pos_0_1) * stride;

  606.         int y_stride_1_1 = (init_y + pos_1_1) * stride;

  607.         for (y = init_y; y < height; y++) {

  608.             for (x = init_x; x < width; x++) {

  609.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  610.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  611.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  612.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  613.             }

  614.             y_stride     += stride;

  615.             y_stride_0_1 += stride;

  616.             y_stride_1_1 += stride;

  617.         }

  618.     }

  619. 

  620.     {

  621.         // Restore pixels that can't be modified

  622.         int save_upper_right = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[1];

  623.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  624.             for(y = init_y+save_upper_right; y< height; y++)

  625.                 dst[y*stride+width-1] = src[y*stride+width-1];

  626.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  627.             for(x = init_x; x<width-save_upper_right; x++)

  628.                 dst[x] = src[x];

  629.         if(diag_edge && sao_eo_class == SAO_EO_45D)

  630.             dst[width-1] = src[width-1];

  631.     }

  632. #undef CMP

  633. }

  634. 

  635. static void FUNC(sao_edge_filter_3)(uint8_t *_dst, uint8_t *_src,

  636.                                     ptrdiff_t stride, SAOParams *sao,

  637.                                     int *borders, int _width, int _height,

  638.                                     int c_idx, uint8_t vert_edge,

  639.                                     uint8_t horiz_edge, uint8_t diag_edge)

  640. {

  641.     int x, y;

  642.     pixel *dst = (pixel *)_dst;

  643.     pixel *src = (pixel *)_src;

  644.     int chroma = !!c_idx;

  645.     int *sao_offset_val = sao->offset_val[c_idx];

  646.     int sao_eo_class    = sao->eo_class[c_idx];

  647.     int init_x = 0, init_y = 0, width = _width, height = _height;

  648. 

  649.     static const int8_t pos[4][2][2] = {

  650.         { { -1,  0 }, {  1, 0 } }, // horizontal

  651.         { {  0, -1 }, {  0, 1 } }, // vertical

  652.         { { -1, -1 }, {  1, 1 } }, // 45 degree

  653.         { {  1, -1 }, { -1, 1 } }, // 135 degree

  654.     };

  655.     static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

  656. 

  657. #define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))

  658. 

  659.     stride /= sizeof(pixel);

  660. 

  661.     init_y = -(4 >> chroma) - 2;

  662.     init_x = -(8 >> chroma) - 2;

  663.     width  =  (8 >> chroma) + 2;

  664.     height =  (4 >> chroma) + 2;

  665. 

  666. 

  667.     dst    = dst + (init_y * stride + init_x);

  668.     src    = src + (init_y * stride + init_x);

  669.     init_y = init_x = 0;

  670. 

  671.     {

  672.         int y_stride = init_y * stride;

  673.         int pos_0_0  = pos[sao_eo_class][0][0];

  674.         int pos_0_1  = pos[sao_eo_class][0][1];

  675.         int pos_1_0  = pos[sao_eo_class][1][0];

  676.         int pos_1_1  = pos[sao_eo_class][1][1];

  677. 

  678.         int y_stride_0_1 = (init_y + pos_0_1) * stride;

  679.         int y_stride_1_1 = (init_y + pos_1_1) * stride;

  680. 

  681.         for (y = init_y; y < height; y++) {

  682.             for (x = init_x; x < width; x++) {

  683.                 int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);

  684.                 int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);

  685.                 int offset_val    = edge_idx[2 + diff0 + diff1];

  686.                 dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);

  687.             }

  688.             y_stride     += stride;

  689.             y_stride_0_1 += stride;

  690.             y_stride_1_1 += stride;

  691.         }

  692.     }

  693. 

  694.     {

  695.         // Restore pixels that can't be modified

  696.         int save_lower_right = !diag_edge && sao_eo_class == SAO_EO_135D;

  697.         if(vert_edge && sao_eo_class != SAO_EO_VERT)

  698.             for(y = init_y; y< height-save_lower_right; y++)

  699.                 dst[y*stride+width-1] = src[y*stride+width-1];

  700.         if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)

  701.             for(x = init_x; x<width-save_lower_right; x++)

  702.                 dst[(height-1)*stride+x] = src[(height-1)*stride+x];

  703.         if(diag_edge && sao_eo_class == SAO_EO_135D)

  704.             dst[stride*(height-1)+width-1] = src[stride*(height-1)+width-1];

  705.     }

  706. #undef CMP

  707. }

  708. 

  709. #undef SET

  710. #undef SCALE

  711. #undef TR_4

  712. #undef TR_8

  713. #undef TR_16

  714. #undef TR_32

  715. 

  716. static av_always_inline void

  717. FUNC(put_hevc_qpel_pixels)(int16_t *dst, ptrdiff_t dststride,

  718.                            uint8_t *_src, ptrdiff_t _srcstride,

  719.                            int width, int height, int mx, int my,

  720.                            int16_t* mcbuffer)

  721. {

  722.     int x, y;

  723.     pixel *src          = (pixel *)_src;

  724.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  725. 

  726.     dststride /= sizeof(*dst);

  727.     for (y = 0; y < height; y++) {

  728.         for (x = 0; x < width; x++)

  729.             dst[x] = src[x] << (14 - BIT_DEPTH);

  730.         src += srcstride;

  731.         dst += dststride;

  732.     }

  733. }

  734. 

  735. #define QPEL_FILTER_1(src, stride)      \

  736.     (1 * -src[x - 3 * stride] +         \

  737.      4 *  src[x - 2 * stride] -         \

  738.     10 *  src[x -     stride] +         \

  739.     58 *  src[x]              +         \

  740.     17 *  src[x +     stride] -         \

  741.      5 *  src[x + 2 * stride] +         \

  742.      1 *  src[x + 3 * stride])

  743. 

  744. #define QPEL_FILTER_2(src, stride)      \

  745.     (1  * -src[x - 3 * stride] +        \

  746.      4  *  src[x - 2 * stride] -        \

  747.     11  *  src[x -     stride] +        \

  748.     40  *  src[x]              +        \

  749.     40  *  src[x +     stride] -        \

  750.     11  *  src[x + 2 * stride] +        \

  751.      4  *  src[x + 3 * stride] -        \

  752.      1  *  src[x + 4 * stride])

  753. 

  754. #define QPEL_FILTER_3(src, stride)      \

  755.     (1  * src[x - 2 * stride] -         \

  756.      5  * src[x -     stride] +         \

  757.     17  * src[x]              +         \

  758.     58  * src[x + stride]     -         \

  759.     10  * src[x + 2 * stride] +         \

  760.      4  * src[x + 3 * stride] -         \

  761.      1  * src[x + 4 * stride])

  762. 

  763. 

  764. #define PUT_HEVC_QPEL_H(H)                                                     \

  765. static void FUNC(put_hevc_qpel_h ## H)(int16_t *dst,  ptrdiff_t dststride,     \

  766.                                        uint8_t *_src, ptrdiff_t _srcstride,    \

  767.                                        int width, int height,                  \

  768.                                        int16_t* mcbuffer)                      \

  769. {                                                                              \

  770.     int x, y;                                                                  \

  771.     pixel *src = (pixel*)_src;                                                 \

  772.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  773.                                                                                \

  774.     dststride /= sizeof(*dst);                                                 \

  775.     for (y = 0; y < height; y++) {                                             \

  776.         for (x = 0; x < width; x++)                                            \

  777.             dst[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \

  778.         src += srcstride;                                                      \

  779.         dst += dststride;                                                      \

  780.     }                                                                          \

  781. }

  782. 

  783. #define PUT_HEVC_QPEL_V(V)                                                     \

  784. static void FUNC(put_hevc_qpel_v ## V)(int16_t *dst,  ptrdiff_t dststride,     \

  785.                                        uint8_t *_src, ptrdiff_t _srcstride,    \

  786.                                        int width, int height,                  \

  787.                                        int16_t* mcbuffer)                      \

  788. {                                                                              \

  789.     int x, y;                                                                  \

  790.     pixel *src = (pixel*)_src;                                                 \

  791.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  792.                                                                                \

  793.     dststride /= sizeof(*dst);                                                 \

  794.     for (y = 0; y < height; y++)  {                                            \

  795.         for (x = 0; x < width; x++)                                            \

  796.             dst[x] = QPEL_FILTER_ ## V(src, srcstride) >> (BIT_DEPTH - 8);     \

  797.         src += srcstride;                                                      \

  798.         dst += dststride;                                                      \

  799.     }                                                                          \

  800. }

  801. 

  802. #define PUT_HEVC_QPEL_HV(H, V)                                                 \

  803. static void FUNC(put_hevc_qpel_h ## H ## v ## V)(int16_t *dst,                 \

  804.                                                  ptrdiff_t dststride,          \

  805.                                                  uint8_t *_src,                \

  806.                                                  ptrdiff_t _srcstride,         \

  807.                                                  int width, int height,        \

  808.                                                  int16_t* mcbuffer)            \

  809. {                                                                              \

  810.     int x, y;                                                                  \

  811.     pixel *src = (pixel*)_src;                                                 \

  812.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \

  813.                                                                                \

  814.     int16_t tmp_array[(MAX_PB_SIZE + 7) * MAX_PB_SIZE];                        \

  815.     int16_t *tmp = tmp_array;                                                  \

  816.                                                                                \

  817.     dststride /= sizeof(*dst);                                                 \

  818.     src -= ff_hevc_qpel_extra_before[V] * srcstride;                           \

  819.                                                                                \

  820.     for (y = 0; y < height + ff_hevc_qpel_extra[V]; y++) {                     \

  821.         for (x = 0; x < width; x++)                                            \

  822.             tmp[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \

  823.         src += srcstride;                                                      \

  824.         tmp += MAX_PB_SIZE;                                                    \

  825.     }                                                                          \

  826.                                                                                \

  827.     tmp = tmp_array + ff_hevc_qpel_extra_before[V] * MAX_PB_SIZE;              \

  828.                                                                                \

  829.     for (y = 0; y < height; y++) {                                             \

  830.         for (x = 0; x < width; x++)                                            \

  831.             dst[x] = QPEL_FILTER_ ## V(tmp, MAX_PB_SIZE) >> 6;                 \

  832.         tmp += MAX_PB_SIZE;                                                    \

  833.         dst += dststride;                                                      \

  834.     }                                                                          \

  835. }

  836. 

  837. PUT_HEVC_QPEL_H(1)

  838. PUT_HEVC_QPEL_H(2)

  839. PUT_HEVC_QPEL_H(3)

  840. PUT_HEVC_QPEL_V(1)

  841. PUT_HEVC_QPEL_V(2)

  842. PUT_HEVC_QPEL_V(3)

  843. PUT_HEVC_QPEL_HV(1, 1)

  844. PUT_HEVC_QPEL_HV(1, 2)

  845. PUT_HEVC_QPEL_HV(1, 3)

  846. PUT_HEVC_QPEL_HV(2, 1)

  847. PUT_HEVC_QPEL_HV(2, 2)

  848. PUT_HEVC_QPEL_HV(2, 3)

  849. PUT_HEVC_QPEL_HV(3, 1)

  850. PUT_HEVC_QPEL_HV(3, 2)

  851. PUT_HEVC_QPEL_HV(3, 3)

  852. 

  853. #define QPEL(W)                                                                             \

  854. static void FUNC(put_hevc_qpel_pixels_ ## W)(int16_t *dst, ptrdiff_t dststride,             \

  855.                                              uint8_t *src, ptrdiff_t srcstride,             \

  856.                                              int height, int mx, int my,                    \

  857.                                              int16_t *mcbuffer)                             \

  858. {                                                                                           \

  859.     FUNC(put_hevc_qpel_pixels)(dst, dststride, src, srcstride, W, height,                   \

  860.                                mx, my, mcbuffer);                                           \

  861. }                                                                                           \

  862.                                                                                             \

  863. static void FUNC(put_hevc_qpel_h_ ## W)(int16_t *dst, ptrdiff_t dststride,                  \

  864.                                         uint8_t *src, ptrdiff_t srcstride,                  \

  865.                                         int height, int mx, int my,                         \

  866.                                         int16_t *mcbuffer)                                  \

  867. {                                                                                           \

  868.     if (mx == 1)                                                                            \

  869.         FUNC(put_hevc_qpel_h1)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  870.     else if (mx == 2)                                                                       \

  871.         FUNC(put_hevc_qpel_h2)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  872.     else                                                                                    \

  873.         FUNC(put_hevc_qpel_h3)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  874. }                                                                                           \

  875.                                                                                             \

  876. static void FUNC(put_hevc_qpel_v_ ## W)(int16_t *dst, ptrdiff_t dststride,                  \

  877.                                              uint8_t *src, ptrdiff_t srcstride,             \

  878.                                              int height, int mx, int my,                    \

  879.                                              int16_t *mcbuffer)                             \

  880. {                                                                                           \

  881.     if (my == 1)                                                                            \

  882.         FUNC(put_hevc_qpel_v1)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  883.     else if (my == 2)                                                                       \

  884.         FUNC(put_hevc_qpel_v2)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  885.     else                                                                                    \

  886.         FUNC(put_hevc_qpel_v3)(dst, dststride, src, srcstride, W, height, mcbuffer);        \

  887. }                                                                                           \

  888.                                                                                             \

  889. static void FUNC(put_hevc_qpel_hv_ ## W)(int16_t *dst, ptrdiff_t dststride,                 \

  890.                                              uint8_t *src, ptrdiff_t srcstride,             \

  891.                                              int height, int mx, int my,                    \

  892.                                              int16_t *mcbuffer)                             \

  893. {                                                                                           \

  894.     if (my == 1) {                                                                          \

  895.         if (mx == 1)                                                                        \

  896.             FUNC(put_hevc_qpel_h1v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  897.         else if (mx == 2)                                                                   \

  898.             FUNC(put_hevc_qpel_h2v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  899.         else                                                                                \

  900.             FUNC(put_hevc_qpel_h3v1)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  901.     } else if (my == 2) {                                                                   \

  902.         if (mx == 1)                                                                        \

  903.             FUNC(put_hevc_qpel_h1v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  904.         else if (mx == 2)                                                                   \

  905.             FUNC(put_hevc_qpel_h2v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  906.         else                                                                                \

  907.             FUNC(put_hevc_qpel_h3v2)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  908.     } else {                                                                                \

  909.         if (mx == 1)                                                                        \

  910.             FUNC(put_hevc_qpel_h1v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  911.         else if (mx == 2)                                                                   \

  912.             FUNC(put_hevc_qpel_h2v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  913.         else                                                                                \

  914.             FUNC(put_hevc_qpel_h3v3)(dst, dststride, src, srcstride, W, height, mcbuffer);  \

  915.     }                                                                                       \

  916. }

  917. 

  918. QPEL(64)

  919. QPEL(48)

  920. QPEL(32)

  921. QPEL(24)

  922. QPEL(16)

  923. QPEL(12)

  924. QPEL(8)

  925. QPEL(4)

  926. 

  927. static inline void FUNC(put_hevc_epel_pixels)(int16_t *dst, ptrdiff_t dststride,

  928.                                               uint8_t *_src, ptrdiff_t _srcstride,

  929.                                               int width, int height, int mx, int my,

  930.                                               int16_t* mcbuffer)

  931. {

  932.     int x, y;

  933.     pixel *src          = (pixel *)_src;

  934.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  935. 

  936.     dststride /= sizeof(*dst);

  937.     for (y = 0; y < height; y++) {

  938.         for (x = 0; x < width; x++)

  939.             dst[x] = src[x] << (14 - BIT_DEPTH);

  940.         src += srcstride;

  941.         dst += dststride;

  942.     }

  943. }

  944. 

  945. #define EPEL_FILTER(src, stride)                \

  946.     (filter_0 * src[x - stride] +               \

  947.      filter_1 * src[x]          +               \

  948.      filter_2 * src[x + stride] +               \

  949.      filter_3 * src[x + 2 * stride])

  950. 

  951. static inline void FUNC(put_hevc_epel_h)(int16_t *dst, ptrdiff_t dststride,

  952.                                          uint8_t *_src, ptrdiff_t _srcstride,

  953.                                          int width, int height, int mx, int my,

  954.                                          int16_t* mcbuffer)

  955. {

  956.     int x, y;

  957.     pixel *src = (pixel *)_src;

  958.     ptrdiff_t srcstride  = _srcstride / sizeof(pixel);

  959.     const int16_t *filter = ff_hevc_epel_coeffs[mx - 1];

  960.     int8_t filter_0 = filter[0];

  961.     int8_t filter_1 = filter[1];

  962.     int8_t filter_2 = filter[2];

  963.     int8_t filter_3 = filter[3];

  964.     dststride /= sizeof(*dst);

  965.     for (y = 0; y < height; y++) {

  966.         for (x = 0; x < width; x++)

  967.             dst[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);

  968.         src += srcstride;

  969.         dst += dststride;

  970.     }

  971. }

  972. 

  973. static inline void FUNC(put_hevc_epel_v)(int16_t *dst, ptrdiff_t dststride,

  974.                                          uint8_t *_src, ptrdiff_t _srcstride,

  975.                                          int width, int height, int mx, int my,

  976.                                          int16_t* mcbuffer)

  977. {

  978.     int x, y;

  979.     pixel *src = (pixel *)_src;

  980.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  981.     const int16_t *filter = ff_hevc_epel_coeffs[my - 1];

  982.     int8_t filter_0 = filter[0];

  983.     int8_t filter_1 = filter[1];

  984.     int8_t filter_2 = filter[2];

  985.     int8_t filter_3 = filter[3];

  986. 

  987.     dststride /= sizeof(*dst);

  988.     for (y = 0; y < height; y++) {

  989.         for (x = 0; x < width; x++)

  990.             dst[x] = EPEL_FILTER(src, srcstride) >> (BIT_DEPTH - 8);

  991.         src += srcstride;

  992.         dst += dststride;

  993.     }

  994. }

  995. 

  996. static inline void FUNC(put_hevc_epel_hv)(int16_t *dst, ptrdiff_t dststride,

  997.                                           uint8_t *_src, ptrdiff_t _srcstride,

  998.                                           int width, int height, int mx, int my,

  999.                                           int16_t* mcbuffer)

  1000. {

  1001.     int x, y;

  1002.     pixel *src = (pixel *)_src;

  1003.     ptrdiff_t srcstride = _srcstride / sizeof(pixel);

  1004.     const int16_t *filter_h = ff_hevc_epel_coeffs[mx - 1];

  1005.     const int16_t *filter_v = ff_hevc_epel_coeffs[my - 1];

  1006.     int8_t filter_0 = filter_h[0];

  1007.     int8_t filter_1 = filter_h[1];

  1008.     int8_t filter_2 = filter_h[2];

  1009.     int8_t filter_3 = filter_h[3];

  1010.     int16_t tmp_array[(MAX_PB_SIZE + 3) * MAX_PB_SIZE];

  1011.     int16_t *tmp = tmp_array;

  1012. 

  1013.     dststride /= sizeof(*dst);

  1014.     src -= EPEL_EXTRA_BEFORE * srcstride;

  1015. 

  1016.     for (y = 0; y < height + EPEL_EXTRA; y++) {

  1017.         for (x = 0; x < width; x++)

  1018.             tmp[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);

  1019.         src += srcstride;

  1020.         tmp += MAX_PB_SIZE;

  1021.     }

  1022. 

  1023.     tmp      = tmp_array + EPEL_EXTRA_BEFORE * MAX_PB_SIZE;

  1024.     filter_0 = filter_v[0];

  1025.     filter_1 = filter_v[1];

  1026.     filter_2 = filter_v[2];

  1027.     filter_3 = filter_v[3];

  1028.     for (y = 0; y < height; y++) {

  1029.         for (x = 0; x < width; x++)

  1030.             dst[x] = EPEL_FILTER(tmp, MAX_PB_SIZE) >> 6;

  1031.         tmp += MAX_PB_SIZE;

  1032.         dst += dststride;

  1033.     }

  1034. }

  1035. 

  1036. #define EPEL(W)                                                                 \

  1037. static void FUNC(put_hevc_epel_pixels_ ## W)(int16_t *dst, ptrdiff_t dststride, \

  1038.                                              uint8_t *src, ptrdiff_t srcstride, \

  1039.                                              int height, int mx, int my,        \

  1040.                                              int16_t *mcbuffer)                 \

  1041. {                                                                               \

  1042.     FUNC(put_hevc_epel_pixels)(dst, dststride, src, srcstride,                  \

  1043.                                W, height, mx, my, mcbuffer);                    \

  1044. }                                                                               \

  1045. static void FUNC(put_hevc_epel_h_ ## W)(int16_t *dst, ptrdiff_t dststride,      \

  1046.                                         uint8_t *src, ptrdiff_t srcstride,      \

  1047.                                         int height, int mx, int my,             \

  1048.                                         int16_t *mcbuffer)                      \

  1049. {                                                                               \

  1050.     FUNC(put_hevc_epel_h)(dst, dststride, src, srcstride,                       \

  1051.                           W, height, mx, my, mcbuffer);                         \

  1052. }                                                                               \

  1053. static void FUNC(put_hevc_epel_v_ ## W)(int16_t *dst, ptrdiff_t dststride,      \

  1054.                                         uint8_t *src, ptrdiff_t srcstride,      \

  1055.                                         int height, int mx, int my,             \

  1056.                                         int16_t *mcbuffer)                      \

  1057. {                                                                               \

  1058.     FUNC(put_hevc_epel_v)(dst, dststride, src, srcstride,                       \

  1059.                           W, height, mx, my, mcbuffer);                         \

  1060. }                                                                               \

  1061. static void FUNC(put_hevc_epel_hv_ ## W)(int16_t *dst, ptrdiff_t dststride,     \

  1062.                                          uint8_t *src, ptrdiff_t srcstride,     \

  1063.                                          int height, int mx, int my,            \

  1064.                                          int16_t *mcbuffer)                     \

  1065. {                                                                               \

  1066.     FUNC(put_hevc_epel_hv)(dst, dststride, src, srcstride,                      \

  1067.                            W, height, mx, my, mcbuffer);                        \

  1068. }

  1069. 

  1070. EPEL(32)

  1071. EPEL(24)

  1072. EPEL(16)

  1073. EPEL(12)

  1074. EPEL(8)

  1075. EPEL(6)

  1076. EPEL(4)

  1077. EPEL(2)

  1078. 

  1079. static av_always_inline void

  1080. FUNC(put_unweighted_pred)(uint8_t *_dst, ptrdiff_t _dststride,

  1081.                           int16_t *src, ptrdiff_t srcstride,

  1082.                           int width, int height)

  1083. {

  1084.     int x, y;

  1085.     pixel *dst          = (pixel *)_dst;

  1086.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1087. 

  1088.     int shift = 14 - BIT_DEPTH;

  1089. #if BIT_DEPTH < 14

  1090.     int offset = 1 << (shift - 1);

  1091. #else

  1092.     int offset = 0;

  1093. #endif

  1094.     srcstride /= sizeof(*src);

  1095.     for (y = 0; y < height; y++) {

  1096.         for (x = 0; x < width; x++)

  1097.             dst[x] = av_clip_pixel((src[x] + offset) >> shift);

  1098.         dst += dststride;

  1099.         src += srcstride;

  1100.     }

  1101. }

  1102. 

  1103. static av_always_inline void

  1104. FUNC(put_unweighted_pred_avg)(uint8_t *_dst, ptrdiff_t _dststride,

  1105.                               int16_t *src1, int16_t *src2,

  1106.                               ptrdiff_t srcstride,

  1107.                               int width, int height)

  1108. {

  1109.     int x, y;

  1110.     pixel *dst          = (pixel *)_dst;

  1111.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1112. 

  1113.     int shift = 14 + 1 - BIT_DEPTH;

  1114. #if BIT_DEPTH < 14

  1115.     int offset = 1 << (shift - 1);

  1116. #else

  1117.     int offset = 0;

  1118. #endif

  1119. 

  1120.     srcstride /= sizeof(*src1);

  1121.     for (y = 0; y < height; y++) {

  1122.         for (x = 0; x < width; x++)

  1123.             dst[x] = av_clip_pixel((src1[x] + src2[x] + offset) >> shift);

  1124.         dst  += dststride;

  1125.         src1 += srcstride;

  1126.         src2 += srcstride;

  1127.     }

  1128. }

  1129. 

  1130. static av_always_inline void

  1131. FUNC(weighted_pred)(uint8_t denom, int16_t wlxFlag, int16_t olxFlag,

  1132.                     uint8_t *_dst, ptrdiff_t _dststride,

  1133.                     int16_t *src, ptrdiff_t srcstride,

  1134.                     int width, int height)

  1135. {

  1136.     int shift, log2Wd, wx, ox, x, y, offset;

  1137.     pixel *dst          = (pixel *)_dst;

  1138.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1139. 

  1140.     shift  = 14 - BIT_DEPTH;

  1141.     log2Wd = denom + shift;

  1142.     offset = 1 << (log2Wd - 1);

  1143.     wx     = wlxFlag;

  1144.     ox     = olxFlag * (1 << (BIT_DEPTH - 8));

  1145. 

  1146.     srcstride /= sizeof(*src);

  1147.     for (y = 0; y < height; y++) {

  1148.         for (x = 0; x < width; x++) {

  1149.             if (log2Wd >= 1) {

  1150.                 dst[x] = av_clip_pixel(((src[x] * wx + offset) >> log2Wd) + ox);

  1151.             } else {

  1152.                 dst[x] = av_clip_pixel(src[x] * wx + ox);

  1153.             }

  1154.         }

  1155.         dst += dststride;

  1156.         src += srcstride;

  1157.     }

  1158. }

  1159. 

  1160. static av_always_inline void

  1161. FUNC(weighted_pred_avg)(uint8_t denom,

  1162.                         int16_t wl0Flag, int16_t wl1Flag,

  1163.                         int16_t ol0Flag, int16_t ol1Flag,

  1164.                         uint8_t *_dst, ptrdiff_t _dststride,

  1165.                         int16_t *src1, int16_t *src2,

  1166.                         ptrdiff_t srcstride,

  1167.                         int width, int height)

  1168. {

  1169.     int shift, log2Wd, w0, w1, o0, o1, x, y;

  1170.     pixel *dst = (pixel *)_dst;

  1171.     ptrdiff_t dststride = _dststride / sizeof(pixel);

  1172. 

  1173.     shift  = 14 - BIT_DEPTH;

  1174.     log2Wd = denom + shift;

  1175.     w0     = wl0Flag;

  1176.     w1     = wl1Flag;

  1177.     o0     = ol0Flag * (1 << (BIT_DEPTH - 8));

  1178.     o1     = ol1Flag * (1 << (BIT_DEPTH - 8));

  1179. 

  1180.     srcstride /= sizeof(*src1);

  1181.     for (y = 0; y < height; y++) {

  1182.         for (x = 0; x < width; x++)

  1183.             dst[x] = av_clip_pixel((src1[x] * w0 + src2[x] * w1 +

  1184.                                     ((o0 + o1 + 1) << log2Wd)) >> (log2Wd + 1));

  1185.         dst  += dststride;

  1186.         src1 += srcstride;

  1187.         src2 += srcstride;

  1188.     }

  1189. }

  1190. 

  1191. #define PUT_PRED(w)                                                                            \

  1192. static void FUNC(put_unweighted_pred_ ## w)(uint8_t *dst, ptrdiff_t dststride,                 \

  1193.                                             int16_t *src, ptrdiff_t srcstride,                 \

  1194.                                             int height)                                        \

  1195. {                                                                                              \

  1196.     FUNC(put_unweighted_pred)(dst, dststride, src, srcstride, w, height);                      \

  1197. }                                                                                              \

  1198. static void FUNC(put_unweighted_pred_avg_ ## w)(uint8_t *dst, ptrdiff_t dststride,             \

  1199.                                                 int16_t *src1, int16_t *src2,                  \

  1200.                                                 ptrdiff_t srcstride, int height)               \

  1201. {                                                                                              \

  1202.     FUNC(put_unweighted_pred_avg)(dst, dststride, src1, src2, srcstride, w, height);           \

  1203. }                                                                                              \

  1204. static void FUNC(put_weighted_pred_ ## w)(uint8_t denom, int16_t weight, int16_t offset,       \

  1205.                                           uint8_t *dst, ptrdiff_t dststride,                   \

  1206.                                           int16_t *src, ptrdiff_t srcstride, int height)       \

  1207. {                                                                                              \

  1208.     FUNC(weighted_pred)(denom, weight, offset,                                                 \

  1209.                         dst, dststride, src, srcstride, w, height);                            \

  1210. }                                                                                              \

  1211. static void FUNC(put_weighted_pred_avg_ ## w)(uint8_t denom, int16_t weight0, int16_t weight1, \

  1212.                                               int16_t offset0, int16_t offset1,                \

  1213.                                               uint8_t *dst, ptrdiff_t dststride,               \

  1214.                                               int16_t *src1, int16_t *src2,                    \

  1215.                                               ptrdiff_t srcstride, int height)                 \

  1216. {                                                                                              \

  1217.     FUNC(weighted_pred_avg)(denom, weight0, weight1, offset0, offset1,                         \

  1218.                             dst, dststride, src1, src2, srcstride, w, height);                 \

  1219. }

  1220. 

  1221. PUT_PRED(64)

  1222. PUT_PRED(48)

  1223. PUT_PRED(32)

  1224. PUT_PRED(24)

  1225. PUT_PRED(16)

  1226. PUT_PRED(12)

  1227. PUT_PRED(8)

  1228. PUT_PRED(6)

  1229. PUT_PRED(4)

  1230. PUT_PRED(2)

  1231. 

  1232. // line zero

  1233. #define P3 pix[-4 * xstride]

  1234. #define P2 pix[-3 * xstride]

  1235. #define P1 pix[-2 * xstride]

  1236. #define P0 pix[-1 * xstride]

  1237. #define Q0 pix[0 * xstride]

  1238. #define Q1 pix[1 * xstride]

  1239. #define Q2 pix[2 * xstride]

  1240. #define Q3 pix[3 * xstride]

  1241. 

  1242. // line three. used only for deblocking decision

  1243. #define TP3 pix[-4 * xstride + 3 * ystride]

  1244. #define TP2 pix[-3 * xstride + 3 * ystride]

  1245. #define TP1 pix[-2 * xstride + 3 * ystride]

  1246. #define TP0 pix[-1 * xstride + 3 * ystride]

  1247. #define TQ0 pix[0  * xstride + 3 * ystride]

  1248. #define TQ1 pix[1  * xstride + 3 * ystride]

  1249. #define TQ2 pix[2  * xstride + 3 * ystride]

  1250. #define TQ3 pix[3  * xstride + 3 * ystride]

  1251. 

  1252. static void FUNC(hevc_loop_filter_luma)(uint8_t *_pix,

  1253.                                         ptrdiff_t _xstride, ptrdiff_t _ystride,

  1254.                                         int beta, int *_tc,

  1255.                                         uint8_t *_no_p, uint8_t *_no_q)

  1256. {

  1257.     int d, j;

  1258.     pixel *pix        = (pixel *)_pix;

  1259.     ptrdiff_t xstride = _xstride / sizeof(pixel);

  1260.     ptrdiff_t ystride = _ystride / sizeof(pixel);

  1261. 

  1262.     beta <<= BIT_DEPTH - 8;

  1263. 

  1264.     for (j = 0; j < 2; j++) {

  1265.         const int dp0  = abs(P2  - 2 * P1  + P0);

  1266.         const int dq0  = abs(Q2  - 2 * Q1  + Q0);

  1267.         const int dp3  = abs(TP2 - 2 * TP1 + TP0);

  1268.         const int dq3  = abs(TQ2 - 2 * TQ1 + TQ0);

  1269.         const int d0   = dp0 + dq0;

  1270.         const int d3   = dp3 + dq3;

  1271.         const int tc   = _tc[j]   << (BIT_DEPTH - 8);

  1272.         const int no_p = _no_p[j];

  1273.         const int no_q = _no_q[j];

  1274. 

  1275.         if (d0 + d3 >= beta) {

  1276.             pix += 4 * ystride;

  1277.             continue;

  1278.         } else {

  1279.             const int beta_3 = beta >> 3;

  1280.             const int beta_2 = beta >> 2;

  1281.             const int tc25   = ((tc * 5 + 1) >> 1);

  1282. 

  1283.             if (abs(P3  -  P0) + abs(Q3  -  Q0) < beta_3 && abs(P0  -  Q0) < tc25 &&

  1284.                 abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&

  1285.                                       (d0 << 1) < beta_2 &&      (d3 << 1) < beta_2) {

  1286.                 // strong filtering

  1287.                 const int tc2 = tc << 1;

  1288.                 for (d = 0; d < 4; d++) {

  1289.                     const int p3 = P3;

  1290.                     const int p2 = P2;

  1291.                     const int p1 = P1;

  1292.                     const int p0 = P0;

  1293.                     const int q0 = Q0;

  1294.                     const int q1 = Q1;

  1295.                     const int q2 = Q2;

  1296.                     const int q3 = Q3;

  1297.                     if (!no_p) {

  1298.                         P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc2, tc2);

  1299.                         P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2);

  1300.                         P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc2, tc2);

  1301.                     }

  1302.                     if (!no_q) {

  1303.                         Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc2, tc2);

  1304.                         Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2);

  1305.                         Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc2, tc2);

  1306.                     }

  1307.                     pix += ystride;

  1308.                 }

  1309.             } else { // normal filtering

  1310.                 int nd_p = 1;

  1311.                 int nd_q = 1;

  1312.                 const int tc_2 = tc >> 1;

  1313.                 if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))

  1314.                     nd_p = 2;

  1315.                 if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))

  1316.                     nd_q = 2;

  1317. 

  1318.                 for (d = 0; d < 4; d++) {

  1319.                     const int p2 = P2;

  1320.                     const int p1 = P1;

  1321.                     const int p0 = P0;

  1322.                     const int q0 = Q0;

  1323.                     const int q1 = Q1;

  1324.                     const int q2 = Q2;

  1325.                     int delta0   = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4;

  1326.                     if (abs(delta0) < 10 * tc) {

  1327.                         delta0 = av_clip(delta0, -tc, tc);

  1328.                         if (!no_p)

  1329.                             P0 = av_clip_pixel(p0 + delta0);

  1330.                         if (!no_q)

  1331.                             Q0 = av_clip_pixel(q0 - delta0);

  1332.                         if (!no_p && nd_p > 1) {

  1333.                             const int deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2);

  1334.                             P1 = av_clip_pixel(p1 + deltap1);

  1335.                         }

  1336.                         if (!no_q && nd_q > 1) {

  1337.                             const int deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2);

  1338.                             Q1 = av_clip_pixel(q1 + deltaq1);

  1339.                         }

  1340.                     }

  1341.                     pix += ystride;

  1342.                 }

  1343.             }

  1344.         }

  1345.     }

  1346. }

  1347. 

  1348. static void FUNC(hevc_loop_filter_chroma)(uint8_t *_pix, ptrdiff_t _xstride,

  1349.                                           ptrdiff_t _ystride, int *_tc,

  1350.                                           uint8_t *_no_p, uint8_t *_no_q)

  1351. {

  1352.     int d, j, no_p, no_q;

  1353.     pixel *pix        = (pixel *)_pix;

  1354.     ptrdiff_t xstride = _xstride / sizeof(pixel);

  1355.     ptrdiff_t ystride = _ystride / sizeof(pixel);

  1356. 

  1357.     for (j = 0; j < 2; j++) {

  1358.         const int tc = _tc[j] << (BIT_DEPTH - 8);

  1359.         if (tc <= 0) {

  1360.             pix += 4 * ystride;

  1361.             continue;

  1362.         }

  1363.         no_p = _no_p[j];

  1364.         no_q = _no_q[j];

  1365. 

  1366.         for (d = 0; d < 4; d++) {

  1367.             int delta0;

  1368.             const int p1 = P1;

  1369.             const int p0 = P0;

  1370.             const int q0 = Q0;

  1371.             const int q1 = Q1;

  1372.             delta0 = av_clip((((q0 - p0) * 4) + p1 - q1 + 4) >> 3, -tc, tc);

  1373.             if (!no_p)

  1374.                 P0 = av_clip_pixel(p0 + delta0);

  1375.             if (!no_q)

  1376.                 Q0 = av_clip_pixel(q0 - delta0);

  1377.             pix += ystride;

  1378.         }

  1379.     }

  1380. }

  1381. 

  1382. static void FUNC(hevc_h_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,

  1383.                                             int *tc, uint8_t *no_p,

  1384.                                             uint8_t *no_q)

  1385. {

  1386.     FUNC(hevc_loop_filter_chroma)(pix, stride, sizeof(pixel), tc, no_p, no_q);

  1387. }

  1388. 

  1389. static void FUNC(hevc_v_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,

  1390.                                             int *tc, uint8_t *no_p,

  1391.                                             uint8_t *no_q)

  1392. {

  1393.     FUNC(hevc_loop_filter_chroma)(pix, sizeof(pixel), stride, tc, no_p, no_q);

  1394. }

  1395. 

  1396. static void FUNC(hevc_h_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,

  1397.                                           int beta, int *tc, uint8_t *no_p,

  1398.                                           uint8_t *no_q)

  1399. {

  1400.     FUNC(hevc_loop_filter_luma)(pix, stride, sizeof(pixel),

  1401.                                 beta, tc, no_p, no_q);

  1402. }

  1403. 

  1404. static void FUNC(hevc_v_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,

  1405.                                           int beta, int *tc, uint8_t *no_p,

  1406.                                           uint8_t *no_q)

  1407. {

  1408.     FUNC(hevc_loop_filter_luma)(pix, sizeof(pixel), stride,

  1409.                                 beta, tc, no_p, no_q);

  1410. }

  1411. 

  1412. #undef P3

  1413. #undef P2

  1414. #undef P1

  1415. #undef P0

  1416. #undef Q0

  1417. #undef Q1

  1418. #undef Q2

  1419. #undef Q3

  1420. 

  1421. #undef TP3

  1422. #undef TP2

  1423. #undef TP1

  1424. #undef TP0

  1425. #undef TQ0

  1426. #undef TQ1

  1427. #undef TQ2

  1428. #undef TQ3