falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
64986 Commits
32 Branches
748MB
Tree: 036f11bdb5
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '036f11bdb5'
${ noResults }
FFmpeg/libavcodec/x86/hevc_mc.asm

1342 lines
43KB
Raw Blame History 

  1. ; /*

  2. ; * Provide SSE luma and chroma mc functions for HEVC decoding

  3. ; * Copyright (c) 2013 Pierre-Edouard LEPERE

  4. ; *

  5. ; * This file is part of FFmpeg.

  6. ; *

  7. ; * FFmpeg is free software; you can redistribute it and/or

  8. ; * modify it under the terms of the GNU Lesser General Public

  9. ; * License as published by the Free Software Foundation; either

  10. ; * version 2.1 of the License, or (at your option) any later version.

  11. ; *

  12. ; * FFmpeg is distributed in the hope that it will be useful,

  13. ; * but WITHOUT ANY WARRANTY; without even the implied warranty of

  14. ; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  15. ; * Lesser General Public License for more details.

  16. ; *

  17. ; * You should have received a copy of the GNU Lesser General Public

  18. ; * License along with FFmpeg; if not, write to the Free Software

  19. ; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  20. ; */

  21. %include "libavutil/x86/x86util.asm"

  22. 

  23. SECTION_RODATA

  24. pw_8:                   times 8 dw (1 << 9)

  25. pw_10:                  times 8 dw (1 << 11)

  26. pw_12:                  times 8 dw (1 << 13)

  27. pw_bi_8:                times 8 dw (1 << 8)

  28. pw_bi_10:               times 8 dw (1 << 10)

  29. pw_bi_12:               times 8 dw (1 << 12)

  30. max_pixels_10:          times 8  dw ((1 << 10)-1)

  31. max_pixels_12:          times 8  dw ((1 << 12)-1)

  32. zero:                   times 4  dd 0

  33. one_per_32:             times 4  dd 1

  34. 

  35. SECTION .text

  36. %macro EPEL_TABLE 4

  37. hevc_epel_filters_%4_%1 times %2 d%3 -2, 58

  38.                         times %2 d%3 10, -2

  39.                         times %2 d%3 -4, 54

  40.                         times %2 d%3 16, -2

  41.                         times %2 d%3 -6, 46

  42.                         times %2 d%3 28, -4

  43.                         times %2 d%3 -4, 36

  44.                         times %2 d%3 36, -4

  45.                         times %2 d%3 -4, 28

  46.                         times %2 d%3 46, -6

  47.                         times %2 d%3 -2, 16

  48.                         times %2 d%3 54, -4

  49.                         times %2 d%3 -2, 10

  50.                         times %2 d%3 58, -2

  51. %endmacro

  52. 

  53. 

  54. 

  55. EPEL_TABLE  8, 8, b, sse4

  56. EPEL_TABLE 10, 4, w, sse4

  57. EPEL_TABLE 12, 4, w, sse4

  58. 

  59. %macro QPEL_TABLE 4

  60. hevc_qpel_filters_%4_%1 times %2 d%3  -1,  4

  61.                         times %2 d%3 -10, 58

  62.                         times %2 d%3  17, -5

  63.                         times %2 d%3   1,  0

  64.                         times %2 d%3  -1,  4

  65.                         times %2 d%3 -11, 40

  66.                         times %2 d%3  40,-11

  67.                         times %2 d%3   4, -1

  68.                         times %2 d%3   0,  1

  69.                         times %2 d%3  -5, 17

  70.                         times %2 d%3  58,-10

  71.                         times %2 d%3   4, -1

  72. %endmacro

  73. 

  74. QPEL_TABLE  8, 8, b, sse4

  75. QPEL_TABLE 10, 4, w, sse4

  76. QPEL_TABLE 12, 4, w, sse4

  77. 

  78. %define hevc_qpel_filters_sse4_14 hevc_qpel_filters_sse4_10

  79. 

  80. %if ARCH_X86_64

  81. 

  82. %macro SIMPLE_BILOAD 4   ;width, tab, r1, r2

  83. %if %1 <= 4

  84.     movq              %3, [%2]                                              ; load data from source2

  85. %elif %1 <= 8

  86.     movdqa            %3, [%2]                                              ; load data from source2

  87. %elif %1 <= 12

  88.     movdqa            %3, [%2]                                              ; load data from source2

  89.     movq              %4, [%2+16]                                           ; load data from source2

  90. %else

  91.     movdqa            %3, [%2]                                              ; load data from source2

  92.     movdqa            %4, [%2+16]                                           ; load data from source2

  93. %endif

  94. %endmacro

  95. 

  96. %macro SIMPLE_LOAD 4    ;width, bitd, tab, r1

  97. %if %1 == 2 || (%2 == 8 && %1 <= 4)

  98.     movd              %4, [%3]                                               ; load data from source

  99. %elif %1 == 4 || (%2 == 8 && %1 <= 8)

  100.     movq              %4, [%3]                                               ; load data from source

  101. %else

  102.     movdqu            %4, [%3]                                               ; load data from source

  103. %endif

  104. %endmacro

  105. 

  106. %macro SIMPLE_8LOAD 5    ;width, bitd, tab, r1, r2

  107. %if %1 == 2 || (%2 == 8 && %1 <= 4)

  108.     movq              %4, [%3]                                              ; load data from source2

  109. %elif %1 == 4 || (%2 == 8 && %1 <= 8)

  110.     movdqa            %4, [%3]                                              ; load data from source2

  111. %elif %1 <= 12

  112.     movdqa            %4, [%3]                                              ; load data from source2

  113.     movq              %5, [%3+16]                                           ; load data from source2

  114. %else

  115.     movdqa            %4, [%3]                                              ; load data from source2

  116.     movdqa            %5, [%3+16]                                           ; load data from source2

  117. %endif

  118. %endmacro

  119. 

  120. %macro EPEL_FILTER 2-4                            ; bit depth, filter index

  121. %ifdef PIC

  122.     lea         rfilterq, [hevc_epel_filters_sse4_%1]

  123. %else

  124.     %define rfilterq hevc_epel_filters_sse4_%1

  125. %endif

  126.     sub              %2q, 1

  127.     shl              %2q, 5                      ; multiply by 32

  128. %if %0 == 2

  129.     movdqa           m14, [rfilterq + %2q]        ; get 2 first values of filters

  130.     movdqa           m15, [rfilterq + %2q+16]     ; get 2 last values of filters

  131. %else

  132.     movdqa           %3, [rfilterq + %2q]        ; get 2 first values of filters

  133.     movdqa           %4, [rfilterq + %2q+16]     ; get 2 last values of filters

  134. %endif

  135. %endmacro

  136. 

  137. %macro EPEL_HV_FILTER 1

  138. %ifdef PIC

  139.     lea         rfilterq, [hevc_epel_filters_sse4_%1]

  140. %else

  141.     %define rfilterq hevc_epel_filters_sse4_%1

  142. %endif

  143.     sub              mxq, 1

  144.     sub              myq, 1

  145.     shl              mxq, 5                      ; multiply by 32

  146.     shl              myq, 5                      ; multiply by 32

  147.     movdqa           m14, [rfilterq + mxq]        ; get 2 first values of filters

  148.     movdqa           m15, [rfilterq + mxq+16]     ; get 2 last values of filters

  149.     lea           r3srcq, [srcstrideq*3]

  150. 

  151. %ifdef PIC

  152.     lea         rfilterq, [hevc_epel_filters_sse4_10]

  153. %else

  154.     %define rfilterq hevc_epel_filters_sse4_10

  155. %endif

  156.     movdqa           m12, [rfilterq + myq]        ; get 2 first values of filters

  157.     movdqa           m13, [rfilterq + myq+16]     ; get 2 last values of filters

  158. %endmacro

  159. 

  160. %macro QPEL_FILTER 2

  161. %ifdef PIC

  162.     lea         rfilterq, [hevc_qpel_filters_sse4_%1]

  163. %else

  164.     %define rfilterq hevc_qpel_filters_sse4_%1

  165. %endif

  166.     lea              %2q, [%2q*8-8]

  167.     movdqa           m12, [rfilterq + %2q*8]       ; get 4 first values of filters

  168.     movdqa           m13, [rfilterq + %2q*8 + 16]  ; get 4 first values of filters

  169.     movdqa           m14, [rfilterq + %2q*8 + 32]  ; get 4 first values of filters

  170.     movdqa           m15, [rfilterq + %2q*8 + 48]  ; get 4 first values of filters

  171. %endmacro

  172. 

  173. %macro EPEL_LOAD 4

  174. %ifdef PIC

  175.     lea rfilterq, [%2]

  176. %else

  177.     %define rfilterq %2

  178. %endif

  179.     movdqu            m0, [rfilterq ]            ;load 128bit of x

  180. %ifnum %3

  181.     movdqu            m1, [rfilterq+  %3]        ;load 128bit of x+stride

  182.     movdqu            m2, [rfilterq+2*%3]        ;load 128bit of x+2*stride

  183.     movdqu            m3, [rfilterq+3*%3]        ;load 128bit of x+3*stride

  184. %else

  185.     movdqu            m1, [rfilterq+  %3q]       ;load 128bit of x+stride

  186.     movdqu            m2, [rfilterq+2*%3q]       ;load 128bit of x+2*stride

  187.     movdqu            m3, [rfilterq+r3srcq]      ;load 128bit of x+2*stride

  188. %endif

  189. 

  190. %if %1 == 8

  191. %if %4 > 8

  192.     SBUTTERFLY        bw, 0, 1, 10

  193.     SBUTTERFLY        bw, 2, 3, 10

  194. %else

  195.     punpcklbw         m0, m1

  196.     punpcklbw         m2, m3

  197. %endif

  198. %else

  199. %if %4 > 4

  200.     SBUTTERFLY        wd, 0, 1, 10

  201.     SBUTTERFLY        wd, 2, 3, 10

  202. %else

  203.     punpcklwd         m0, m1

  204.     punpcklwd         m2, m3

  205. %endif

  206. %endif

  207. %endmacro

  208. 

  209. 

  210. %macro QPEL_H_LOAD 4

  211. %assign %%stride (%1+7)/8

  212. %if %1 == 8

  213. %if %3 <= 4

  214. %define %%load movd

  215. %elif %3 == 8

  216. %define %%load movq

  217. %else

  218. %define %%load movdqu

  219. %endif

  220. %else

  221. %if %3 == 2

  222. %define %%load movd

  223. %elif %3 == 4

  224. %define %%load movq

  225. %else

  226. %define %%load movdqu

  227. %endif

  228. %endif

  229.     %%load            m0, [%2-3*%%stride]        ;load data from source

  230.     %%load            m1, [%2-2*%%stride]

  231.     %%load            m2, [%2-%%stride  ]

  232.     %%load            m3, [%2           ]

  233.     %%load            m4, [%2+%%stride  ]

  234.     %%load            m5, [%2+2*%%stride]

  235.     %%load            m6, [%2+3*%%stride]

  236.     %%load            m7, [%2+4*%%stride]

  237. 

  238. %if %1 == 8

  239. %if %3 > 8

  240.     SBUTTERFLY        wd, 0, 1, %4

  241.     SBUTTERFLY        wd, 2, 3, %4

  242.     SBUTTERFLY        wd, 4, 5, %4

  243.     SBUTTERFLY        wd, 6, 7, %4

  244. %else

  245.     punpcklwd         m0, m1

  246.     punpcklwd         m2, m3

  247.     punpcklwd         m4, m5

  248.     punpcklwd         m6, m7

  249. %endif

  250. %else

  251. %if %3 > 4

  252.     SBUTTERFLY        dq, 0, 1, %4

  253.     SBUTTERFLY        dq, 2, 3, %4

  254.     SBUTTERFLY        dq, 4, 5, %4

  255.     SBUTTERFLY        dq, 6, 7, %4

  256. %else

  257.     punpckldq         m0, m1

  258.     punpckldq         m2, m3

  259.     punpckldq         m4, m5

  260.     punpckldq         m6, m7

  261. %endif

  262. %endif

  263. %endmacro

  264. 

  265. %macro QPEL_V_LOAD 4

  266.     lea             r12q, [%2]

  267.     sub             r12q, r3srcq

  268.     movdqu            m0, [r12            ]      ;load x- 3*srcstride

  269.     movdqu            m1, [r12+   %3q     ]      ;load x- 2*srcstride

  270.     movdqu            m2, [r12+ 2*%3q     ]      ;load x-srcstride

  271.     movdqu            m3, [%2       ]      ;load x

  272.     movdqu            m4, [%2+   %3q]      ;load x+stride

  273.     movdqu            m5, [%2+ 2*%3q]      ;load x+2*stride

  274.     movdqu            m6, [%2+r3srcq]      ;load x+3*stride

  275.     movdqu            m7, [%2+ 4*%3q]      ;load x+4*stride

  276. %if %1 == 8

  277. %if %4 > 8

  278.     SBUTTERFLY        bw, 0, 1, 8

  279.     SBUTTERFLY        bw, 2, 3, 8

  280.     SBUTTERFLY        bw, 4, 5, 8

  281.     SBUTTERFLY        bw, 6, 7, 8

  282. %else

  283.     punpcklbw         m0, m1

  284.     punpcklbw         m2, m3

  285.     punpcklbw         m4, m5

  286.     punpcklbw         m6, m7

  287. %endif

  288. %else

  289. %if %4 > 4

  290.     SBUTTERFLY        wd, 0, 1, 8

  291.     SBUTTERFLY        wd, 2, 3, 8

  292.     SBUTTERFLY        wd, 4, 5, 8

  293.     SBUTTERFLY        wd, 6, 7, 8

  294. %else

  295.     punpcklwd         m0, m1

  296.     punpcklwd         m2, m3

  297.     punpcklwd         m4, m5

  298.     punpcklwd         m6, m7

  299. %endif

  300. %endif

  301. %endmacro

  302. 

  303. %macro PEL_12STORE2 3

  304.     movd           [%1], %2

  305. %endmacro

  306. %macro PEL_12STORE4 3

  307.     movq           [%1], %2

  308. %endmacro

  309. %macro PEL_12STORE6 3

  310.     movq           [%1], %2

  311.     psrldq            %2, 8

  312.     movd         [%1+8], %2

  313. %endmacro

  314. %macro PEL_12STORE8 3

  315.     movdqa         [%1], %2

  316. %endmacro

  317. %macro PEL_12STORE12 3

  318.     movdqa         [%1], %2

  319.     movq        [%1+16], %3

  320. %endmacro

  321. %macro PEL_12STORE16 3

  322.     PEL_12STORE8      %1, %2, %3

  323.     movdqa       [%1+16], %3

  324. %endmacro

  325. 

  326. %macro PEL_10STORE2 3

  327.     movd           [%1], %2

  328. %endmacro

  329. %macro PEL_10STORE4 3

  330.     movq           [%1], %2

  331. %endmacro

  332. %macro PEL_10STORE6 3

  333.     movq           [%1], %2

  334.     psrldq            %2, 8

  335.     movd         [%1+8], %2

  336. %endmacro

  337. %macro PEL_10STORE8 3

  338.     movdqa         [%1], %2

  339. %endmacro

  340. %macro PEL_10STORE12 3

  341.     movdqa         [%1], %2

  342.     movq        [%1+16], %3

  343. %endmacro

  344. %macro PEL_10STORE16 3

  345.     PEL_10STORE8      %1, %2, %3

  346.     movdqa       [%1+16], %3

  347. %endmacro

  348. 

  349. %macro PEL_8STORE2 3

  350.     pextrw          [%1], %2, 0

  351. %endmacro

  352. %macro PEL_8STORE4 3

  353.     movd            [%1], %2

  354. %endmacro

  355. %macro PEL_8STORE6 3

  356.     movd            [%1], %2

  357.     pextrw        [%1+4], %2, 2

  358. %endmacro

  359. %macro PEL_8STORE8 3

  360.     movq           [%1], %2

  361. %endmacro

  362. %macro PEL_8STORE12 3

  363.     movq            [%1], %2

  364.     psrldq            %2, 8

  365.     movd          [%1+8], %2

  366. %endmacro

  367. %macro PEL_8STORE16 3

  368.     movdqa          [%1], %2

  369. %endmacro

  370. 

  371. %macro LOOP_END 4

  372.     lea              %1q, [%1q+2*%2q]            ; dst += dststride

  373.     lea              %3q, [%3q+  %4q]            ; src += srcstride

  374.     dec          heightd                         ; cmp height

  375.     jnz               .loop                      ; height loop

  376. %endmacro

  377. 

  378. 

  379. %macro MC_PIXEL_COMPUTE 2 ;width, bitdepth

  380. %if %2 == 8

  381. %if %1 > 8

  382.     punpckhbw         m1, m0, m2

  383.     psllw             m1, 14-%2

  384. %endif

  385.     punpcklbw         m0, m2

  386. %endif

  387.     psllw             m0, 14-%2

  388. %endmacro

  389. 

  390. 

  391. %macro EPEL_COMPUTE 4 ; bitdepth, width, filter1, filter2

  392. %if %1 == 8

  393.     pmaddubsw         m0, %3   ;x1*c1+x2*c2

  394.     pmaddubsw         m2, %4   ;x3*c3+x4*c4

  395.     paddw             m0, m2

  396. %if %2 > 8

  397.     pmaddubsw         m1, %3

  398.     pmaddubsw         m3, %4

  399.     paddw             m1, m3

  400. %endif

  401. %else

  402.     pmaddwd           m0, %3

  403.     pmaddwd           m2, %4

  404.     paddd             m0, m2

  405. %if %2 > 4

  406.     pmaddwd           m1, %3

  407.     pmaddwd           m3, %4

  408.     paddd             m1, m3

  409. %endif

  410. %if %1 != 8

  411.     psrad             m0, %1-8

  412.     psrad             m1, %1-8

  413. %endif

  414.     packssdw          m0, m1

  415. %endif

  416. %endmacro

  417. 

  418. %macro QPEL_HV_COMPUTE 4     ; width, bitdepth, filter idx

  419. %ifdef PIC

  420.     lea         rfilterq, [hevc_qpel_filters_sse4_%2]

  421. %else

  422.     %define rfilterq hevc_qpel_filters_sse4_%2

  423. %endif

  424. 

  425. %if %2 == 8

  426.     pmaddubsw         m0, [rfilterq + %3q*8   ]   ;x1*c1+x2*c2

  427.     pmaddubsw         m2, [rfilterq + %3q*8+16]   ;x3*c3+x4*c4

  428.     pmaddubsw         m4, [rfilterq + %3q*8+32]   ;x5*c5+x6*c6

  429.     pmaddubsw         m6, [rfilterq + %3q*8+48]   ;x7*c7+x8*c8

  430.     paddw             m0, m2

  431.     paddw             m4, m6

  432.     paddw             m0, m4

  433. %else

  434.     pmaddwd           m0, [rfilterq + %3q*8   ]

  435.     pmaddwd           m2, [rfilterq + %3q*8+16]

  436.     pmaddwd           m4, [rfilterq + %3q*8+32]

  437.     pmaddwd           m6, [rfilterq + %3q*8+48]

  438.     paddd             m0, m2

  439.     paddd             m4, m6

  440.     paddd             m0, m4

  441. %if %2 != 8

  442.     psrad             m0, %2-8

  443. %endif

  444. %if %1 > 4

  445.     pmaddwd           m1, [rfilterq + %3q*8   ]

  446.     pmaddwd           m3, [rfilterq + %3q*8+16]

  447.     pmaddwd           m5, [rfilterq + %3q*8+32]

  448.     pmaddwd           m7, [rfilterq + %3q*8+48]

  449.     paddd             m1, m3

  450.     paddd             m5, m7

  451.     paddd             m1, m5

  452. %if %2 != 8

  453.     psrad             m1, %2-8

  454. %endif

  455. %endif

  456.     p%4               m0, m1

  457. %endif

  458. %endmacro

  459. 

  460. %macro QPEL_COMPUTE 2     ; width, bitdepth

  461. %if %2 == 8

  462.     pmaddubsw         m0, m12   ;x1*c1+x2*c2

  463.     pmaddubsw         m2, m13   ;x3*c3+x4*c4

  464.     pmaddubsw         m4, m14   ;x5*c5+x6*c6

  465.     pmaddubsw         m6, m15   ;x7*c7+x8*c8

  466.     paddw             m0, m2

  467.     paddw             m4, m6

  468.     paddw             m0, m4

  469. %if %1 > 8

  470.     pmaddubsw         m1, m12

  471.     pmaddubsw         m3, m13

  472.     pmaddubsw         m5, m14

  473.     pmaddubsw         m7, m15

  474.     paddw             m1, m3

  475.     paddw             m5, m7

  476.     paddw             m1, m5

  477. %endif

  478. %else

  479.     pmaddwd           m0, m12

  480.     pmaddwd           m2, m13

  481.     pmaddwd           m4, m14

  482.     pmaddwd           m6, m15

  483.     paddd             m0, m2

  484.     paddd             m4, m6

  485.     paddd             m0, m4

  486. %if %2 != 8

  487.     psrad             m0, %2-8

  488. %endif

  489. %if %1 > 4

  490.     pmaddwd           m1, m12

  491.     pmaddwd           m3, m13

  492.     pmaddwd           m5, m14

  493.     pmaddwd           m7, m15

  494.     paddd             m1, m3

  495.     paddd             m5, m7

  496.     paddd             m1, m5

  497. %if %2 != 8

  498.     psrad             m1, %2-8

  499. %endif

  500. %endif

  501. %endif

  502. %endmacro

  503. 

  504. %macro BI_COMPUTE 7     ; width, bitd, src1l, src1h, scr2l, scr2h, pw

  505.     paddsw            %3, %5

  506. %if %1 > 8

  507.     paddsw            %4, %6

  508. %endif

  509.     UNI_COMPUTE       %1, %2, %3, %4, %7

  510. %endmacro

  511. 

  512. %macro UNI_COMPUTE 5

  513.     pmulhrsw          %3, %5

  514. %if %1 > 8 || (%2 > 8 && %1 > 4)

  515.     pmulhrsw          %4, %5

  516. %endif

  517. %if %2 == 8

  518.     packuswb          %3, %4

  519. %else

  520.     pminsw            %3, [max_pixels_%2]

  521.     pmaxsw            %3, [zero]

  522. %if %1 > 8

  523.     pminsw            %4, [max_pixels_%2]

  524.     pmaxsw            %4, [zero]

  525. %endif

  526. %endif

  527. %endmacro

  528. 

  529. INIT_XMM sse4                                    ; adds ff_ and _sse4 to function name

  530. ; ******************************

  531. ; void put_hevc_mc_pixels(int16_t *dst, ptrdiff_t dststride,

  532. ;                         uint8_t *_src, ptrdiff_t _srcstride,

  533. ;                         int height, int mx, int my)

  534. ; ******************************

  535. 

  536. %macro HEVC_PUT_HEVC_PEL_PIXELS 2

  537. cglobal hevc_put_hevc_pel_pixels%1_%2, 5, 5, 3, dst, dststride, src, srcstride,height

  538.     pxor               m2, m2

  539. .loop

  540.     SIMPLE_LOAD       %1, %2, srcq, m0

  541.     MC_PIXEL_COMPUTE  %1, %2

  542.     PEL_10STORE%1     dstq, m0, m1

  543.     LOOP_END         dst, dststride, src, srcstride

  544.     RET

  545. 

  546. cglobal hevc_put_hevc_uni_pel_pixels%1_%2, 5, 5, 3, dst, dststride, src, srcstride,height

  547.     pxor              m2, m2

  548. .loop

  549.     SIMPLE_LOAD       %1, %2, srcq, m0

  550.     PEL_%2STORE%1   dstq, m0, m1

  551.     add             dstq, dststrideq             ; dst += dststride

  552.     add             srcq, srcstrideq             ; src += srcstride

  553.     dec          heightd                         ; cmp height

  554.     jnz               .loop                      ; height loop

  555.     RET

  556. 

  557. cglobal hevc_put_hevc_bi_pel_pixels%1_%2, 7, 7, 6, dst, dststride, src, srcstride, src2, src2stride,height

  558.     pxor              m2, m2

  559.     movdqa            m5, [pw_bi_%2]

  560. .loop

  561.     SIMPLE_LOAD       %1, %2, srcq, m0

  562.     SIMPLE_BILOAD     %1, src2q, m3, m4

  563.     MC_PIXEL_COMPUTE  %1, %2

  564.     BI_COMPUTE        %1, %2, m0, m1, m3, m4, m5

  565.     PEL_%2STORE%1   dstq, m0, m1

  566.     add             dstq, dststrideq             ; dst += dststride

  567.     add             srcq, srcstrideq             ; src += srcstride

  568.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  569.     dec          heightd                         ; cmp height

  570.     jnz               .loop                      ; height loop

  571.     RET

  572. 

  573. %endmacro

  574. 

  575. 

  576. ; ******************************

  577. ; void put_hevc_epel_hX(int16_t *dst, ptrdiff_t dststride,

  578. ;                       uint8_t *_src, ptrdiff_t _srcstride,

  579. ;                       int width, int height, int mx, int my,

  580. ;                       int16_t* mcbuffer)

  581. ; ******************************

  582. 

  583. 

  584. %macro HEVC_PUT_HEVC_EPEL 2

  585. cglobal hevc_put_hevc_epel_h%1_%2, 6, 7, 6, dst, dststride, src, srcstride, height, mx, rfilter

  586. %assign %%stride ((%2 + 7)/8)

  587.     EPEL_FILTER       %2, mx, m4, m5

  588. .loop

  589.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  590.     EPEL_COMPUTE      %2, %1, m4, m5

  591.     PEL_10STORE%1      dstq, m0, m1

  592.     LOOP_END         dst, dststride, src, srcstride

  593.     RET

  594. 

  595. cglobal hevc_put_hevc_uni_epel_h%1_%2, 6, 7, 7, dst, dststride, src, srcstride, height, mx, rfilter

  596. %assign %%stride ((%2 + 7)/8)

  597.     movdqa            m6, [pw_%2]

  598.     EPEL_FILTER       %2, mx, m4, m5

  599. .loop

  600.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  601.     EPEL_COMPUTE      %2, %1, m4, m5

  602.     UNI_COMPUTE       %1, %2, m0, m1, m6

  603.     PEL_%2STORE%1   dstq, m0, m1

  604.     add             dstq, dststrideq             ; dst += dststride

  605.     add             srcq, srcstrideq             ; src += srcstride

  606.     dec          heightd                         ; cmp height

  607.     jnz               .loop                      ; height loop

  608.     RET

  609. 

  610. cglobal hevc_put_hevc_bi_epel_h%1_%2, 8, 9, 7, dst, dststride, src, srcstride, src2, src2stride,height, mx, rfilter

  611.     movdqa            m6, [pw_bi_%2]

  612.     EPEL_FILTER       %2, mx, m4, m5

  613. .loop

  614.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  615.     EPEL_COMPUTE      %2, %1, m4, m5

  616.     SIMPLE_BILOAD     %1, src2q, m2, m3

  617.     BI_COMPUTE        %1, %2, m0, m1, m2, m3, m6

  618.     PEL_%2STORE%1   dstq, m0, m1

  619.     add             dstq, dststrideq             ; dst += dststride

  620.     add             srcq, srcstrideq             ; src += srcstride

  621.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  622.     dec          heightd                         ; cmp height

  623.     jnz               .loop                      ; height loop

  624.     RET

  625. 

  626. ; ******************************

  627. ; void put_hevc_epel_v(int16_t *dst, ptrdiff_t dststride,

  628. ;                      uint8_t *_src, ptrdiff_t _srcstride,

  629. ;                      int width, int height, int mx, int my,

  630. ;                      int16_t* mcbuffer)

  631. ; ******************************

  632. 

  633. cglobal hevc_put_hevc_epel_v%1_%2, 7, 8, 6, dst, dststride, src, srcstride, height, r3src, my, rfilter

  634.     lea           r3srcq, [srcstrideq*3]

  635.     sub             srcq, srcstrideq

  636.     EPEL_FILTER       %2, my, m4, m5

  637. .loop

  638.     EPEL_LOAD         %2, srcq, srcstride, %1

  639.     EPEL_COMPUTE      %2, %1, m4, m5

  640.     PEL_10STORE%1     dstq, m0, m1

  641.     LOOP_END          dst, dststride, src, srcstride

  642.     RET

  643. 

  644. cglobal hevc_put_hevc_uni_epel_v%1_%2, 7, 8, 7, dst, dststride, src, srcstride, height, r3src, my, rfilter

  645.     lea           r3srcq, [srcstrideq*3]

  646.     movdqa            m6, [pw_%2]

  647.     sub             srcq, srcstrideq

  648.     EPEL_FILTER       %2, my, m4, m5

  649. .loop

  650.     EPEL_LOAD         %2, srcq, srcstride, %1

  651.     EPEL_COMPUTE      %2, %1, m4, m5

  652.     UNI_COMPUTE       %1, %2, m0, m1, m6

  653.     PEL_%2STORE%1   dstq, m0, m1

  654.     add             dstq, dststrideq             ; dst += dststride

  655.     add             srcq, srcstrideq             ; src += srcstride

  656.     dec          heightd                         ; cmp height

  657.     jnz               .loop                      ; height loop

  658.     RET

  659. 

  660. 

  661. cglobal hevc_put_hevc_bi_epel_v%1_%2, 9, 10, 7, dst, dststride, src, srcstride, src2, src2stride,height, r3src, my, rfilter

  662.     lea           r3srcq, [srcstrideq*3]

  663.     movdqa            m6, [pw_bi_%2]

  664.     sub             srcq, srcstrideq

  665.     EPEL_FILTER       %2, my, m4, m5

  666. .loop

  667.     EPEL_LOAD         %2, srcq, srcstride, %1

  668.     EPEL_COMPUTE      %2, %1, m4, m5

  669.     SIMPLE_BILOAD     %1, src2q, m2, m3

  670.     BI_COMPUTE        %1, %2, m0, m1, m2, m3, m6

  671.     PEL_%2STORE%1   dstq, m0, m1

  672.     add             dstq, dststrideq             ; dst += dststride

  673.     add             srcq, srcstrideq             ; src += srcstride

  674.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  675.     dec          heightd                         ; cmp height

  676.     jnz               .loop                      ; height loop

  677.     RET

  678. %endmacro

  679. 

  680. 

  681. ; ******************************

  682. ; void put_hevc_epel_hv(int16_t *dst, ptrdiff_t dststride,

  683. ;                       uint8_t *_src, ptrdiff_t _srcstride,

  684. ;                       int width, int height, int mx, int my)

  685. ; ******************************

  686. 

  687. %macro HEVC_PUT_HEVC_EPEL_HV 2

  688. cglobal hevc_put_hevc_epel_hv%1_%2, 7, 9, 12 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter

  689. %assign %%stride ((%2 + 7)/8)

  690.     sub             srcq, srcstrideq

  691.     EPEL_HV_FILTER    %2

  692.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  693.     EPEL_COMPUTE      %2, %1, m14, m15

  694.     SWAP              m4, m0

  695.     add             srcq, srcstrideq

  696.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  697.     EPEL_COMPUTE      %2, %1, m14, m15

  698.     SWAP              m5, m0

  699.     add             srcq, srcstrideq

  700.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  701.     EPEL_COMPUTE      %2, %1, m14, m15

  702.     SWAP              m6, m0

  703.     add             srcq, srcstrideq

  704. .loop

  705.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  706.     EPEL_COMPUTE      %2, %1, m14, m15

  707.     SWAP              m7, m0

  708.     punpcklwd         m0, m4, m5

  709.     punpcklwd         m2, m6, m7

  710. %if %1 > 4

  711.     punpckhwd         m1, m4, m5

  712.     punpckhwd         m3, m6, m7

  713. %endif

  714.     EPEL_COMPUTE      14, %1, m12, m13

  715.     PEL_10STORE%1     dstq, m0, m1

  716.     movdqa            m4, m5

  717.     movdqa            m5, m6

  718.     movdqa            m6, m7

  719.     LOOP_END         dst, dststride, src, srcstride

  720.     RET

  721. 

  722. cglobal hevc_put_hevc_uni_epel_hv%1_%2, 7, 9, 12 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter

  723. %assign %%stride ((%2 + 7)/8)

  724.     sub             srcq, srcstrideq

  725.     EPEL_HV_FILTER    %2

  726.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  727.     EPEL_COMPUTE      %2, %1, m14, m15

  728.     SWAP              m4, m0

  729.     add             srcq, srcstrideq

  730.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  731.     EPEL_COMPUTE      %2, %1, m14, m15

  732.     SWAP              m5, m0

  733.     add             srcq, srcstrideq

  734.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  735.     EPEL_COMPUTE      %2, %1, m14, m15

  736.     SWAP              m6, m0

  737.     add             srcq, srcstrideq

  738. .loop

  739.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  740.     EPEL_COMPUTE      %2, %1, m14, m15

  741.     SWAP              m7, m0

  742.     punpcklwd         m0, m4, m5

  743.     punpcklwd         m2, m6, m7

  744. %if %1 > 4

  745.     punpckhwd         m1, m4, m5

  746.     punpckhwd         m3, m6, m7

  747. %endif

  748.     EPEL_COMPUTE      14, %1, m12, m13

  749.     UNI_COMPUTE       %1, %2, m0, m1, [pw_%2]

  750.     PEL_%2STORE%1   dstq, m0, m1

  751.     movdqa            m4, m5

  752.     movdqa            m5, m6

  753.     movdqa            m6, m7

  754.     add             dstq, dststrideq             ; dst += dststride

  755.     add             srcq, srcstrideq             ; src += srcstride

  756.     dec          heightd                         ; cmp height

  757.     jnz               .loop                      ; height loop

  758.     RET

  759. 

  760. 

  761. cglobal hevc_put_hevc_bi_epel_hv%1_%2, 9, 11, 16, dst, dststride, src, srcstride, src2, src2stride, height, mx, my, r3src, rfilter

  762. %assign %%stride ((%2 + 7)/8)

  763.     sub             srcq, srcstrideq

  764.     EPEL_HV_FILTER    %2

  765.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  766.     EPEL_COMPUTE      %2, %1, m14, m15

  767.     SWAP              m4, m0

  768.     add             srcq, srcstrideq

  769.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  770.     EPEL_COMPUTE      %2, %1, m14, m15

  771.     SWAP              m5, m0

  772.     add             srcq, srcstrideq

  773.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  774.     EPEL_COMPUTE      %2, %1, m14, m15

  775.     SWAP              m6, m0

  776.     add             srcq, srcstrideq

  777. .loop

  778.     EPEL_LOAD         %2, srcq-%%stride, %%stride, %1

  779.     EPEL_COMPUTE      %2, %1, m14, m15

  780.     SWAP              m7, m0

  781.     punpcklwd         m0, m4, m5

  782.     punpcklwd         m2, m6, m7

  783. %if %1 > 4

  784.     punpckhwd         m1, m4, m5

  785.     punpckhwd         m3, m6, m7

  786. %endif

  787.     EPEL_COMPUTE      14, %1, m12, m13

  788.     SIMPLE_BILOAD     %1, src2q, m8, m9

  789.     BI_COMPUTE        %1, %2, m0, m1, m8, m9, [pw_bi_%2]

  790.     PEL_%2STORE%1   dstq, m0, m1

  791.     movdqa            m4, m5

  792.     movdqa            m5, m6

  793.     movdqa            m6, m7

  794.     add             dstq, dststrideq             ; dst += dststride

  795.     add             srcq, srcstrideq             ; src += srcstride

  796.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  797.     dec          heightd                         ; cmp height

  798.     jnz               .loop                      ; height loop

  799.     RET

  800. %endmacro

  801. 

  802. ; ******************************

  803. ; void put_hevc_qpel_hX_X_X(int16_t *dst, ptrdiff_t dststride,

  804. ;                       uint8_t *_src, ptrdiff_t _srcstride,

  805. ;                       int width, int height, int mx, int my)

  806. ; ******************************

  807. 

  808. %macro HEVC_PUT_HEVC_QPEL 2

  809. cglobal hevc_put_hevc_qpel_h%1_%2, 6, 7, 15 , dst, dststride, src, srcstride, height, mx, rfilter

  810.     QPEL_FILTER       %2, mx

  811. .loop

  812.     QPEL_H_LOAD       %2, srcq, %1, 10

  813.     QPEL_COMPUTE      %1, %2

  814. %if %2 > 8

  815.     packssdw          m0, m1

  816. %endif

  817.     PEL_10STORE%1     dstq, m0, m1

  818.     LOOP_END          dst, dststride, src, srcstride

  819.     RET

  820. 

  821. cglobal hevc_put_hevc_uni_qpel_h%1_%2, 6, 7, 15 , dst, dststride, src, srcstride, height, mx, rfilter

  822.     movdqa            m9, [pw_%2]

  823.     QPEL_FILTER       %2, mx

  824. .loop

  825.     QPEL_H_LOAD       %2, srcq, %1, 10

  826.     QPEL_COMPUTE      %1, %2

  827. %if %2 > 8

  828.     packssdw          m0, m1

  829. %endif

  830.     UNI_COMPUTE       %1, %2, m0, m1, m9

  831.     PEL_%2STORE%1   dstq, m0, m1

  832.     add             dstq, dststrideq             ; dst += dststride

  833.     add             srcq, srcstrideq             ; src += srcstride

  834.     dec          heightd                         ; cmp height

  835.     jnz               .loop                      ; height loop

  836.     RET

  837. 

  838. cglobal hevc_put_hevc_bi_qpel_h%1_%2, 8, 9, 16 , dst, dststride, src, srcstride, src2, src2stride, height, mx, rfilter

  839.     movdqa            m9, [pw_bi_%2]

  840.     QPEL_FILTER       %2, mx

  841. .loop

  842.     QPEL_H_LOAD       %2, srcq, %1, 10

  843.     QPEL_COMPUTE      %1, %2

  844. %if %2 > 8

  845.     packssdw          m0, m1

  846. %endif

  847.     SIMPLE_BILOAD     %1, src2q, m10, m11

  848.     BI_COMPUTE        %1, %2, m0, m1, m10, m11, m9

  849.     PEL_%2STORE%1   dstq, m0, m1

  850.     add             dstq, dststrideq             ; dst += dststride

  851.     add             srcq, srcstrideq             ; src += srcstride

  852.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  853.     dec          heightd                         ; cmp height

  854.     jnz               .loop                      ; height loop

  855.     RET

  856. 

  857. 

  858. ; ******************************

  859. ; void put_hevc_qpel_vX_X_X(int16_t *dst, ptrdiff_t dststride,

  860. ;                       uint8_t *_src, ptrdiff_t _srcstride,

  861. ;                       int width, int height, int mx, int my)

  862. ; ******************************

  863. 

  864. cglobal hevc_put_hevc_qpel_v%1_%2, 7, 14, 15 , dst, dststride, src, srcstride, height, r3src, my, rfilter

  865.     lea           r3srcq, [srcstrideq*3]

  866.     QPEL_FILTER       %2, my

  867. .loop

  868.     QPEL_V_LOAD       %2, srcq, srcstride, %1

  869.     QPEL_COMPUTE      %1, %2

  870. %if %2 > 8

  871.     packssdw          m0, m1

  872. %endif

  873.     PEL_10STORE%1     dstq, m0, m1

  874.     LOOP_END         dst, dststride, src, srcstride

  875.     RET

  876. 

  877. cglobal hevc_put_hevc_uni_qpel_v%1_%2, 7, 14, 15 , dst, dststride, src, srcstride, height, r3src, my, rfilter

  878.     movdqa            m9, [pw_%2]

  879.     lea           r3srcq, [srcstrideq*3]

  880.     QPEL_FILTER       %2, my

  881. .loop

  882.     QPEL_V_LOAD       %2, srcq, srcstride, %1

  883.     QPEL_COMPUTE      %1, %2

  884. %if %2 > 8

  885.     packusdw          m0, m1

  886. %endif

  887.     UNI_COMPUTE       %1, %2, m0, m1, m9

  888.     PEL_%2STORE%1   dstq, m0, m1

  889.     add             dstq, dststrideq             ; dst += dststride

  890.     add             srcq, srcstrideq             ; src += srcstride

  891.     dec          heightd                         ; cmp height

  892.     jnz               .loop                      ; height loop

  893.     RET

  894. 

  895. cglobal hevc_put_hevc_bi_qpel_v%1_%2, 9, 14, 16 , dst, dststride, src, srcstride, src2, src2stride, height, r3src, my, rfilter

  896.     movdqa            m9, [pw_bi_%2]

  897.     lea           r3srcq, [srcstrideq*3]

  898.     QPEL_FILTER       %2, my

  899. .loop

  900.     SIMPLE_BILOAD     %1, src2q, m10, m11

  901.     QPEL_V_LOAD       %2, srcq, srcstride, %1

  902.     QPEL_COMPUTE      %1, %2

  903. %if %2 > 8

  904.     packssdw          m0, m1

  905. %endif

  906.     BI_COMPUTE        %1, %2, m0, m1, m10, m11, m9

  907.     PEL_%2STORE%1   dstq, m0, m1

  908.     add             dstq, dststrideq             ; dst += dststride

  909.     add             srcq, srcstrideq             ; src += srcstride

  910.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  911.     dec          heightd                         ; cmp height

  912.     jnz               .loop                      ; height loop

  913.     RET

  914. %endmacro

  915. 

  916. 

  917. ; ******************************

  918. ; void put_hevc_qpel_hvX_X(int16_t *dst, ptrdiff_t dststride,

  919. ;                       uint8_t *_src, ptrdiff_t _srcstride,

  920. ;                       int height, int mx, int my)

  921. ; ******************************

  922. %macro HEVC_PUT_HEVC_QPEL_HV 2

  923. cglobal hevc_put_hevc_qpel_hv%1_%2, 7, 9, 12 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter

  924.     lea              mxq, [mxq*8-8]

  925.     lea              myq, [myq*8-8]

  926.     lea           r3srcq, [srcstrideq*3]

  927.     sub             srcq, r3srcq

  928.     QPEL_H_LOAD       %2, srcq, %1, 15

  929.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  930.     SWAP              m8, m0

  931.     add             srcq, srcstrideq

  932.     QPEL_H_LOAD       %2, srcq, %1, 15

  933.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  934.     SWAP              m9, m0

  935.     add             srcq, srcstrideq

  936.     QPEL_H_LOAD       %2, srcq, %1, 15

  937.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  938.     SWAP             m10, m0

  939.     add             srcq, srcstrideq

  940.     QPEL_H_LOAD       %2, srcq, %1, 15

  941.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  942.     SWAP             m11, m0

  943.     add             srcq, srcstrideq

  944.     QPEL_H_LOAD       %2, srcq, %1, 15

  945.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  946.     SWAP             m12, m0

  947.     add             srcq, srcstrideq

  948.     QPEL_H_LOAD       %2, srcq, %1, 15

  949.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  950.     SWAP             m13, m0

  951.     add             srcq, srcstrideq

  952.     QPEL_H_LOAD       %2, srcq, %1, 15

  953.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  954.     SWAP             m14, m0

  955.     add             srcq, srcstrideq

  956. .loop

  957.     QPEL_H_LOAD       %2, srcq, %1, 15

  958.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  959.     SWAP             m15, m0

  960.     punpcklwd         m0, m8, m9

  961.     punpcklwd         m2, m10, m11

  962.     punpcklwd         m4, m12, m13

  963.     punpcklwd         m6, m14, m15

  964. %if %1 > 4

  965.     punpckhwd         m1, m8, m9

  966.     punpckhwd         m3, m10, m11

  967.     punpckhwd         m5, m12, m13

  968.     punpckhwd         m7, m14, m15

  969. %endif

  970.     QPEL_HV_COMPUTE   %1, 14, my, ackssdw

  971.     PEL_10STORE%1     dstq, m0, m1

  972. %if %1 <= 4

  973.     movq              m8, m9

  974.     movq              m9, m10

  975.     movq             m10, m11

  976.     movq             m11, m12

  977.     movq             m12, m13

  978.     movq             m13, m14

  979.     movq             m14, m15

  980. %else

  981.     movdqa            m8, m9

  982.     movdqa            m9, m10

  983.     movdqa           m10, m11

  984.     movdqa           m11, m12

  985.     movdqa           m12, m13

  986.     movdqa           m13, m14

  987.     movdqa           m14, m15

  988. %endif

  989.     LOOP_END         dst, dststride, src, srcstride

  990.     RET

  991. 

  992. cglobal hevc_put_hevc_uni_qpel_hv%1_%2, 7, 9, 12 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter

  993.     lea              mxq, [mxq*8-8]

  994.     lea              myq, [myq*8-8]

  995.     lea           r3srcq, [srcstrideq*3]

  996.     sub             srcq, r3srcq

  997.     QPEL_H_LOAD       %2, srcq, %1, 15

  998.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  999.     SWAP              m8, m0

  1000.     add             srcq, srcstrideq

  1001.     QPEL_H_LOAD       %2, srcq, %1, 15

  1002.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1003.     SWAP              m9, m0

  1004.     add             srcq, srcstrideq

  1005.     QPEL_H_LOAD       %2, srcq, %1, 15

  1006.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1007.     SWAP             m10, m0

  1008.     add             srcq, srcstrideq

  1009.     QPEL_H_LOAD       %2, srcq, %1, 15

  1010.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1011.     SWAP             m11, m0

  1012.     add             srcq, srcstrideq

  1013.     QPEL_H_LOAD       %2, srcq, %1, 15

  1014.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1015.     SWAP             m12, m0

  1016.     add             srcq, srcstrideq

  1017.     QPEL_H_LOAD       %2, srcq, %1, 15

  1018.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1019.     SWAP             m13, m0

  1020.     add             srcq, srcstrideq

  1021.     QPEL_H_LOAD       %2, srcq, %1, 15

  1022.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1023.     SWAP             m14, m0

  1024.     add             srcq, srcstrideq

  1025. .loop

  1026.     QPEL_H_LOAD       %2, srcq, %1, 15

  1027.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1028.     SWAP             m15, m0

  1029.     punpcklwd         m0, m8, m9

  1030.     punpcklwd         m2, m10, m11

  1031.     punpcklwd         m4, m12, m13

  1032.     punpcklwd         m6, m14, m15

  1033. %if %1 > 4

  1034.     punpckhwd         m1, m8, m9

  1035.     punpckhwd         m3, m10, m11

  1036.     punpckhwd         m5, m12, m13

  1037.     punpckhwd         m7, m14, m15

  1038. %endif

  1039.     QPEL_HV_COMPUTE   %1, 14, my, ackusdw

  1040.     UNI_COMPUTE       %1, %2, m0, m1, [pw_%2]

  1041.     PEL_%2STORE%1   dstq, m0, m1

  1042. 

  1043. %if %1 <= 4

  1044.     movq              m8, m9

  1045.     movq              m9, m10

  1046.     movq             m10, m11

  1047.     movq             m11, m12

  1048.     movq             m12, m13

  1049.     movq             m13, m14

  1050.     movq             m14, m15

  1051. %else

  1052.     movdqa            m8, m9

  1053.     movdqa            m9, m10

  1054.     movdqa           m10, m11

  1055.     movdqa           m11, m12

  1056.     movdqa           m12, m13

  1057.     movdqa           m13, m14

  1058.     movdqa           m14, m15

  1059. %endif

  1060.     add             dstq, dststrideq             ; dst += dststride

  1061.     add             srcq, srcstrideq             ; src += srcstride

  1062.     dec          heightd                         ; cmp height

  1063.     jnz               .loop                      ; height loop

  1064.     RET

  1065. 

  1066. cglobal hevc_put_hevc_bi_qpel_hv%1_%2, 9, 11, 16, dst, dststride, src, srcstride, src2, src2stride, height, mx, my, r3src, rfilter

  1067.     lea              mxq, [mxq*8-8]

  1068.     lea              myq, [myq*8-8]

  1069.     lea           r3srcq, [srcstrideq*3]

  1070.     sub             srcq, r3srcq

  1071.     QPEL_H_LOAD       %2, srcq, %1, 15

  1072.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1073.     SWAP              m8, m0

  1074.     add             srcq, srcstrideq

  1075.     QPEL_H_LOAD       %2, srcq, %1, 15

  1076.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1077.     SWAP              m9, m0

  1078.     add             srcq, srcstrideq

  1079.     QPEL_H_LOAD       %2, srcq, %1, 15

  1080.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1081.     SWAP             m10, m0

  1082.     add             srcq, srcstrideq

  1083.     QPEL_H_LOAD       %2, srcq, %1, 15

  1084.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1085.     SWAP             m11, m0

  1086.     add             srcq, srcstrideq

  1087.     QPEL_H_LOAD       %2, srcq, %1, 15

  1088.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1089.     SWAP             m12, m0

  1090.     add             srcq, srcstrideq

  1091.     QPEL_H_LOAD       %2, srcq, %1, 15

  1092.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1093.     SWAP             m13, m0

  1094.     add             srcq, srcstrideq

  1095.     QPEL_H_LOAD       %2, srcq, %1, 15

  1096.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1097.     SWAP             m14, m0

  1098.     add             srcq, srcstrideq

  1099. .loop

  1100.     QPEL_H_LOAD       %2, srcq, %1, 15

  1101.     QPEL_HV_COMPUTE   %1, %2, mx, ackssdw

  1102.     SWAP             m15, m0

  1103.     punpcklwd         m0, m8, m9

  1104.     punpcklwd         m2, m10, m11

  1105.     punpcklwd         m4, m12, m13

  1106.     punpcklwd         m6, m14, m15

  1107. %if %1 > 4

  1108.     punpckhwd         m1, m8, m9

  1109.     punpckhwd         m3, m10, m11

  1110.     punpckhwd         m5, m12, m13

  1111.     punpckhwd         m7, m14, m15

  1112. %endif

  1113.     QPEL_HV_COMPUTE   %1, 14, my, ackssdw

  1114.     SIMPLE_BILOAD     %1, src2q, m8, m9 ;m9 not used in this case

  1115.     BI_COMPUTE        %1, %2, m0, m1, m8, m9, [pw_bi_%2]

  1116.     PEL_%2STORE%1   dstq, m0, m1

  1117. 

  1118. %if %1 <= 4

  1119.     movq              m8, m9

  1120.     movq              m9, m10

  1121.     movq             m10, m11

  1122.     movq             m11, m12

  1123.     movq             m12, m13

  1124.     movq             m13, m14

  1125.     movq             m14, m15

  1126. %else

  1127.     movdqa            m8, m9

  1128.     movdqa            m9, m10

  1129.     movdqa           m10, m11

  1130.     movdqa           m11, m12

  1131.     movdqa           m12, m13

  1132.     movdqa           m13, m14

  1133.     movdqa           m14, m15

  1134. %endif

  1135.     add             dstq, dststrideq             ; dst += dststride

  1136.     add             srcq, srcstrideq             ; src += srcstride

  1137.     lea            src2q, [src2q+2*src2strideq]  ; src += srcstride

  1138.     dec          heightd                         ; cmp height

  1139.     jnz               .loop                      ; height loop

  1140.     RET

  1141. %endmacro

  1142. 

  1143. %macro WEIGHTING_FUNCS 2

  1144. %if WIN64 || ARCH_X86_32

  1145. cglobal hevc_put_hevc_uni_w%1_%2, 4, 5, 7, dst, dststride, src, srcstride, height, denom, wx, ox

  1146.     mov             r4d, denomm

  1147. %define SHIFT  r4d

  1148. %else

  1149. cglobal hevc_put_hevc_uni_w%1_%2, 6, 6, 7, dst, dststride, src, srcstride, height, denom, wx, ox

  1150. %define SHIFT  denomd

  1151. %endif

  1152.     lea           SHIFT, [SHIFT+14-%2]          ; shift = 14 - bitd + denom

  1153.     movd             m2, wxm        ; WX

  1154.     movd             m4, SHIFT      ; shift

  1155.     punpcklwd        m2, m2

  1156.     dec           SHIFT

  1157.     movdqu           m5, [one_per_32]

  1158.     movd             m6, SHIFT

  1159.     pshufd           m2, m2, 0

  1160.     mov           SHIFT, oxm

  1161.     pslld            m5, m6

  1162. %if %2 != 8

  1163.     shl           SHIFT, %2-8       ; ox << (bitd - 8)

  1164. %endif

  1165.     movd             m3, SHIFT      ; OX

  1166.     pshufd           m3, m3, 0

  1167. %if WIN64 || ARCH_X86_32

  1168.     mov           SHIFT, heightm

  1169. %endif

  1170. .loop

  1171.    SIMPLE_LOAD        %1, 10, srcq, m0

  1172.     pmulhw            m6, m0, m2

  1173.     pmullw            m0, m2

  1174.     punpckhwd         m1, m0, m6

  1175.     punpcklwd         m0, m6

  1176.     paddd             m0, m5

  1177.     paddd             m1, m5

  1178.     psrad             m0, m4

  1179.     psrad             m1, m4

  1180.     paddd             m0, m3

  1181.     paddd             m1, m3

  1182.     packusdw          m0, m1

  1183. %if %2 == 8

  1184.     packuswb          m0, m0

  1185. %else

  1186.     pminsw            m0, [max_pixels_%2]

  1187. %endif

  1188.     PEL_%2STORE%1   dstq, m0, m1

  1189.     add             dstq, dststrideq             ; dst += dststride

  1190.     lea             srcq, [srcq+2*srcstrideq]      ; src += srcstride

  1191.     dec          heightd                         ; cmp height

  1192.     jnz               .loop                      ; height loop

  1193.     RET

  1194. 

  1195. cglobal hevc_put_hevc_bi_w%1_%2, 6, 7, 10, dst, dststride, src, srcstride, src2, src2stride, height, denom, wx0, wx1, ox0, ox1

  1196.     mov              r6d, denomm

  1197.     movd              m2, wx0m         ; WX0

  1198.     lea              r6d, [r6d+14-%2]  ; shift = 14 - bitd + denom

  1199.     movd              m3, wx1m         ; WX1

  1200.     movd              m0, r6d          ; shift

  1201.     punpcklwd         m2, m2

  1202.     inc              r6d

  1203.     punpcklwd         m3, m3

  1204.     movd              m5, r6d          ; shift+1

  1205.     pshufd            m2, m2, 0

  1206.     mov              r6d, ox0m

  1207.     pshufd            m3, m3, 0

  1208.     add              r6d, ox1m

  1209. %if %2 != 8

  1210.     shl              r6d, %2-8         ; ox << (bitd - 8)

  1211. %endif

  1212.     inc              r6d

  1213.     movd              m4, r6d          ; offset

  1214.     pshufd            m4, m4, 0

  1215.     mov              r6d, heightm

  1216.     pslld             m4, m0

  1217. 

  1218. .loop

  1219.    SIMPLE_LOAD        %1, 10, srcq,  m0

  1220.    SIMPLE_LOAD        %1, 10, src2q, m8

  1221.     pmulhw            m6, m0, m3

  1222.     pmullw            m0, m3

  1223.     pmulhw            m7, m8, m2

  1224.     pmullw            m8, m2

  1225.     punpckhwd         m1, m0, m6

  1226.     punpcklwd         m0, m6

  1227.     punpckhwd         m9, m8, m7

  1228.     punpcklwd         m8, m7

  1229.     paddd             m0, m8

  1230.     paddd             m1, m9

  1231.     paddd             m0, m4

  1232.     paddd             m1, m4

  1233.     psrad             m0, m5

  1234.     psrad             m1, m5

  1235.     packusdw          m0, m1

  1236. %if %2 == 8

  1237.     packuswb          m0, m0

  1238. %else

  1239.     pminsw            m0, [max_pixels_%2]

  1240. %endif

  1241.     PEL_%2STORE%1   dstq, m0, m1

  1242.     add             dstq, dststrideq             ; dst += dststride

  1243.     lea             srcq, [srcq+2*srcstrideq]      ; src += srcstride

  1244.     lea            src2q, [src2q+2*src2strideq]      ; src2 += srcstride

  1245.     dec              r6d                         ; cmp height

  1246.     jnz               .loop                      ; height loop

  1247.     RET

  1248. %endmacro

  1249. 

  1250. WEIGHTING_FUNCS 2, 8

  1251. WEIGHTING_FUNCS 4, 8

  1252. WEIGHTING_FUNCS 6, 8

  1253. WEIGHTING_FUNCS 8, 8

  1254. 

  1255. WEIGHTING_FUNCS 2, 10

  1256. WEIGHTING_FUNCS 4, 10

  1257. WEIGHTING_FUNCS 6, 10

  1258. WEIGHTING_FUNCS 8, 10

  1259. 

  1260. WEIGHTING_FUNCS 2, 12

  1261. WEIGHTING_FUNCS 4, 12

  1262. WEIGHTING_FUNCS 6, 12

  1263. WEIGHTING_FUNCS 8, 12

  1264. 

  1265. HEVC_PUT_HEVC_PEL_PIXELS  2, 8

  1266. HEVC_PUT_HEVC_PEL_PIXELS  4, 8

  1267. HEVC_PUT_HEVC_PEL_PIXELS  6, 8

  1268. HEVC_PUT_HEVC_PEL_PIXELS  8, 8

  1269. HEVC_PUT_HEVC_PEL_PIXELS 12, 8

  1270. HEVC_PUT_HEVC_PEL_PIXELS 16, 8

  1271. 

  1272. HEVC_PUT_HEVC_PEL_PIXELS 2, 10

  1273. HEVC_PUT_HEVC_PEL_PIXELS 4, 10

  1274. HEVC_PUT_HEVC_PEL_PIXELS 6, 10

  1275. HEVC_PUT_HEVC_PEL_PIXELS 8, 10

  1276. 

  1277. HEVC_PUT_HEVC_PEL_PIXELS 2, 12

  1278. HEVC_PUT_HEVC_PEL_PIXELS 4, 12

  1279. HEVC_PUT_HEVC_PEL_PIXELS 6, 12

  1280. HEVC_PUT_HEVC_PEL_PIXELS 8, 12

  1281. 

  1282. HEVC_PUT_HEVC_EPEL 2,  8

  1283. HEVC_PUT_HEVC_EPEL 4,  8

  1284. HEVC_PUT_HEVC_EPEL 6,  8

  1285. HEVC_PUT_HEVC_EPEL 8,  8

  1286. HEVC_PUT_HEVC_EPEL 12, 8

  1287. HEVC_PUT_HEVC_EPEL 16, 8

  1288. 

  1289. 

  1290. HEVC_PUT_HEVC_EPEL 2, 10

  1291. HEVC_PUT_HEVC_EPEL 4, 10

  1292. HEVC_PUT_HEVC_EPEL 6, 10

  1293. HEVC_PUT_HEVC_EPEL 8, 10

  1294. 

  1295. HEVC_PUT_HEVC_EPEL 2, 12

  1296. HEVC_PUT_HEVC_EPEL 4, 12

  1297. HEVC_PUT_HEVC_EPEL 6, 12

  1298. HEVC_PUT_HEVC_EPEL 8, 12

  1299. 

  1300. HEVC_PUT_HEVC_EPEL_HV 2,  8

  1301. HEVC_PUT_HEVC_EPEL_HV 4,  8

  1302. HEVC_PUT_HEVC_EPEL_HV 6,  8

  1303. HEVC_PUT_HEVC_EPEL_HV 8,  8

  1304. 

  1305. HEVC_PUT_HEVC_EPEL_HV 2, 10

  1306. HEVC_PUT_HEVC_EPEL_HV 4, 10

  1307. HEVC_PUT_HEVC_EPEL_HV 6, 10

  1308. HEVC_PUT_HEVC_EPEL_HV 8, 10

  1309. 

  1310. HEVC_PUT_HEVC_EPEL_HV 2, 12

  1311. HEVC_PUT_HEVC_EPEL_HV 4, 12

  1312. HEVC_PUT_HEVC_EPEL_HV 6, 12

  1313. HEVC_PUT_HEVC_EPEL_HV 8, 12

  1314. 

  1315. HEVC_PUT_HEVC_QPEL 4,  8

  1316. HEVC_PUT_HEVC_QPEL 8,  8

  1317. HEVC_PUT_HEVC_QPEL 12, 8

  1318. HEVC_PUT_HEVC_QPEL 16, 8

  1319. 

  1320. HEVC_PUT_HEVC_QPEL 4, 10

  1321. HEVC_PUT_HEVC_QPEL 8, 10

  1322. 

  1323. HEVC_PUT_HEVC_QPEL 4, 12

  1324. HEVC_PUT_HEVC_QPEL 8, 12

  1325. 

  1326. HEVC_PUT_HEVC_QPEL_HV 2, 8

  1327. HEVC_PUT_HEVC_QPEL_HV 4, 8

  1328. HEVC_PUT_HEVC_QPEL_HV 6, 8

  1329. HEVC_PUT_HEVC_QPEL_HV 8, 8

  1330. 

  1331. HEVC_PUT_HEVC_QPEL_HV 2, 10

  1332. HEVC_PUT_HEVC_QPEL_HV 4, 10

  1333. HEVC_PUT_HEVC_QPEL_HV 6, 10

  1334. HEVC_PUT_HEVC_QPEL_HV 8, 10

  1335. 

  1336. HEVC_PUT_HEVC_QPEL_HV 2, 12

  1337. HEVC_PUT_HEVC_QPEL_HV 4, 12

  1338. HEVC_PUT_HEVC_QPEL_HV 6, 12

  1339. HEVC_PUT_HEVC_QPEL_HV 8, 12

  1340. 

  1341. %endif ; ARCH_X86_64