falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
25017 Commits
32 Branches
301MB
Tree: 7af8fbd348
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '7af8fbd348'
${ noResults }
FFmpeg/libavcodec/x86/vp8dsp.asm

1372 lines
38KB
Raw Blame History 

  1. ;******************************************************************************

  2. ;* VP8 MMXEXT optimizations

  3. ;* Copyright (c) 2010 Ronald S. Bultje <rsbultje@gmail.com>

  4. ;* Copyright (c) 2010 Jason Garrett-Glaser <darkshikari@gmail.com>

  5. ;*

  6. ;* This file is part of FFmpeg.

  7. ;*

  8. ;* FFmpeg is free software; you can redistribute it and/or

  9. ;* modify it under the terms of the GNU Lesser General Public

  10. ;* License as published by the Free Software Foundation; either

  11. ;* version 2.1 of the License, or (at your option) any later version.

  12. ;*

  13. ;* FFmpeg is distributed in the hope that it will be useful,

  14. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  15. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  16. ;* Lesser General Public License for more details.

  17. ;*

  18. ;* You should have received a copy of the GNU Lesser General Public

  19. ;* License along with FFmpeg; if not, write to the Free Software

  20. ;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  21. ;******************************************************************************

  22. 

  23. %include "x86inc.asm"

  24. %include "x86util.asm"

  25. 

  26. SECTION_RODATA

  27. 

  28. fourtap_filter_hw_m: times 4 dw  -6, 123

  29.                      times 4 dw  12,  -1

  30.                      times 4 dw  -9,  93

  31.                      times 4 dw  50,  -6

  32.                      times 4 dw  -6,  50

  33.                      times 4 dw  93,  -9

  34.                      times 4 dw  -1,  12

  35.                      times 4 dw 123,  -6

  36. 

  37. sixtap_filter_hw_m:  times 4 dw   2, -11

  38.                      times 4 dw 108,  36

  39.                      times 4 dw  -8,   1

  40.                      times 4 dw   3, -16

  41.                      times 4 dw  77,  77

  42.                      times 4 dw -16,   3

  43.                      times 4 dw   1,  -8

  44.                      times 4 dw  36, 108

  45.                      times 4 dw -11,   2

  46. 

  47. fourtap_filter_hb_m: times 8 db  -6, 123

  48.                      times 8 db  12,  -1

  49.                      times 8 db  -9,  93

  50.                      times 8 db  50,  -6

  51.                      times 8 db  -6,  50

  52.                      times 8 db  93,  -9

  53.                      times 8 db  -1,  12

  54.                      times 8 db 123,  -6

  55. 

  56. sixtap_filter_hb_m:  times 8 db   2,   1

  57.                      times 8 db -11, 108

  58.                      times 8 db  36,  -8

  59.                      times 8 db   3,   3

  60.                      times 8 db -16,  77

  61.                      times 8 db  77, -16

  62.                      times 8 db   1,   2

  63.                      times 8 db  -8,  36

  64.                      times 8 db 108, -11

  65. 

  66. fourtap_filter_v_m:  times 8 dw  -6

  67.                      times 8 dw 123

  68.                      times 8 dw  12

  69.                      times 8 dw  -1

  70.                      times 8 dw  -9

  71.                      times 8 dw  93

  72.                      times 8 dw  50

  73.                      times 8 dw  -6

  74.                      times 8 dw  -6

  75.                      times 8 dw  50

  76.                      times 8 dw  93

  77.                      times 8 dw  -9

  78.                      times 8 dw  -1

  79.                      times 8 dw  12

  80.                      times 8 dw 123

  81.                      times 8 dw  -6

  82. 

  83. sixtap_filter_v_m:   times 8 dw   2

  84.                      times 8 dw -11

  85.                      times 8 dw 108

  86.                      times 8 dw  36

  87.                      times 8 dw  -8

  88.                      times 8 dw   1

  89.                      times 8 dw   3

  90.                      times 8 dw -16

  91.                      times 8 dw  77

  92.                      times 8 dw  77

  93.                      times 8 dw -16

  94.                      times 8 dw   3

  95.                      times 8 dw   1

  96.                      times 8 dw  -8

  97.                      times 8 dw  36

  98.                      times 8 dw 108

  99.                      times 8 dw -11

  100.                      times 8 dw   2

  101. 

  102. bilinear_filter_vw_m: times 8 dw 1

  103.                       times 8 dw 2

  104.                       times 8 dw 3

  105.                       times 8 dw 4

  106.                       times 8 dw 5

  107.                       times 8 dw 6

  108.                       times 8 dw 7

  109. 

  110. bilinear_filter_vb_m: times 8 db 7, 1

  111.                       times 8 db 6, 2

  112.                       times 8 db 5, 3

  113.                       times 8 db 4, 4

  114.                       times 8 db 3, 5

  115.                       times 8 db 2, 6

  116.                       times 8 db 1, 7

  117. 

  118. %ifdef PIC

  119. %define fourtap_filter_hw    r11

  120. %define sixtap_filter_hw     r11

  121. %define fourtap_filter_hb    r11

  122. %define sixtap_filter_hb     r11

  123. %define fourtap_filter_v     r11

  124. %define sixtap_filter_v      r11

  125. %define bilinear_filter_vw   r11

  126. %define bilinear_filter_vb   r11

  127. %else

  128. %define fourtap_filter_hw fourtap_filter_hw_m

  129. %define sixtap_filter_hw  sixtap_filter_hw_m

  130. %define fourtap_filter_hb fourtap_filter_hb_m

  131. %define sixtap_filter_hb  sixtap_filter_hb_m

  132. %define fourtap_filter_v  fourtap_filter_v_m

  133. %define sixtap_filter_v   sixtap_filter_v_m

  134. %define bilinear_filter_vw bilinear_filter_vw_m

  135. %define bilinear_filter_vb bilinear_filter_vb_m

  136. %endif

  137. 

  138. filter_h2_shuf:  db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,  6, 6,  7,  7,  8

  139. filter_h4_shuf:  db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,  8, 8,  9,  9, 10

  140. 

  141. filter_h6_shuf1: db 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11,  7, 12

  142. filter_h6_shuf2: db 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,  7, 7,  8,  8,  9

  143. filter_h6_shuf3: db 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,  9, 9, 10, 10, 11

  144. 

  145. pw_20091: times 4 dw 20091

  146. pw_17734: times 4 dw 17734

  147. 

  148. cextern pw_3

  149. cextern pb_3

  150. cextern pw_4

  151. cextern pb_4

  152. cextern pw_64

  153. cextern pb_80

  154. cextern pb_F8

  155. cextern pb_FE

  156. 

  157. SECTION .text

  158. 

  159. ;-----------------------------------------------------------------------------

  160. ; subpel MC functions:

  161. ;

  162. ; void put_vp8_epel<size>_h<htap>v<vtap>_<opt>(uint8_t *dst, int deststride,

  163. ;                                              uint8_t *src, int srcstride,

  164. ;                                              int height,   int mx, int my);

  165. ;-----------------------------------------------------------------------------

  166. 

  167. %macro FILTER_SSSE3 3

  168. cglobal put_vp8_epel%1_h6_ssse3, 6, 6, %2

  169.     lea      r5d, [r5*3]

  170.     mova      m3, [filter_h6_shuf2]

  171.     mova      m4, [filter_h6_shuf3]

  172. %ifdef PIC

  173.     lea      r11, [sixtap_filter_hb_m]

  174. %endif

  175.     mova      m5, [sixtap_filter_hb+r5*8-48] ; set up 6tap filter in bytes

  176.     mova      m6, [sixtap_filter_hb+r5*8-32]

  177.     mova      m7, [sixtap_filter_hb+r5*8-16]

  178. 

  179. .nextrow

  180.     movu      m0, [r2-2]

  181.     mova      m1, m0

  182.     mova      m2, m0

  183. %ifidn %1, 4

  184. ; For epel4, we need 9 bytes, but only 8 get loaded; to compensate, do the

  185. ; shuffle with a memory operand

  186.     punpcklbw m0, [r2+3]

  187. %else

  188.     pshufb    m0, [filter_h6_shuf1]

  189. %endif

  190.     pshufb    m1, m3

  191.     pshufb    m2, m4

  192.     pmaddubsw m0, m5

  193.     pmaddubsw m1, m6

  194.     pmaddubsw m2, m7

  195.     paddsw    m0, m1

  196.     paddsw    m0, m2

  197.     paddsw    m0, [pw_64]

  198.     psraw     m0, 7

  199.     packuswb  m0, m0

  200.     movh    [r0], m0        ; store

  201. 

  202.     ; go to next line

  203.     add       r0, r1

  204.     add       r2, r3

  205.     dec       r4            ; next row

  206.     jg .nextrow

  207.     REP_RET

  208. 

  209. cglobal put_vp8_epel%1_h4_ssse3, 6, 6, %3

  210.     shl      r5d, 4

  211.     mova      m2, [pw_64]

  212.     mova      m3, [filter_h2_shuf]

  213.     mova      m4, [filter_h4_shuf]

  214. %ifdef PIC

  215.     lea      r11, [fourtap_filter_hb_m]

  216. %endif

  217.     mova      m5, [fourtap_filter_hb+r5-16] ; set up 4tap filter in bytes

  218.     mova      m6, [fourtap_filter_hb+r5]

  219. 

  220. .nextrow

  221.     movu      m0, [r2-1]

  222.     mova      m1, m0

  223.     pshufb    m0, m3

  224.     pshufb    m1, m4

  225.     pmaddubsw m0, m5

  226.     pmaddubsw m1, m6

  227.     paddsw    m0, m2

  228.     paddsw    m0, m1

  229.     psraw     m0, 7

  230.     packuswb  m0, m0

  231.     movh    [r0], m0        ; store

  232. 

  233.     ; go to next line

  234.     add       r0, r1

  235.     add       r2, r3

  236.     dec       r4            ; next row

  237.     jg .nextrow

  238.     REP_RET

  239. 

  240. cglobal put_vp8_epel%1_v4_ssse3, 7, 7, %2

  241.     shl      r6d, 4

  242. %ifdef PIC

  243.     lea      r11, [fourtap_filter_hb_m]

  244. %endif

  245.     mova      m5, [fourtap_filter_hb+r6-16]

  246.     mova      m6, [fourtap_filter_hb+r6]

  247.     mova      m7, [pw_64]

  248. 

  249.     ; read 3 lines

  250.     sub       r2, r3

  251.     movh      m0, [r2]

  252.     movh      m1, [r2+  r3]

  253.     movh      m2, [r2+2*r3]

  254.     add       r2, r3

  255. 

  256. .nextrow

  257.     movh      m3, [r2+2*r3]                ; read new row

  258.     mova      m4, m0

  259.     mova      m0, m1

  260.     punpcklbw m4, m1

  261.     mova      m1, m2

  262.     punpcklbw m2, m3

  263.     pmaddubsw m4, m5

  264.     pmaddubsw m2, m6

  265.     paddsw    m4, m2

  266.     mova      m2, m3

  267.     paddsw    m4, m7

  268.     psraw     m4, 7

  269.     packuswb  m4, m4

  270.     movh    [r0], m4

  271. 

  272.     ; go to next line

  273.     add        r0, r1

  274.     add        r2, r3

  275.     dec        r4                          ; next row

  276.     jg .nextrow

  277.     REP_RET

  278. 

  279. cglobal put_vp8_epel%1_v6_ssse3, 7, 7, %2

  280.     lea      r6d, [r6*3]

  281. %ifdef PIC

  282.     lea      r11, [sixtap_filter_hb_m]

  283. %endif

  284.     lea       r6, [sixtap_filter_hb+r6*8]

  285. 

  286.     ; read 5 lines

  287.     sub       r2, r3

  288.     sub       r2, r3

  289.     movh      m0, [r2]

  290.     movh      m1, [r2+r3]

  291.     movh      m2, [r2+r3*2]

  292.     lea       r2, [r2+r3*2]

  293.     add       r2, r3

  294.     movh      m3, [r2]

  295.     movh      m4, [r2+r3]

  296. 

  297. .nextrow

  298.     movh      m5, [r2+2*r3]                ; read new row

  299.     mova      m6, m0

  300.     punpcklbw m6, m5

  301.     mova      m0, m1

  302.     punpcklbw m1, m2

  303.     mova      m7, m3

  304.     punpcklbw m7, m4

  305.     pmaddubsw m6, [r6-48]

  306.     pmaddubsw m1, [r6-32]

  307.     pmaddubsw m7, [r6-16]

  308.     paddsw    m6, m1

  309.     paddsw    m6, m7

  310.     mova      m1, m2

  311.     paddsw    m6, [pw_64]

  312.     mova      m2, m3

  313.     psraw     m6, 7

  314.     mova      m3, m4

  315.     packuswb  m6, m6

  316.     mova      m4, m5

  317.     movh    [r0], m6

  318. 

  319.     ; go to next line

  320.     add        r0, r1

  321.     add        r2, r3

  322.     dec        r4                          ; next row

  323.     jg .nextrow

  324.     REP_RET

  325. %endmacro

  326. 

  327. INIT_MMX

  328. FILTER_SSSE3 4, 0, 0

  329. INIT_XMM

  330. FILTER_SSSE3 8, 8, 7

  331. 

  332. ; 4x4 block, H-only 4-tap filter

  333. cglobal put_vp8_epel4_h4_mmxext, 6, 6

  334.     shl       r5d, 4

  335. %ifdef PIC

  336.     lea       r11, [fourtap_filter_hw_m]

  337. %endif

  338.     movq      mm4, [fourtap_filter_hw+r5-16] ; set up 4tap filter in words

  339.     movq      mm5, [fourtap_filter_hw+r5]

  340.     movq      mm7, [pw_64]

  341.     pxor      mm6, mm6

  342. 

  343. .nextrow

  344.     movq      mm1, [r2-1]                  ; (ABCDEFGH) load 8 horizontal pixels

  345. 

  346.     ; first set of 2 pixels

  347.     movq      mm2, mm1                     ; byte ABCD..

  348.     punpcklbw mm1, mm6                     ; byte->word ABCD

  349.     pshufw    mm0, mm2, 9                  ; byte CDEF..

  350.     punpcklbw mm0, mm6                     ; byte->word CDEF

  351.     pshufw    mm3, mm1, 0x94               ; word ABBC

  352.     pshufw    mm1, mm0, 0x94               ; word CDDE

  353.     pmaddwd   mm3, mm4                     ; multiply 2px with F0/F1

  354.     movq      mm0, mm1                     ; backup for second set of pixels

  355.     pmaddwd   mm1, mm5                     ; multiply 2px with F2/F3

  356.     paddd     mm3, mm1                     ; finish 1st 2px

  357. 

  358.     ; second set of 2 pixels, use backup of above

  359.     punpckhbw mm2, mm6                     ; byte->word EFGH

  360.     pmaddwd   mm0, mm4                     ; multiply backed up 2px with F0/F1

  361.     pshufw    mm1, mm2, 0x94               ; word EFFG

  362.     pmaddwd   mm1, mm5                     ; multiply 2px with F2/F3

  363.     paddd     mm0, mm1                     ; finish 2nd 2px

  364. 

  365.     ; merge two sets of 2 pixels into one set of 4, round/clip/store

  366.     packssdw  mm3, mm0                     ; merge dword->word (4px)

  367.     paddsw    mm3, mm7                     ; rounding

  368.     psraw     mm3, 7

  369.     packuswb  mm3, mm6                     ; clip and word->bytes

  370.     movd     [r0], mm3                     ; store

  371. 

  372.     ; go to next line

  373.     add        r0, r1

  374.     add        r2, r3

  375.     dec        r4                          ; next row

  376.     jg .nextrow

  377.     REP_RET

  378. 

  379. ; 4x4 block, H-only 6-tap filter

  380. cglobal put_vp8_epel4_h6_mmxext, 6, 6

  381.     lea       r5d, [r5*3]

  382. %ifdef PIC

  383.     lea       r11, [sixtap_filter_hw_m]

  384. %endif

  385.     movq      mm4, [sixtap_filter_hw+r5*8-48] ; set up 4tap filter in words

  386.     movq      mm5, [sixtap_filter_hw+r5*8-32]

  387.     movq      mm6, [sixtap_filter_hw+r5*8-16]

  388.     movq      mm7, [pw_64]

  389.     pxor      mm3, mm3

  390. 

  391. .nextrow

  392.     movq      mm1, [r2-2]                  ; (ABCDEFGH) load 8 horizontal pixels

  393. 

  394.     ; first set of 2 pixels

  395.     movq      mm2, mm1                     ; byte ABCD..

  396.     punpcklbw mm1, mm3                     ; byte->word ABCD

  397.     pshufw    mm0, mm2, 0x9                ; byte CDEF..

  398.     punpckhbw mm2, mm3                     ; byte->word EFGH

  399.     punpcklbw mm0, mm3                     ; byte->word CDEF

  400.     pshufw    mm1, mm1, 0x94               ; word ABBC

  401.     pshufw    mm2, mm2, 0x94               ; word EFFG

  402.     pmaddwd   mm1, mm4                     ; multiply 2px with F0/F1

  403.     pshufw    mm3, mm0, 0x94               ; word CDDE

  404.     movq      mm0, mm3                     ; backup for second set of pixels

  405.     pmaddwd   mm3, mm5                     ; multiply 2px with F2/F3

  406.     paddd     mm1, mm3                     ; add to 1st 2px cache

  407.     movq      mm3, mm2                     ; backup for second set of pixels

  408.     pmaddwd   mm2, mm6                     ; multiply 2px with F4/F5

  409.     paddd     mm1, mm2                     ; finish 1st 2px

  410. 

  411.     ; second set of 2 pixels, use backup of above

  412.     movd      mm2, [r2+3]                  ; byte FGHI (prevent overreads)

  413.     pmaddwd   mm0, mm4                     ; multiply 1st backed up 2px with F0/F1

  414.     pmaddwd   mm3, mm5                     ; multiply 2nd backed up 2px with F2/F3

  415.     paddd     mm0, mm3                     ; add to 2nd 2px cache

  416.     pxor      mm3, mm3

  417.     punpcklbw mm2, mm3                     ; byte->word FGHI

  418.     pshufw    mm2, mm2, 0xE9               ; word GHHI

  419.     pmaddwd   mm2, mm6                     ; multiply 2px with F4/F5

  420.     paddd     mm0, mm2                     ; finish 2nd 2px

  421. 

  422.     ; merge two sets of 2 pixels into one set of 4, round/clip/store

  423.     packssdw  mm1, mm0                     ; merge dword->word (4px)

  424.     paddsw    mm1, mm7                     ; rounding

  425.     psraw     mm1, 7

  426.     packuswb  mm1, mm3                     ; clip and word->bytes

  427.     movd     [r0], mm1                     ; store

  428. 

  429.     ; go to next line

  430.     add        r0, r1

  431.     add        r2, r3

  432.     dec        r4                          ; next row

  433.     jg .nextrow

  434.     REP_RET

  435. 

  436. ; 4x4 block, H-only 4-tap filter

  437. INIT_XMM

  438. cglobal put_vp8_epel8_h4_sse2, 6, 6, 8

  439.     shl      r5d, 4

  440. %ifdef PIC

  441.     lea      r11, [fourtap_filter_hw_m]

  442. %endif

  443.     mova      m5, [fourtap_filter_hw+r5-16] ; set up 4tap filter in words

  444.     mova      m6, [fourtap_filter_hw+r5]

  445.     pxor      m7, m7

  446. 

  447. .nextrow

  448.     movh      m0, [r2-1]

  449.     punpcklbw m0, m7        ; ABCDEFGH

  450.     mova      m1, m0

  451.     mova      m2, m0

  452.     mova      m3, m0

  453.     psrldq    m1, 2         ; BCDEFGH

  454.     psrldq    m2, 4         ; CDEFGH

  455.     psrldq    m3, 6         ; DEFGH

  456.     punpcklwd m0, m1        ; ABBCCDDE

  457.     punpcklwd m2, m3        ; CDDEEFFG

  458.     pmaddwd   m0, m5

  459.     pmaddwd   m2, m6

  460.     paddd     m0, m2

  461. 

  462.     movh      m1, [r2+3]

  463.     punpcklbw m1, m7        ; ABCDEFGH

  464.     mova      m2, m1

  465.     mova      m3, m1

  466.     mova      m4, m1

  467.     psrldq    m2, 2         ; BCDEFGH

  468.     psrldq    m3, 4         ; CDEFGH

  469.     psrldq    m4, 6         ; DEFGH

  470.     punpcklwd m1, m2        ; ABBCCDDE

  471.     punpcklwd m3, m4        ; CDDEEFFG

  472.     pmaddwd   m1, m5

  473.     pmaddwd   m3, m6

  474.     paddd     m1, m3

  475. 

  476.     packssdw  m0, m1

  477.     paddsw    m0, [pw_64]

  478.     psraw     m0, 7

  479.     packuswb  m0, m7

  480.     movh    [r0], m0        ; store

  481. 

  482.     ; go to next line

  483.     add       r0, r1

  484.     add       r2, r3

  485.     dec       r4            ; next row

  486.     jg .nextrow

  487.     REP_RET

  488. 

  489. cglobal put_vp8_epel8_h6_sse2, 6, 6, 8

  490.     lea      r5d, [r5*3]

  491. %ifdef PIC

  492.     lea      r11, [sixtap_filter_hw_m]

  493. %endif

  494.     lea       r5, [sixtap_filter_hw+r5*8]

  495.     pxor      m7, m7

  496. 

  497. .nextrow

  498.     movu      m0, [r2-2]

  499.     mova      m6, m0

  500.     mova      m4, m0

  501.     punpcklbw m0, m7        ; ABCDEFGHI

  502.     mova      m1, m0

  503.     mova      m2, m0

  504.     mova      m3, m0

  505.     psrldq    m1, 2         ; BCDEFGH

  506.     psrldq    m2, 4         ; CDEFGH

  507.     psrldq    m3, 6         ; DEFGH

  508.     psrldq    m4, 4

  509.     punpcklbw m4, m7        ; EFGH

  510.     mova      m5, m4

  511.     psrldq    m5, 2         ; FGH

  512.     punpcklwd m0, m1        ; ABBCCDDE

  513.     punpcklwd m2, m3        ; CDDEEFFG

  514.     punpcklwd m4, m5        ; EFFGGHHI

  515.     pmaddwd   m0, [r5-48]

  516.     pmaddwd   m2, [r5-32]

  517.     pmaddwd   m4, [r5-16]

  518.     paddd     m0, m2

  519.     paddd     m0, m4

  520. 

  521.     psrldq    m6, 4

  522.     mova      m4, m6

  523.     punpcklbw m6, m7        ; ABCDEFGHI

  524.     mova      m1, m6

  525.     mova      m2, m6

  526.     mova      m3, m6

  527.     psrldq    m1, 2         ; BCDEFGH

  528.     psrldq    m2, 4         ; CDEFGH

  529.     psrldq    m3, 6         ; DEFGH

  530.     psrldq    m4, 4

  531.     punpcklbw m4, m7        ; EFGH

  532.     mova      m5, m4

  533.     psrldq    m5, 2         ; FGH

  534.     punpcklwd m6, m1        ; ABBCCDDE

  535.     punpcklwd m2, m3        ; CDDEEFFG

  536.     punpcklwd m4, m5        ; EFFGGHHI

  537.     pmaddwd   m6, [r5-48]

  538.     pmaddwd   m2, [r5-32]

  539.     pmaddwd   m4, [r5-16]

  540.     paddd     m6, m2

  541.     paddd     m6, m4

  542. 

  543.     packssdw  m0, m6

  544.     paddsw    m0, [pw_64]

  545.     psraw     m0, 7

  546.     packuswb  m0, m7

  547.     movh    [r0], m0        ; store

  548. 

  549.     ; go to next line

  550.     add       r0, r1

  551.     add       r2, r3

  552.     dec       r4            ; next row

  553.     jg .nextrow

  554.     REP_RET

  555. 

  556. %macro FILTER_V 3

  557. ; 4x4 block, V-only 4-tap filter

  558. cglobal put_vp8_epel%2_v4_%1, 7, 7, %3

  559.     shl      r6d, 5

  560. %ifdef PIC

  561.     lea      r11, [fourtap_filter_v_m]

  562. %endif

  563.     lea       r6, [fourtap_filter_v+r6-32]

  564.     mova      m6, [pw_64]

  565.     pxor      m7, m7

  566.     mova      m5, [r6+48]

  567. 

  568.     ; read 3 lines

  569.     sub       r2, r3

  570.     movh      m0, [r2]

  571.     movh      m1, [r2+  r3]

  572.     movh      m2, [r2+2*r3]

  573.     add       r2, r3

  574.     punpcklbw m0, m7

  575.     punpcklbw m1, m7

  576.     punpcklbw m2, m7

  577. 

  578. .nextrow

  579.     ; first calculate negative taps (to prevent losing positive overflows)

  580.     movh      m4, [r2+2*r3]                ; read new row

  581.     punpcklbw m4, m7

  582.     mova      m3, m4

  583.     pmullw    m0, [r6+0]

  584.     pmullw    m4, m5

  585.     paddsw    m4, m0

  586. 

  587.     ; then calculate positive taps

  588.     mova      m0, m1

  589.     pmullw    m1, [r6+16]

  590.     paddsw    m4, m1

  591.     mova      m1, m2

  592.     pmullw    m2, [r6+32]

  593.     paddsw    m4, m2

  594.     mova      m2, m3

  595. 

  596.     ; round/clip/store

  597.     paddsw    m4, m6

  598.     psraw     m4, 7

  599.     packuswb  m4, m7

  600.     movh    [r0], m4

  601. 

  602.     ; go to next line

  603.     add       r0, r1

  604.     add       r2, r3

  605.     dec       r4                           ; next row

  606.     jg .nextrow

  607.     REP_RET

  608. 

  609. 

  610. ; 4x4 block, V-only 6-tap filter

  611. cglobal put_vp8_epel%2_v6_%1, 7, 7, %3

  612.     shl      r6d, 4

  613.     lea       r6, [r6*3]

  614. %ifdef PIC

  615.     lea      r11, [sixtap_filter_v_m]

  616. %endif

  617.     lea       r6, [sixtap_filter_v+r6-96]

  618.     pxor      m7, m7

  619. 

  620.     ; read 5 lines

  621.     sub       r2, r3

  622.     sub       r2, r3

  623.     movh      m0, [r2]

  624.     movh      m1, [r2+r3]

  625.     movh      m2, [r2+r3*2]

  626.     lea       r2, [r2+r3*2]

  627.     add       r2, r3

  628.     movh      m3, [r2]

  629.     movh      m4, [r2+r3]

  630.     punpcklbw m0, m7

  631.     punpcklbw m1, m7

  632.     punpcklbw m2, m7

  633.     punpcklbw m3, m7

  634.     punpcklbw m4, m7

  635. 

  636. .nextrow

  637.     ; first calculate negative taps (to prevent losing positive overflows)

  638.     mova      m5, m1

  639.     pmullw    m5, [r6+16]

  640.     mova      m6, m4

  641.     pmullw    m6, [r6+64]

  642.     paddsw    m6, m5

  643. 

  644.     ; then calculate positive taps

  645.     movh      m5, [r2+2*r3]                ; read new row

  646.     punpcklbw m5, m7

  647.     pmullw    m0, [r6+0]

  648.     paddsw    m6, m0

  649.     mova      m0, m1

  650.     mova      m1, m2

  651.     pmullw    m2, [r6+32]

  652.     paddsw    m6, m2

  653.     mova      m2, m3

  654.     pmullw    m3, [r6+48]

  655.     paddsw    m6, m3

  656.     mova      m3, m4

  657.     mova      m4, m5

  658.     pmullw    m5, [r6+80]

  659.     paddsw    m6, m5

  660. 

  661.     ; round/clip/store

  662.     paddsw    m6, [pw_64]

  663.     psraw     m6, 7

  664.     packuswb  m6, m7

  665.     movh    [r0], m6

  666. 

  667.     ; go to next line

  668.     add       r0, r1

  669.     add       r2, r3

  670.     dec       r4                           ; next row

  671.     jg .nextrow

  672.     REP_RET

  673. %endmacro

  674. 

  675. INIT_MMX

  676. FILTER_V mmxext, 4, 0

  677. INIT_XMM

  678. FILTER_V sse2,   8, 8

  679. 

  680. %macro FILTER_BILINEAR 3

  681. cglobal put_vp8_bilinear%2_v_%1, 7,7,%3

  682.     mov      r5d, 8*16

  683.     shl      r6d, 4

  684.     sub      r5d, r6d

  685. %ifdef PIC

  686.     lea      r11, [bilinear_filter_vw_m]

  687. %endif

  688.     pxor      m6, m6

  689.     mova      m4, [bilinear_filter_vw+r5-16]

  690.     mova      m5, [bilinear_filter_vw+r6-16]

  691. .nextrow

  692.     movh      m0, [r2+r3*0]

  693.     movh      m1, [r2+r3*1]

  694.     movh      m3, [r2+r3*2]

  695.     punpcklbw m0, m6

  696.     punpcklbw m1, m6

  697.     punpcklbw m3, m6

  698.     mova      m2, m1

  699.     pmullw    m0, m4

  700.     pmullw    m1, m5

  701.     pmullw    m2, m4

  702.     pmullw    m3, m5

  703.     paddsw    m0, m1

  704.     paddsw    m2, m3

  705.     psraw     m0, 2

  706.     psraw     m2, 2

  707.     pavgw     m0, m6

  708.     pavgw     m2, m6

  709. %ifidn %1, mmxext

  710.     packuswb  m0, m0

  711.     packuswb  m2, m2

  712.     movh [r0+r1*0], m0

  713.     movh [r0+r1*1], m2

  714. %else

  715.     packuswb  m0, m2

  716.     movh   [r0+r1*0], m0

  717.     movhps [r0+r1*1], m0

  718. %endif

  719. 

  720.     lea       r0, [r0+r1*2]

  721.     lea       r2, [r2+r3*2]

  722.     sub       r4, 2

  723.     jg .nextrow

  724.     REP_RET

  725. 

  726. cglobal put_vp8_bilinear%2_h_%1, 7,7,%3

  727.     mov      r6d, 8*16

  728.     shl      r5d, 4

  729.     sub      r6d, r5d

  730. %ifdef PIC

  731.     lea      r11, [bilinear_filter_vw_m]

  732. %endif

  733.     pxor      m6, m6

  734.     mova      m4, [bilinear_filter_vw+r6-16]

  735.     mova      m5, [bilinear_filter_vw+r5-16]

  736. .nextrow

  737.     movh      m0, [r2+r3*0+0]

  738.     movh      m1, [r2+r3*0+1]

  739.     movh      m2, [r2+r3*1+0]

  740.     movh      m3, [r2+r3*1+1]

  741.     punpcklbw m0, m6

  742.     punpcklbw m1, m6

  743.     punpcklbw m2, m6

  744.     punpcklbw m3, m6

  745.     pmullw    m0, m4

  746.     pmullw    m1, m5

  747.     pmullw    m2, m4

  748.     pmullw    m3, m5

  749.     paddsw    m0, m1

  750.     paddsw    m2, m3

  751.     psraw     m0, 2

  752.     psraw     m2, 2

  753.     pavgw     m0, m6

  754.     pavgw     m2, m6

  755. %ifidn %1, mmxext

  756.     packuswb  m0, m0

  757.     packuswb  m2, m2

  758.     movh [r0+r1*0], m0

  759.     movh [r0+r1*1], m2

  760. %else

  761.     packuswb  m0, m2

  762.     movh   [r0+r1*0], m0

  763.     movhps [r0+r1*1], m0

  764. %endif

  765. 

  766.     lea       r0, [r0+r1*2]

  767.     lea       r2, [r2+r3*2]

  768.     sub       r4, 2

  769.     jg .nextrow

  770.     REP_RET

  771. %endmacro

  772. 

  773. INIT_MMX

  774. FILTER_BILINEAR mmxext, 4, 0

  775. INIT_XMM

  776. FILTER_BILINEAR   sse2, 8, 7

  777. 

  778. %macro FILTER_BILINEAR_SSSE3 1

  779. cglobal put_vp8_bilinear%1_v_ssse3, 7,7

  780.     shl      r6d, 4

  781. %ifdef PIC

  782.     lea      r11, [bilinear_filter_vb_m]

  783. %endif

  784.     pxor      m4, m4

  785.     mova      m3, [bilinear_filter_vb+r6-16]

  786. .nextrow

  787.     movh      m0, [r2+r3*0]

  788.     movh      m1, [r2+r3*1]

  789.     movh      m2, [r2+r3*2]

  790.     punpcklbw m0, m1

  791.     punpcklbw m1, m2

  792.     pmaddubsw m0, m3

  793.     pmaddubsw m1, m3

  794.     psraw     m0, 2

  795.     psraw     m1, 2

  796.     pavgw     m0, m4

  797.     pavgw     m1, m4

  798. %if mmsize==8

  799.     packuswb  m0, m0

  800.     packuswb  m1, m1

  801.     movh [r0+r1*0], m0

  802.     movh [r0+r1*1], m1

  803. %else

  804.     packuswb  m0, m1

  805.     movh   [r0+r1*0], m0

  806.     movhps [r0+r1*1], m0

  807. %endif

  808. 

  809.     lea       r0, [r0+r1*2]

  810.     lea       r2, [r2+r3*2]

  811.     sub       r4, 2

  812.     jg .nextrow

  813.     REP_RET

  814. 

  815. cglobal put_vp8_bilinear%1_h_ssse3, 7,7

  816.     shl      r5d, 4

  817. %ifdef PIC

  818.     lea      r11, [bilinear_filter_vb_m]

  819. %endif

  820.     pxor      m4, m4

  821.     mova      m2, [filter_h2_shuf]

  822.     mova      m3, [bilinear_filter_vb+r5-16]

  823. .nextrow

  824.     movu      m0, [r2+r3*0]

  825.     movu      m1, [r2+r3*1]

  826.     pshufb    m0, m2

  827.     pshufb    m1, m2

  828.     pmaddubsw m0, m3

  829.     pmaddubsw m1, m3

  830.     psraw     m0, 2

  831.     psraw     m1, 2

  832.     pavgw     m0, m4

  833.     pavgw     m1, m4

  834. %if mmsize==8

  835.     packuswb  m0, m0

  836.     packuswb  m1, m1

  837.     movh [r0+r1*0], m0

  838.     movh [r0+r1*1], m1

  839. %else

  840.     packuswb  m0, m1

  841.     movh   [r0+r1*0], m0

  842.     movhps [r0+r1*1], m0

  843. %endif

  844. 

  845.     lea       r0, [r0+r1*2]

  846.     lea       r2, [r2+r3*2]

  847.     sub       r4, 2

  848.     jg .nextrow

  849.     REP_RET

  850. %endmacro

  851. 

  852. INIT_MMX

  853. FILTER_BILINEAR_SSSE3 4

  854. INIT_XMM

  855. FILTER_BILINEAR_SSSE3 8

  856. 

  857. cglobal put_vp8_pixels8_mmx, 5,5

  858. .nextrow:

  859.     movq  mm0, [r2+r3*0]

  860.     movq  mm1, [r2+r3*1]

  861.     lea    r2, [r2+r3*2]

  862.     movq [r0+r1*0], mm0

  863.     movq [r0+r1*1], mm1

  864.     lea    r0, [r0+r1*2]

  865.     sub   r4d, 2

  866.     jg .nextrow

  867.     REP_RET

  868. 

  869. cglobal put_vp8_pixels16_mmx, 5,5

  870. .nextrow:

  871.     movq  mm0, [r2+r3*0+0]

  872.     movq  mm1, [r2+r3*0+8]

  873.     movq  mm2, [r2+r3*1+0]

  874.     movq  mm3, [r2+r3*1+8]

  875.     lea    r2, [r2+r3*2]

  876.     movq [r0+r1*0+0], mm0

  877.     movq [r0+r1*0+8], mm1

  878.     movq [r0+r1*1+0], mm2

  879.     movq [r0+r1*1+8], mm3

  880.     lea    r0, [r0+r1*2]

  881.     sub   r4d, 2

  882.     jg .nextrow

  883.     REP_RET

  884. 

  885. cglobal put_vp8_pixels16_sse, 5,5,2

  886. .nextrow:

  887.     movups xmm0, [r2+r3*0]

  888.     movups xmm1, [r2+r3*1]

  889.     lea     r2, [r2+r3*2]

  890.     movaps [r0+r1*0], xmm0

  891.     movaps [r0+r1*1], xmm1

  892.     lea     r0, [r0+r1*2]

  893.     sub    r4d, 2

  894.     jg .nextrow

  895.     REP_RET

  896. 

  897. ;-----------------------------------------------------------------------------

  898. ; IDCT functions:

  899. ;

  900. ; void vp8_idct_dc_add_<opt>(uint8_t *dst, DCTELEM block[16], int stride);

  901. ;-----------------------------------------------------------------------------

  902. 

  903. cglobal vp8_idct_dc_add_mmx, 3, 3

  904.     ; load data

  905.     movd       mm0, [r1]

  906. 

  907.     ; calculate DC

  908.     paddw      mm0, [pw_4]

  909.     pxor       mm1, mm1

  910.     psraw      mm0, 3

  911.     psubw      mm1, mm0

  912.     packuswb   mm0, mm0

  913.     packuswb   mm1, mm1

  914.     punpcklbw  mm0, mm0

  915.     punpcklbw  mm1, mm1

  916.     punpcklwd  mm0, mm0

  917.     punpcklwd  mm1, mm1

  918. 

  919.     ; add DC

  920.     lea         r1, [r0+r2*2]

  921.     movd       mm2, [r0]

  922.     movd       mm3, [r0+r2]

  923.     movd       mm4, [r1]

  924.     movd       mm5, [r1+r2]

  925.     paddusb    mm2, mm0

  926.     paddusb    mm3, mm0

  927.     paddusb    mm4, mm0

  928.     paddusb    mm5, mm0

  929.     psubusb    mm2, mm1

  930.     psubusb    mm3, mm1

  931.     psubusb    mm4, mm1

  932.     psubusb    mm5, mm1

  933.     movd      [r0], mm2

  934.     movd   [r0+r2], mm3

  935.     movd      [r1], mm4

  936.     movd   [r1+r2], mm5

  937.     RET

  938. 

  939. cglobal vp8_idct_dc_add_sse4, 3, 3, 6

  940.     ; load data

  941.     movd       xmm0, [r1]

  942.     lea          r1, [r0+r2*2]

  943.     pxor       xmm1, xmm1

  944.     movq       xmm2, [pw_4]

  945. 

  946.     ; calculate DC

  947.     paddw      xmm0, xmm2

  948.     movd       xmm2, [r0]

  949.     movd       xmm3, [r0+r2]

  950.     movd       xmm4, [r1]

  951.     movd       xmm5, [r1+r2]

  952.     psraw      xmm0, 3

  953.     pshuflw    xmm0, xmm0, 0

  954.     punpcklqdq xmm0, xmm0

  955.     punpckldq  xmm2, xmm3

  956.     punpckldq  xmm4, xmm5

  957.     punpcklbw  xmm2, xmm1

  958.     punpcklbw  xmm4, xmm1

  959.     paddw      xmm2, xmm0

  960.     paddw      xmm4, xmm0

  961.     packuswb   xmm2, xmm4

  962.     movd       [r0], xmm2

  963.     pextrd  [r0+r2], xmm2, 1

  964.     pextrd     [r1], xmm2, 2

  965.     pextrd  [r1+r2], xmm2, 3

  966.     RET

  967. 

  968. ;-----------------------------------------------------------------------------

  969. ; void vp8_idct_add_<opt>(uint8_t *dst, DCTELEM block[16], int stride);

  970. ;-----------------------------------------------------------------------------

  971. 

  972. ; calculate %1=mul_35468(%1)-mul_20091(%2); %2=mul_20091(%1)+mul_35468(%2)

  973. ;           this macro assumes that m6/m7 have words for 20091/17734 loaded

  974. %macro VP8_MULTIPLY_SUMSUB 4

  975.     mova      %3, %1

  976.     mova      %4, %2

  977.     pmulhw    %3, m6 ;20091(1)

  978.     pmulhw    %4, m6 ;20091(2)

  979.     paddw     %3, %1

  980.     paddw     %4, %2

  981.     paddw     %1, %1

  982.     paddw     %2, %2

  983.     pmulhw    %1, m7 ;35468(1)

  984.     pmulhw    %2, m7 ;35468(2)

  985.     psubw     %1, %4

  986.     paddw     %2, %3

  987. %endmacro

  988. 

  989. ; calculate x0=%1+%3; x1=%1-%3

  990. ;           x2=mul_35468(%2)-mul_20091(%4); x3=mul_20091(%2)+mul_35468(%4)

  991. ;           %1=x0+x3 (tmp0); %2=x1+x2 (tmp1); %3=x1-x2 (tmp2); %4=x0-x3 (tmp3)

  992. ;           %5/%6 are temporary registers

  993. ;           we assume m6/m7 have constant words 20091/17734 loaded in them

  994. %macro VP8_IDCT_TRANSFORM4x4_1D 6

  995.     SUMSUB_BA           m%3, m%1, m%5     ;t0, t1

  996.     VP8_MULTIPLY_SUMSUB m%2, m%4, m%5,m%6 ;t2, t3

  997.     SUMSUB_BA           m%4, m%3, m%5     ;tmp0, tmp3

  998.     SUMSUB_BA           m%2, m%1, m%5     ;tmp1, tmp2

  999.     SWAP                 %4,  %1

  1000.     SWAP                 %4,  %3

  1001. %endmacro

  1002. 

  1003. INIT_MMX

  1004. cglobal vp8_idct_add_mmx, 3, 3

  1005.     ; load block data

  1006.     movq         m0, [r1]

  1007.     movq         m1, [r1+8]

  1008.     movq         m2, [r1+16]

  1009.     movq         m3, [r1+24]

  1010.     movq         m6, [pw_20091]

  1011.     movq         m7, [pw_17734]

  1012. 

  1013.     ; actual IDCT

  1014.     VP8_IDCT_TRANSFORM4x4_1D 0, 1, 2, 3, 4, 5

  1015.     TRANSPOSE4x4W            0, 1, 2, 3, 4

  1016.     paddw        m0, [pw_4]

  1017.     VP8_IDCT_TRANSFORM4x4_1D 0, 1, 2, 3, 4, 5

  1018.     TRANSPOSE4x4W            0, 1, 2, 3, 4

  1019. 

  1020.     ; store

  1021.     pxor         m4, m4

  1022.     lea          r1, [r0+2*r2]

  1023.     STORE_DIFFx2 m0, m1, m6, m7, m4, 3, r0, r2

  1024.     STORE_DIFFx2 m2, m3, m6, m7, m4, 3, r1, r2

  1025. 

  1026.     RET

  1027. 

  1028. ;-----------------------------------------------------------------------------

  1029. ; void vp8_luma_dc_wht_mmxext(DCTELEM block[4][4][16], DCTELEM dc[16])

  1030. ;-----------------------------------------------------------------------------

  1031. 

  1032. %macro SCATTER_WHT 1

  1033.     pextrw r1d, m0, %1

  1034.     pextrw r2d, m1, %1

  1035.     mov [r0+2*16*0], r1w

  1036.     mov [r0+2*16*1], r2w

  1037.     pextrw r1d, m2, %1

  1038.     pextrw r2d, m3, %1

  1039.     mov [r0+2*16*2], r1w

  1040.     mov [r0+2*16*3], r2w

  1041. %endmacro

  1042. 

  1043. %macro HADAMARD4_1D 4

  1044.     SUMSUB_BADC m%2, m%1, m%4, m%3

  1045.     SUMSUB_BADC m%4, m%2, m%3, m%1

  1046.     SWAP %1, %4, %3

  1047. %endmacro

  1048. 

  1049. INIT_MMX

  1050. cglobal vp8_luma_dc_wht_mmxext, 2,3

  1051.     movq          m0, [r1]

  1052.     movq          m1, [r1+8]

  1053.     movq          m2, [r1+16]

  1054.     movq          m3, [r1+24]

  1055.     HADAMARD4_1D  0, 1, 2, 3

  1056.     TRANSPOSE4x4W 0, 1, 2, 3, 4

  1057.     paddw         m0, [pw_3]

  1058.     HADAMARD4_1D  0, 1, 2, 3

  1059.     psraw         m0, 3

  1060.     psraw         m1, 3

  1061.     psraw         m2, 3

  1062.     psraw         m3, 3

  1063.     SCATTER_WHT   0

  1064.     add           r0, 2*16*4

  1065.     SCATTER_WHT   1

  1066.     add           r0, 2*16*4

  1067.     SCATTER_WHT   2

  1068.     add           r0, 2*16*4

  1069.     SCATTER_WHT   3

  1070.     RET

  1071. 

  1072. ;-----------------------------------------------------------------------------

  1073. ; void vp8_h/v_loop_filter_simple_<opt>(uint8_t *dst, int stride, int flim);

  1074. ;-----------------------------------------------------------------------------

  1075. 

  1076. ; macro called with 7 mm register indexes as argument, and 4 regular registers

  1077. ;

  1078. ; first 4 mm registers will carry the transposed pixel data

  1079. ; the other three are scratchspace (one would be sufficient, but this allows

  1080. ; for more spreading/pipelining and thus faster execution on OOE CPUs)

  1081. ;

  1082. ; first two regular registers are buf+4*stride and buf+5*stride

  1083. ; third is -stride, fourth is +stride

  1084. %macro READ_8x4_INTERLEAVED 11

  1085.     ; interleave 8 (A-H) rows of 4 pixels each

  1086.     movd          m%1, [%8+%10*4]   ; A0-3

  1087.     movd          m%5, [%9+%10*4]   ; B0-3

  1088.     movd          m%2, [%8+%10*2]   ; C0-3

  1089.     movd          m%6, [%8+%10]     ; D0-3

  1090.     movd          m%3, [%8]         ; E0-3

  1091.     movd          m%7, [%9]         ; F0-3

  1092.     movd          m%4, [%9+%11]     ; G0-3

  1093.     punpcklbw     m%1, m%5          ; A/B interleaved

  1094.     movd          m%5, [%9+%11*2]   ; H0-3

  1095.     punpcklbw     m%2, m%6          ; C/D interleaved

  1096.     punpcklbw     m%3, m%7          ; E/F interleaved

  1097.     punpcklbw     m%4, m%5          ; G/H interleaved

  1098. %endmacro

  1099. 

  1100. ; macro called with 7 mm register indexes as argument, and 5 regular registers

  1101. ; first 11 mean the same as READ_8x4_TRANSPOSED above

  1102. ; fifth regular register is scratchspace to reach the bottom 8 rows, it

  1103. ; will be set to second regular register + 8*stride at the end

  1104. %macro READ_16x4_INTERLEAVED 12

  1105.     ; transpose 16 (A-P) rows of 4 pixels each

  1106.     lea           %12, [r0+8*r2]

  1107. 

  1108.     ; read (and interleave) those addressable by %8 (=r0), A/C/D/E/I/K/L/M

  1109.     movd          m%1, [%8+%10*4]   ; A0-3

  1110.     movd          m%3, [%12+%10*4]  ; I0-3

  1111.     movd          m%2, [%8+%10*2]   ; C0-3

  1112.     movd          m%4, [%12+%10*2]  ; K0-3

  1113.     movd          m%6, [%8+%10]     ; D0-3

  1114.     movd          m%5, [%12+%10]    ; L0-3

  1115.     movd          m%7, [%12]        ; M0-3

  1116.     add           %12, %11

  1117.     punpcklbw     m%1, m%3          ; A/I

  1118.     movd          m%3, [%8]         ; E0-3

  1119.     punpcklbw     m%2, m%4          ; C/K

  1120.     punpcklbw     m%6, m%5          ; D/L

  1121.     punpcklbw     m%3, m%7          ; E/M

  1122.     punpcklbw     m%2, m%6          ; C/D/K/L interleaved

  1123. 

  1124.     ; read (and interleave) those addressable by %9 (=r4), B/F/G/H/J/N/O/P

  1125.     movd         m%5, [%9+%10*4]   ; B0-3

  1126.     movd         m%4, [%12+%10*4]  ; J0-3

  1127.     movd         m%7, [%9]         ; F0-3

  1128.     movd         m%6, [%12]        ; N0-3

  1129.     punpcklbw    m%5, m%4          ; B/J

  1130.     punpcklbw    m%7, m%6          ; F/N

  1131.     punpcklbw    m%1, m%5          ; A/B/I/J interleaved

  1132.     punpcklbw    m%3, m%7          ; E/F/M/N interleaved

  1133.     movd         m%4, [%9+%11]     ; G0-3

  1134.     movd         m%6, [%12+%11]    ; O0-3

  1135.     movd         m%5, [%9+%11*2]   ; H0-3

  1136.     movd         m%7, [%12+%11*2]  ; P0-3

  1137.     punpcklbw    m%4, m%6          ; G/O

  1138.     punpcklbw    m%5, m%7          ; H/P

  1139.     punpcklbw    m%4, m%5          ; G/H/O/P interleaved

  1140. %endmacro

  1141. 

  1142. ; write 4 mm registers of 2 dwords each

  1143. ; first four arguments are mm register indexes containing source data

  1144. ; last four are registers containing buf+4*stride, buf+5*stride,

  1145. ; -stride and +stride

  1146. %macro WRITE_4x2D 8

  1147.     ; write out (2 dwords per register)

  1148.     movd    [%5+%7*4], m%1

  1149.     movd    [%5+%7*2], m%2

  1150.     movd         [%5], m%3

  1151.     movd      [%6+%8], m%4

  1152.     punpckhdq     m%1, m%1

  1153.     punpckhdq     m%2, m%2

  1154.     punpckhdq     m%3, m%3

  1155.     punpckhdq     m%4, m%4

  1156.     movd    [%6+%7*4], m%1

  1157.     movd      [%5+%7], m%2

  1158.     movd         [%6], m%3

  1159.     movd    [%6+%8*2], m%4

  1160. %endmacro

  1161. 

  1162. ; write 4 xmm registers of 4 dwords each

  1163. ; arguments same as WRITE_2x4D, but with an extra register, so that the 5 regular

  1164. ; registers contain buf+4*stride, buf+5*stride, buf+12*stride, -stride and +stride

  1165. ; we add 1*stride to the third regular registry in the process

  1166. %macro WRITE_4x4D 9

  1167.     ; write out (4 dwords per register), start with dwords zero

  1168.     movd    [%5+%8*4], m%1

  1169.     movd         [%5], m%2

  1170.     movd    [%5+%9*4], m%3

  1171.     movd    [%5+%9*8], m%4

  1172. 

  1173.     ; store dwords 1

  1174.     psrldq        m%1, 4

  1175.     psrldq        m%2, 4

  1176.     psrldq        m%3, 4

  1177.     psrldq        m%4, 4

  1178.     movd    [%6+%8*4], m%1

  1179.     movd         [%6], m%2

  1180.     movd    [%6+%9*4], m%3

  1181.     movd    [%6+%9*8], m%4

  1182. 

  1183.     ; write dwords 2

  1184.     psrldq        m%1, 4

  1185.     psrldq        m%2, 4

  1186.     psrldq        m%3, 4

  1187.     psrldq        m%4, 4

  1188.     movd    [%5+%8*2], m%1

  1189.     movd      [%6+%9], m%2

  1190.     movd    [%7+%8*2], m%3

  1191.     movd    [%7+%9*2], m%4

  1192.     add            %7, %9

  1193. 

  1194.     ; store dwords 3

  1195.     psrldq        m%1, 4

  1196.     psrldq        m%2, 4

  1197.     psrldq        m%3, 4

  1198.     psrldq        m%4, 4

  1199.     movd      [%5+%8], m%1

  1200.     movd    [%6+%9*2], m%2

  1201.     movd    [%7+%8*2], m%3

  1202.     movd    [%7+%9*2], m%4

  1203. %endmacro

  1204. 

  1205. %macro SIMPLE_LOOPFILTER 3

  1206. cglobal vp8_%2_loop_filter_simple_%1, 3, %3

  1207. %ifidn %2, h

  1208.     mov            r5, rsp          ; backup stack pointer

  1209.     and           rsp, ~(mmsize-1)  ; align stack

  1210. %endif

  1211. %if mmsize == 8 ; mmx/mmxext

  1212.     mov            r3, 2

  1213. %endif

  1214. 

  1215.     ; splat register with "flim"

  1216.     movd           m7, r2

  1217.     punpcklbw      m7, m7

  1218. %if mmsize == 16 ; sse2

  1219.     punpcklwd      m7, m7

  1220.     pshufd         m7, m7, 0x0

  1221. %elifidn %1, mmx

  1222.     punpcklwd      m7, m7

  1223.     punpckldq      m7, m7

  1224. %else ; mmxext

  1225.     pshufw         m7, m7, 0x0

  1226. %endif

  1227. 

  1228.     ; set up indexes to address 4 rows

  1229.     mov            r2, r1

  1230.     neg            r1

  1231. %ifidn %2, h

  1232.     lea            r0, [r0+4*r2-2]

  1233.     sub           rsp, mmsize*2     ; (aligned) storage space for saving p1/q1

  1234. %endif

  1235. 

  1236. %if mmsize == 8 ; mmx / mmxext

  1237. .next8px

  1238. %endif

  1239. %ifidn %2, v

  1240.     ; read 4 half/full rows of pixels

  1241.     mova           m0, [r0+r1*2]    ; p1

  1242.     mova           m1, [r0+r1]      ; p0

  1243.     mova           m2, [r0]         ; q0

  1244.     mova           m3, [r0+r2]      ; q1

  1245. %else ; h

  1246.     lea            r4, [r0+r2]

  1247. 

  1248. %if mmsize == 8 ; mmx/mmxext

  1249.     READ_8x4_INTERLEAVED  0, 1, 2, 3, 4, 5, 6, r0, r4, r1, r2

  1250. %else ; sse2

  1251.     READ_16x4_INTERLEAVED 0, 1, 2, 3, 4, 5, 6, r0, r4, r1, r2, r3

  1252. %endif

  1253.     TRANSPOSE4x4W         0, 1, 2, 3, 4

  1254. 

  1255.     mova        [rsp], m0           ; store p1

  1256.     mova [rsp+mmsize], m3           ; store q1

  1257. %endif

  1258. 

  1259.     ; simple_limit

  1260.     mova           m5, m2           ; m5=backup of q0

  1261.     mova           m6, m1           ; m6=backup of p0

  1262.     psubusb        m1, m2           ; p0-q0

  1263.     psubusb        m2, m6           ; q0-p0

  1264.     por            m1, m2           ; FFABS(p0-q0)

  1265.     paddusb        m1, m1           ; m1=FFABS(p0-q0)*2

  1266. 

  1267.     mova           m4, m3

  1268.     mova           m2, m0

  1269.     psubusb        m3, m0           ; q1-p1

  1270.     psubusb        m0, m4           ; p1-q1

  1271.     por            m3, m0           ; FFABS(p1-q1)

  1272.     mova           m0, [pb_80]

  1273.     pxor           m2, m0

  1274.     pxor           m4, m0

  1275.     psubsb         m2, m4           ; m2=p1-q1 (signed) backup for below

  1276.     pand           m3, [pb_FE]

  1277.     psrlq          m3, 1            ; m3=FFABS(p1-q1)/2, this can be used signed

  1278.     paddusb        m3, m1

  1279.     psubusb        m3, m7

  1280.     pxor           m1, m1

  1281.     pcmpeqb        m3, m1           ; abs(p0-q0)*2+abs(p1-q1)/2<=flim mask(0xff/0x0)

  1282. 

  1283.     ; filter_common (use m2/p1-q1, m4=q0, m6=p0, m5/q0-p0 and m3/mask)

  1284.     mova           m4, m5

  1285.     pxor           m5, m0

  1286.     pxor           m0, m6

  1287.     psubsb         m5, m0           ; q0-p0 (signed)

  1288.     paddsb         m2, m5

  1289.     paddsb         m2, m5

  1290.     paddsb         m2, m5           ; a=(p1-q1) + 3*(q0-p0)

  1291.     pand           m2, m3           ; apply filter mask (m3)

  1292. 

  1293.     mova           m3, [pb_F8]

  1294.     mova           m1, m2

  1295.     paddsb         m2, [pb_4]       ; f1<<3=a+4

  1296.     paddsb         m1, [pb_3]       ; f2<<3=a+3

  1297.     pand           m2, m3

  1298.     pand           m1, m3           ; cache f2<<3

  1299. 

  1300.     pxor           m0, m0

  1301.     pxor           m3, m3

  1302.     pcmpgtb        m0, m2           ; which values are <0?

  1303.     psubb          m3, m2           ; -f1<<3

  1304.     psrlq          m2, 3            ; +f1

  1305.     psrlq          m3, 3            ; -f1

  1306.     pand           m3, m0

  1307.     pandn          m0, m2

  1308.     psubusb        m4, m0

  1309.     paddusb        m4, m3           ; q0-f1

  1310. 

  1311.     pxor           m0, m0

  1312.     pxor           m3, m3

  1313.     pcmpgtb        m0, m1           ; which values are <0?

  1314.     psubb          m3, m1           ; -f2<<3

  1315.     psrlq          m1, 3            ; +f2

  1316.     psrlq          m3, 3            ; -f2

  1317.     pand           m3, m0

  1318.     pandn          m0, m1

  1319.     paddusb        m6, m0

  1320.     psubusb        m6, m3           ; p0+f2

  1321. 

  1322.     ; store

  1323. %ifidn %2, v

  1324.     mova         [r0], m4

  1325.     mova      [r0+r1], m6

  1326. %else ; h

  1327.     mova           m0, [rsp]        ; p1

  1328.     SWAP            2, 4            ; p0

  1329.     SWAP            1, 6            ; q0

  1330.     mova           m3, [rsp+mmsize] ; q1

  1331. 

  1332.     TRANSPOSE4x4B  0, 1, 2, 3, 4

  1333. %if mmsize == 16 ; sse2

  1334.     add            r3, r1           ; change from r4*8*stride to r0+8*stride

  1335.     WRITE_4x4D 0, 1, 2, 3, r0, r4, r3, r1, r2

  1336. %else ; mmx/mmxext

  1337.     WRITE_4x2D 0, 1, 2, 3, r0, r4, r1, r2

  1338. %endif

  1339. %endif

  1340. 

  1341. %if mmsize == 8 ; mmx/mmxext

  1342.     ; next 8 pixels

  1343. %ifidn %2, v

  1344.     add            r0, 8            ; advance 8 cols = pixels

  1345. %else ; h

  1346.     lea            r0, [r0+r2*8]    ; advance 8 rows = lines

  1347. %endif

  1348.     dec            r3

  1349.     jg .next8px

  1350. %ifidn %2, v

  1351.     REP_RET

  1352. %else ; h

  1353.     mov           rsp, r5           ; restore stack pointer

  1354.     RET

  1355. %endif

  1356. %else ; sse2

  1357. %ifidn %2, h

  1358.     mov           rsp, r5           ; restore stack pointer

  1359. %endif

  1360.     RET

  1361. %endif

  1362. %endmacro

  1363. 

  1364. INIT_MMX

  1365. SIMPLE_LOOPFILTER mmx,    v, 4

  1366. SIMPLE_LOOPFILTER mmx,    h, 6

  1367. SIMPLE_LOOPFILTER mmxext, v, 4

  1368. SIMPLE_LOOPFILTER mmxext, h, 6

  1369. INIT_XMM

  1370. SIMPLE_LOOPFILTER sse2,   v, 3

  1371. SIMPLE_LOOPFILTER sse2,   h, 6