falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
43714 Commits
32 Branches
1.0GB
Tree: 6c180b35c4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '6c180b35c4'
${ noResults }
FFmpeg/libavcodec/x86/fft_mmx.asm

1115 lines
26KB
Raw Blame History 

  1. ;******************************************************************************

  2. ;* FFT transform with SSE/3DNow optimizations

  3. ;* Copyright (c) 2008 Loren Merritt

  4. ;* Copyright (c) 2011 Vitor Sessak

  5. ;*

  6. ;* This algorithm (though not any of the implementation details) is

  7. ;* based on libdjbfft by D. J. Bernstein.

  8. ;*

  9. ;* This file is part of FFmpeg.

  10. ;*

  11. ;* FFmpeg is free software; you can redistribute it and/or

  12. ;* modify it under the terms of the GNU Lesser General Public

  13. ;* License as published by the Free Software Foundation; either

  14. ;* version 2.1 of the License, or (at your option) any later version.

  15. ;*

  16. ;* FFmpeg is distributed in the hope that it will be useful,

  17. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  18. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  19. ;* Lesser General Public License for more details.

  20. ;*

  21. ;* You should have received a copy of the GNU Lesser General Public

  22. ;* License along with FFmpeg; if not, write to the Free Software

  23. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  24. ;******************************************************************************

  25. 

  26. ; These functions are not individually interchangeable with the C versions.

  27. ; While C takes arrays of FFTComplex, SSE/3DNow leave intermediate results

  28. ; in blocks as conventient to the vector size.

  29. ; i.e. {4x real, 4x imaginary, 4x real, ...} (or 2x respectively)

  30. 

  31. %include "libavutil/x86/x86inc.asm"

  32. %include "libavutil/x86/x86util.asm"

  33. 

  34. %if ARCH_X86_64

  35. %define pointer resq

  36. %else

  37. %define pointer resd

  38. %endif

  39. 

  40. SECTION_RODATA

  41. 

  42. struc FFTContext

  43.     .nbits:    resd 1

  44.     .reverse:  resd 1

  45.     .revtab:   pointer 1

  46.     .tmpbuf:   pointer 1

  47.     .mdctsize: resd 1

  48.     .mdctbits: resd 1

  49.     .tcos:     pointer 1

  50.     .tsin:     pointer 1

  51.     .fftperm:  pointer 1

  52.     .fftcalc:  pointer 1

  53.     .imdctcalc:pointer 1

  54.     .imdcthalf:pointer 1

  55. endstruc

  56. 

  57. %define M_SQRT1_2 0.70710678118654752440

  58. %define M_COS_PI_1_8 0.923879532511287

  59. %define M_COS_PI_3_8 0.38268343236509

  60. 

  61. align 32

  62. ps_cos16_1: dd 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8, 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8

  63. ps_cos16_2: dd 0, M_COS_PI_3_8, M_SQRT1_2, M_COS_PI_1_8, 0, -M_COS_PI_3_8, -M_SQRT1_2, -M_COS_PI_1_8

  64. 

  65. ps_root2: times 8 dd M_SQRT1_2

  66. ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2

  67. ps_p1p1m1p1: dd 0, 0, 1<<31, 0, 0, 0, 1<<31, 0

  68. 

  69. perm1: dd 0x00, 0x02, 0x03, 0x01, 0x03, 0x00, 0x02, 0x01

  70. perm2: dd 0x00, 0x01, 0x02, 0x03, 0x01, 0x00, 0x02, 0x03

  71. ps_p1p1m1p1root2: dd 1.0, 1.0, -1.0, 1.0, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2

  72. ps_m1m1p1m1p1m1m1m1: dd 1<<31, 1<<31, 0, 1<<31, 0, 1<<31, 1<<31, 1<<31

  73. ps_m1m1m1m1: times 4 dd 1<<31

  74. ps_m1p1: dd 1<<31, 0

  75. 

  76. %assign i 16

  77. %rep 13

  78. cextern cos_ %+ i

  79. %assign i i<<1

  80. %endrep

  81. 

  82. %if ARCH_X86_64

  83.     %define pointer dq

  84. %else

  85.     %define pointer dd

  86. %endif

  87. 

  88. %macro IF0 1+

  89. %endmacro

  90. %macro IF1 1+

  91.     %1

  92. %endmacro

  93. 

  94. SECTION_TEXT

  95. 

  96. %macro T2_3DNOW 4 ; z0, z1, mem0, mem1

  97.     mova     %1, %3

  98.     mova     %2, %1

  99.     pfadd    %1, %4

  100.     pfsub    %2, %4

  101. %endmacro

  102. 

  103. %macro T4_3DNOW 6 ; z0, z1, z2, z3, tmp0, tmp1

  104.     mova     %5, %3

  105.     pfsub    %3, %4

  106.     pfadd    %5, %4 ; {t6,t5}

  107.     pxor     %3, [ps_m1p1] ; {t8,t7}

  108.     mova     %6, %1

  109.     PSWAPD   %3, %3

  110.     pfadd    %1, %5 ; {r0,i0}

  111.     pfsub    %6, %5 ; {r2,i2}

  112.     mova     %4, %2

  113.     pfadd    %2, %3 ; {r1,i1}

  114.     pfsub    %4, %3 ; {r3,i3}

  115.     SWAP     %3, %6

  116. %endmacro

  117. 

  118. ;  in: %1 = {r0,i0,r2,i2,r4,i4,r6,i6}

  119. ;      %2 = {r1,i1,r3,i3,r5,i5,r7,i7}

  120. ;      %3, %4, %5 tmp

  121. ; out: %1 = {r0,r1,r2,r3,i0,i1,i2,i3}

  122. ;      %2 = {r4,r5,r6,r7,i4,i5,i6,i7}

  123. %macro T8_AVX 5

  124.     vsubps     %5, %1, %2       ; v  = %1 - %2

  125.     vaddps     %3, %1, %2       ; w  = %1 + %2

  126.     vmulps     %2, %5, [ps_p1p1m1p1root2]  ; v *= vals1

  127.     vpermilps  %2, %2, [perm1]

  128.     vblendps   %1, %2, %3, 0x33 ; q = {w1,w2,v4,v2,w5,w6,v7,v6}

  129.     vshufps    %5, %3, %2, 0x4e ; r = {w3,w4,v1,v3,w7,w8,v8,v5}

  130.     vsubps     %4, %5, %1       ; s = r - q

  131.     vaddps     %1, %5, %1       ; u = r + q

  132.     vpermilps  %1, %1, [perm2]  ; k  = {u1,u2,u3,u4,u6,u5,u7,u8}

  133.     vshufps    %5, %4, %1, 0xbb

  134.     vshufps    %3, %4, %1, 0xee

  135.     vperm2f128 %3, %3, %5, 0x13

  136.     vxorps     %4, %4, [ps_m1m1p1m1p1m1m1m1]  ; s *= {1,1,-1,-1,1,-1,-1,-1}

  137.     vshufps    %2, %1, %4, 0xdd

  138.     vshufps    %1, %1, %4, 0x88

  139.     vperm2f128 %4, %2, %1, 0x02 ; v  = {k1,k3,s1,s3,k2,k4,s2,s4}

  140.     vperm2f128 %1, %1, %2, 0x13 ; w  = {k6,k8,s6,s8,k5,k7,s5,s7}

  141.     vsubps     %5, %1, %3

  142.     vblendps   %1, %5, %1, 0x55 ; w -= {0,s7,0,k7,0,s8,0,k8}

  143.     vsubps     %2, %4, %1       ; %2 = v - w

  144.     vaddps     %1, %4, %1       ; %1 = v + w

  145. %endmacro

  146. 

  147. ; In SSE mode do one fft4 transforms

  148. ; in:  %1={r0,i0,r2,i2} %2={r1,i1,r3,i3}

  149. ; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}

  150. ;

  151. ; In AVX mode do two fft4 transforms

  152. ; in:  %1={r0,i0,r2,i2,r4,i4,r6,i6} %2={r1,i1,r3,i3,r5,i5,r7,i7}

  153. ; out: %1={r0,r1,r2,r3,r4,r5,r6,r7} %2={i0,i1,i2,i3,i4,i5,i6,i7}

  154. %macro T4_SSE 3

  155.     subps    %3, %1, %2       ; {t3,t4,-t8,t7}

  156.     addps    %1, %1, %2       ; {t1,t2,t6,t5}

  157.     xorps    %3, %3, [ps_p1p1m1p1]

  158.     shufps   %2, %1, %3, 0xbe ; {t6,t5,t7,t8}

  159.     shufps   %1, %1, %3, 0x44 ; {t1,t2,t3,t4}

  160.     subps    %3, %1, %2       ; {r2,i2,r3,i3}

  161.     addps    %1, %1, %2       ; {r0,i0,r1,i1}

  162.     shufps   %2, %1, %3, 0xdd ; {i0,i1,i2,i3}

  163.     shufps   %1, %1, %3, 0x88 ; {r0,r1,r2,r3}

  164. %endmacro

  165. 

  166. ; In SSE mode do one FFT8

  167. ; in:  %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %3={r4,i4,r6,i6} %4={r5,i5,r7,i7}

  168. ; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %1={r4,r5,r6,r7} %2={i4,i5,i6,i7}

  169. ;

  170. ; In AVX mode do two FFT8

  171. ; in:  %1={r0,i0,r2,i2,r8, i8, r10,i10} %2={r1,i1,r3,i3,r9, i9, r11,i11}

  172. ;      %3={r4,i4,r6,i6,r12,i12,r14,i14} %4={r5,i5,r7,i7,r13,i13,r15,i15}

  173. ; out: %1={r0,r1,r2,r3,r8, r9, r10,r11} %2={i0,i1,i2,i3,i8, i9, i10,i11}

  174. ;      %3={r4,r5,r6,r7,r12,r13,r14,r15} %4={i4,i5,i6,i7,i12,i13,i14,i15}

  175. %macro T8_SSE 6

  176.     addps    %6, %3, %4       ; {t1,t2,t3,t4}

  177.     subps    %3, %3, %4       ; {r5,i5,r7,i7}

  178.     shufps   %4, %3, %3, 0xb1 ; {i5,r5,i7,r7}

  179.     mulps    %3, %3, [ps_root2mppm] ; {-r5,i5,r7,-i7}

  180.     mulps    %4, %4, [ps_root2]

  181.     addps    %3, %3, %4       ; {t8,t7,ta,t9}

  182.     shufps   %4, %6, %3, 0x9c ; {t1,t4,t7,ta}

  183.     shufps   %6, %6, %3, 0x36 ; {t3,t2,t9,t8}

  184.     subps    %3, %6, %4       ; {t6,t5,tc,tb}

  185.     addps    %6, %6, %4       ; {t1,t2,t9,ta}

  186.     shufps   %5, %6, %3, 0x8d ; {t2,ta,t6,tc}

  187.     shufps   %6, %6, %3, 0xd8 ; {t1,t9,t5,tb}

  188.     subps    %3, %1, %6       ; {r4,r5,r6,r7}

  189.     addps    %1, %1, %6       ; {r0,r1,r2,r3}

  190.     subps    %4, %2, %5       ; {i4,i5,i6,i7}

  191.     addps    %2, %2, %5       ; {i0,i1,i2,i3}

  192. %endmacro

  193. 

  194. ; scheduled for cpu-bound sizes

  195. %macro PASS_SMALL 3 ; (to load m4-m7), wre, wim

  196. IF%1 mova    m4, Z(4)

  197. IF%1 mova    m5, Z(5)

  198.     mova     m0, %2 ; wre

  199.     mova     m1, %3 ; wim

  200.     mulps    m2, m4, m0 ; r2*wre

  201. IF%1 mova    m6, Z2(6)

  202.     mulps    m3, m5, m1 ; i2*wim

  203. IF%1 mova    m7, Z2(7)

  204.     mulps    m4, m4, m1 ; r2*wim

  205.     mulps    m5, m5, m0 ; i2*wre

  206.     addps    m2, m2, m3 ; r2*wre + i2*wim

  207.     mulps    m3, m1, m7 ; i3*wim

  208.     subps    m5, m5, m4 ; i2*wre - r2*wim

  209.     mulps    m1, m1, m6 ; r3*wim

  210.     mulps    m4, m0, m6 ; r3*wre

  211.     mulps    m0, m0, m7 ; i3*wre

  212.     subps    m4, m4, m3 ; r3*wre - i3*wim

  213.     mova     m3, Z(0)

  214.     addps    m0, m0, m1 ; i3*wre + r3*wim

  215.     subps    m1, m4, m2 ; t3

  216.     addps    m4, m4, m2 ; t5

  217.     subps    m3, m3, m4 ; r2

  218.     addps    m4, m4, Z(0) ; r0

  219.     mova     m6, Z(2)

  220.     mova   Z(4), m3

  221.     mova   Z(0), m4

  222.     subps    m3, m5, m0 ; t4

  223.     subps    m4, m6, m3 ; r3

  224.     addps    m3, m3, m6 ; r1

  225.     mova  Z2(6), m4

  226.     mova   Z(2), m3

  227.     mova     m2, Z(3)

  228.     addps    m3, m5, m0 ; t6

  229.     subps    m2, m2, m1 ; i3

  230.     mova     m7, Z(1)

  231.     addps    m1, m1, Z(3) ; i1

  232.     mova  Z2(7), m2

  233.     mova   Z(3), m1

  234.     subps    m4, m7, m3 ; i2

  235.     addps    m3, m3, m7 ; i0

  236.     mova   Z(5), m4

  237.     mova   Z(1), m3

  238. %endmacro

  239. 

  240. ; scheduled to avoid store->load aliasing

  241. %macro PASS_BIG 1 ; (!interleave)

  242.     mova     m4, Z(4) ; r2

  243.     mova     m5, Z(5) ; i2

  244.     mova     m0, [wq] ; wre

  245.     mova     m1, [wq+o1q] ; wim

  246.     mulps    m2, m4, m0 ; r2*wre

  247.     mova     m6, Z2(6) ; r3

  248.     mulps    m3, m5, m1 ; i2*wim

  249.     mova     m7, Z2(7) ; i3

  250.     mulps    m4, m4, m1 ; r2*wim

  251.     mulps    m5, m5, m0 ; i2*wre

  252.     addps    m2, m2, m3 ; r2*wre + i2*wim

  253.     mulps    m3, m1, m7 ; i3*wim

  254.     mulps    m1, m1, m6 ; r3*wim

  255.     subps    m5, m5, m4 ; i2*wre - r2*wim

  256.     mulps    m4, m0, m6 ; r3*wre

  257.     mulps    m0, m0, m7 ; i3*wre

  258.     subps    m4, m4, m3 ; r3*wre - i3*wim

  259.     mova     m3, Z(0)

  260.     addps    m0, m0, m1 ; i3*wre + r3*wim

  261.     subps    m1, m4, m2 ; t3

  262.     addps    m4, m4, m2 ; t5

  263.     subps    m3, m3, m4 ; r2

  264.     addps    m4, m4, Z(0) ; r0

  265.     mova     m6, Z(2)

  266.     mova   Z(4), m3

  267.     mova   Z(0), m4

  268.     subps    m3, m5, m0 ; t4

  269.     subps    m4, m6, m3 ; r3

  270.     addps    m3, m3, m6 ; r1

  271. IF%1 mova Z2(6), m4

  272. IF%1 mova  Z(2), m3

  273.     mova     m2, Z(3)

  274.     addps    m5, m5, m0 ; t6

  275.     subps    m2, m2, m1 ; i3

  276.     mova     m7, Z(1)

  277.     addps    m1, m1, Z(3) ; i1

  278. IF%1 mova Z2(7), m2

  279. IF%1 mova  Z(3), m1

  280.     subps    m6, m7, m5 ; i2

  281.     addps    m5, m5, m7 ; i0

  282. IF%1 mova  Z(5), m6

  283. IF%1 mova  Z(1), m5

  284. %if %1==0

  285.     INTERL m1, m3, m7, Z, 2

  286.     INTERL m2, m4, m0, Z2, 6

  287. 

  288.     mova     m1, Z(0)

  289.     mova     m2, Z(4)

  290. 

  291.     INTERL m5, m1, m3, Z, 0

  292.     INTERL m6, m2, m7, Z, 4

  293. %endif

  294. %endmacro

  295. 

  296. %macro PUNPCK 3

  297.     mova      %3, %1

  298.     punpckldq %1, %2

  299.     punpckhdq %3, %2

  300. %endmacro

  301. 

  302. %define Z(x) [r0+mmsize*x]

  303. %define Z2(x) [r0+mmsize*x]

  304. %define ZH(x) [r0+mmsize*x+mmsize/2]

  305. 

  306. INIT_YMM avx

  307. 

  308. %if HAVE_AVX

  309. align 16

  310. fft8_avx:

  311.     mova      m0, Z(0)

  312.     mova      m1, Z(1)

  313.     T8_AVX    m0, m1, m2, m3, m4

  314.     mova      Z(0), m0

  315.     mova      Z(1), m1

  316.     ret

  317. 

  318. 

  319. align 16

  320. fft16_avx:

  321.     mova       m2, Z(2)

  322.     mova       m3, Z(3)

  323.     T4_SSE     m2, m3, m7

  324. 

  325.     mova       m0, Z(0)

  326.     mova       m1, Z(1)

  327.     T8_AVX     m0, m1, m4, m5, m7

  328. 

  329.     mova       m4, [ps_cos16_1]

  330.     mova       m5, [ps_cos16_2]

  331.     vmulps     m6, m2, m4

  332.     vmulps     m7, m3, m5

  333.     vaddps     m7, m7, m6

  334.     vmulps     m2, m2, m5

  335.     vmulps     m3, m3, m4

  336.     vsubps     m3, m3, m2

  337.     vblendps   m2, m7, m3, 0xf0

  338.     vperm2f128 m3, m7, m3, 0x21

  339.     vaddps     m4, m2, m3

  340.     vsubps     m2, m3, m2

  341.     vperm2f128 m2, m2, m2, 0x01

  342.     vsubps     m3, m1, m2

  343.     vaddps     m1, m1, m2

  344.     vsubps     m5, m0, m4

  345.     vaddps     m0, m0, m4

  346.     vextractf128   Z(0), m0, 0

  347.     vextractf128  ZH(0), m1, 0

  348.     vextractf128   Z(1), m0, 1

  349.     vextractf128  ZH(1), m1, 1

  350.     vextractf128   Z(2), m5, 0

  351.     vextractf128  ZH(2), m3, 0

  352.     vextractf128   Z(3), m5, 1

  353.     vextractf128  ZH(3), m3, 1

  354.     ret

  355. 

  356. align 16

  357. fft32_avx:

  358.     call fft16_avx

  359. 

  360.     mova m0, Z(4)

  361.     mova m1, Z(5)

  362. 

  363.     T4_SSE      m0, m1, m4

  364. 

  365.     mova m2, Z(6)

  366.     mova m3, Z(7)

  367. 

  368.     T8_SSE      m0, m1, m2, m3, m4, m6

  369.     ; m0={r0,r1,r2,r3,r8, r9, r10,r11} m1={i0,i1,i2,i3,i8, i9, i10,i11}

  370.     ; m2={r4,r5,r6,r7,r12,r13,r14,r15} m3={i4,i5,i6,i7,i12,i13,i14,i15}

  371. 

  372.     vperm2f128  m4, m0, m2, 0x20

  373.     vperm2f128  m5, m1, m3, 0x20

  374.     vperm2f128  m6, m0, m2, 0x31

  375.     vperm2f128  m7, m1, m3, 0x31

  376. 

  377.     PASS_SMALL 0, [cos_32], [cos_32+32]

  378. 

  379.     ret

  380. 

  381. fft32_interleave_avx:

  382.     call fft32_avx

  383.     mov r2d, 32

  384. .deint_loop:

  385.     mova     m2, Z(0)

  386.     mova     m3, Z(1)

  387.     vunpcklps      m0, m2, m3

  388.     vunpckhps      m1, m2, m3

  389.     vextractf128   Z(0), m0, 0

  390.     vextractf128  ZH(0), m1, 0

  391.     vextractf128   Z(1), m0, 1

  392.     vextractf128  ZH(1), m1, 1

  393.     add r0, mmsize*2

  394.     sub r2d, mmsize/4

  395.     jg .deint_loop

  396.     ret

  397. 

  398. %endif

  399. 

  400. INIT_XMM sse

  401. 

  402. align 16

  403. fft4_avx:

  404. fft4_sse:

  405.     mova     m0, Z(0)

  406.     mova     m1, Z(1)

  407.     T4_SSE   m0, m1, m2

  408.     mova   Z(0), m0

  409.     mova   Z(1), m1

  410.     ret

  411. 

  412. align 16

  413. fft8_sse:

  414.     mova     m0, Z(0)

  415.     mova     m1, Z(1)

  416.     T4_SSE   m0, m1, m2

  417.     mova     m2, Z(2)

  418.     mova     m3, Z(3)

  419.     T8_SSE   m0, m1, m2, m3, m4, m5

  420.     mova   Z(0), m0

  421.     mova   Z(1), m1

  422.     mova   Z(2), m2

  423.     mova   Z(3), m3

  424.     ret

  425. 

  426. align 16

  427. fft16_sse:

  428.     mova     m0, Z(0)

  429.     mova     m1, Z(1)

  430.     T4_SSE   m0, m1, m2

  431.     mova     m2, Z(2)

  432.     mova     m3, Z(3)

  433.     T8_SSE   m0, m1, m2, m3, m4, m5

  434.     mova     m4, Z(4)

  435.     mova     m5, Z(5)

  436.     mova   Z(0), m0

  437.     mova   Z(1), m1

  438.     mova   Z(2), m2

  439.     mova   Z(3), m3

  440.     T4_SSE   m4, m5, m6

  441.     mova     m6, Z2(6)

  442.     mova     m7, Z2(7)

  443.     T4_SSE   m6, m7, m0

  444.     PASS_SMALL 0, [cos_16], [cos_16+16]

  445.     ret

  446. 

  447. 

  448. %macro FFT48_3DNOW 0

  449. align 16

  450. fft4 %+ SUFFIX:

  451.     T2_3DNOW m0, m1, Z(0), Z(1)

  452.     mova     m2, Z(2)

  453.     mova     m3, Z(3)

  454.     T4_3DNOW m0, m1, m2, m3, m4, m5

  455.     PUNPCK   m0, m1, m4

  456.     PUNPCK   m2, m3, m5

  457.     mova   Z(0), m0

  458.     mova   Z(1), m4

  459.     mova   Z(2), m2

  460.     mova   Z(3), m5

  461.     ret

  462. 

  463. align 16

  464. fft8 %+ SUFFIX:

  465.     T2_3DNOW m0, m1, Z(0), Z(1)

  466.     mova     m2, Z(2)

  467.     mova     m3, Z(3)

  468.     T4_3DNOW m0, m1, m2, m3, m4, m5

  469.     mova   Z(0), m0

  470.     mova   Z(2), m2

  471.     T2_3DNOW m4, m5,  Z(4),  Z(5)

  472.     T2_3DNOW m6, m7, Z2(6), Z2(7)

  473.     PSWAPD   m0, m5

  474.     PSWAPD   m2, m7

  475.     pxor     m0, [ps_m1p1]

  476.     pxor     m2, [ps_m1p1]

  477.     pfsub    m5, m0

  478.     pfadd    m7, m2

  479.     pfmul    m5, [ps_root2]

  480.     pfmul    m7, [ps_root2]

  481.     T4_3DNOW m1, m3, m5, m7, m0, m2

  482.     mova   Z(5), m5

  483.     mova  Z2(7), m7

  484.     mova     m0, Z(0)

  485.     mova     m2, Z(2)

  486.     T4_3DNOW m0, m2, m4, m6, m5, m7

  487.     PUNPCK   m0, m1, m5

  488.     PUNPCK   m2, m3, m7

  489.     mova   Z(0), m0

  490.     mova   Z(1), m5

  491.     mova   Z(2), m2

  492.     mova   Z(3), m7

  493.     PUNPCK   m4,  Z(5), m5

  494.     PUNPCK   m6, Z2(7), m7

  495.     mova   Z(4), m4

  496.     mova   Z(5), m5

  497.     mova  Z2(6), m6

  498.     mova  Z2(7), m7

  499.     ret

  500. %endmacro

  501. 

  502. %if ARCH_X86_32

  503. %macro PSWAPD 2

  504. %if cpuflag(3dnowext)

  505.     pswapd %1, %2

  506. %elifidn %1, %2

  507.     movd [r0+12], %1

  508.     punpckhdq %1, [r0+8]

  509. %else

  510.     movq  %1, %2

  511.     psrlq %1, 32

  512.     punpckldq %1, %2

  513. %endif

  514. %endmacro

  515. 

  516. INIT_MMX 3dnowext

  517. FFT48_3DNOW

  518. 

  519. INIT_MMX 3dnow

  520. FFT48_3DNOW

  521. %endif

  522. 

  523. %define Z(x) [zcq + o1q*(x&6) + mmsize*(x&1)]

  524. %define Z2(x) [zcq + o3q + mmsize*(x&1)]

  525. %define ZH(x) [zcq + o1q*(x&6) + mmsize*(x&1) + mmsize/2]

  526. %define Z2H(x) [zcq + o3q + mmsize*(x&1) + mmsize/2]

  527. 

  528. %macro DECL_PASS 2+ ; name, payload

  529. align 16

  530. %1:

  531. DEFINE_ARGS zc, w, n, o1, o3

  532.     lea o3q, [nq*3]

  533.     lea o1q, [nq*8]

  534.     shl o3q, 4

  535. .loop:

  536.     %2

  537.     add zcq, mmsize*2

  538.     add  wq, mmsize

  539.     sub  nd, mmsize/8

  540.     jg .loop

  541.     rep ret

  542. %endmacro

  543. 

  544. %macro FFT_DISPATCH 2; clobbers 5 GPRs, 8 XMMs

  545.     lea r2, [dispatch_tab%1]

  546.     mov r2, [r2 + (%2q-2)*gprsize]

  547. %ifdef PIC

  548.     lea r3, [$$]

  549.     add r2, r3

  550. %endif

  551.     call r2

  552. %endmacro ; FFT_DISPATCH

  553. 

  554. INIT_YMM avx

  555. 

  556. %if HAVE_AVX

  557. %macro INTERL_AVX 5

  558.     vunpckhps      %3, %2, %1

  559.     vunpcklps      %2, %2, %1

  560.     vextractf128   %4(%5), %2, 0

  561.     vextractf128  %4 %+ H(%5), %3, 0

  562.     vextractf128   %4(%5 + 1), %2, 1

  563.     vextractf128  %4 %+ H(%5 + 1), %3, 1

  564. %endmacro

  565. 

  566. %define INTERL INTERL_AVX

  567. 

  568. DECL_PASS pass_avx, PASS_BIG 1

  569. DECL_PASS pass_interleave_avx, PASS_BIG 0

  570. 

  571. cglobal fft_calc, 2,5,8

  572.     mov     r3d, [r0 + FFTContext.nbits]

  573.     mov     r0, r1

  574.     mov     r1, r3

  575.     FFT_DISPATCH _interleave %+ SUFFIX, r1

  576.     REP_RET

  577. 

  578. %endif

  579. 

  580. INIT_XMM sse

  581. 

  582. %macro INTERL_SSE 5

  583.     mova     %3, %2

  584.     unpcklps %2, %1

  585.     unpckhps %3, %1

  586.     mova  %4(%5), %2

  587.     mova  %4(%5+1), %3

  588. %endmacro

  589. 

  590. %define INTERL INTERL_SSE

  591. 

  592. DECL_PASS pass_sse, PASS_BIG 1

  593. DECL_PASS pass_interleave_sse, PASS_BIG 0

  594. 

  595. %macro FFT_CALC_FUNC 0

  596. cglobal fft_calc, 2,5,8

  597.     mov     r3d, [r0 + FFTContext.nbits]

  598.     PUSH    r1

  599.     PUSH    r3

  600.     mov     r0, r1

  601.     mov     r1, r3

  602.     FFT_DISPATCH _interleave %+ SUFFIX, r1

  603.     POP     rcx

  604.     POP     r4

  605.     cmp     rcx, 3+(mmsize/16)

  606.     jg      .end

  607.     mov     r2, -1

  608.     add     rcx, 3

  609.     shl     r2, cl

  610.     sub     r4, r2

  611. .loop:

  612. %if mmsize == 8

  613.     PSWAPD  m0, [r4 + r2 + 4]

  614.     mova [r4 + r2 + 4], m0

  615. %else

  616.     movaps   xmm0, [r4 + r2]

  617.     movaps   xmm1, xmm0

  618.     unpcklps xmm0, [r4 + r2 + 16]

  619.     unpckhps xmm1, [r4 + r2 + 16]

  620.     movaps   [r4 + r2],      xmm0

  621.     movaps   [r4 + r2 + 16], xmm1

  622. %endif

  623.     add      r2, mmsize*2

  624.     jl       .loop

  625. .end:

  626. %if cpuflag(3dnow)

  627.     femms

  628.     RET

  629. %else

  630.     REP_RET

  631. %endif

  632. %endmacro

  633. 

  634. %if ARCH_X86_32

  635. INIT_MMX 3dnow

  636. FFT_CALC_FUNC

  637. INIT_MMX 3dnowext

  638. FFT_CALC_FUNC

  639. %endif

  640. INIT_XMM sse

  641. FFT_CALC_FUNC

  642. 

  643. cglobal fft_permute, 2,7,1

  644.     mov     r4,  [r0 + FFTContext.revtab]

  645.     mov     r5,  [r0 + FFTContext.tmpbuf]

  646.     mov     ecx, [r0 + FFTContext.nbits]

  647.     mov     r2, 1

  648.     shl     r2, cl

  649.     xor     r0, r0

  650. %if ARCH_X86_32

  651.     mov     r1, r1m

  652. %endif

  653. .loop:

  654.     movaps  xmm0, [r1 + 8*r0]

  655.     movzx   r6, word [r4 + 2*r0]

  656.     movzx   r3, word [r4 + 2*r0 + 2]

  657.     movlps  [r5 + 8*r6], xmm0

  658.     movhps  [r5 + 8*r3], xmm0

  659.     add     r0, 2

  660.     cmp     r0, r2

  661.     jl      .loop

  662.     shl     r2, 3

  663.     add     r1, r2

  664.     add     r5, r2

  665.     neg     r2

  666. ; nbits >= 2 (FFT4) and sizeof(FFTComplex)=8 => at least 32B

  667. .loopcopy:

  668.     movaps  xmm0, [r5 + r2]

  669.     movaps  xmm1, [r5 + r2 + 16]

  670.     movaps  [r1 + r2], xmm0

  671.     movaps  [r1 + r2 + 16], xmm1

  672.     add     r2, 32

  673.     jl      .loopcopy

  674.     REP_RET

  675. 

  676. %macro IMDCT_CALC_FUNC 0

  677. cglobal imdct_calc, 3,5,3

  678.     mov     r3d, [r0 + FFTContext.mdctsize]

  679.     mov     r4,  [r0 + FFTContext.imdcthalf]

  680.     add     r1,  r3

  681.     PUSH    r3

  682.     PUSH    r1

  683. %if ARCH_X86_32

  684.     push    r2

  685.     push    r1

  686.     push    r0

  687. %else

  688.     sub     rsp, 8

  689. %endif

  690.     call    r4

  691. %if ARCH_X86_32

  692.     add     esp, 12

  693. %else

  694.     add     rsp, 8

  695. %endif

  696.     POP     r1

  697.     POP     r3

  698.     lea     r0, [r1 + 2*r3]

  699.     mov     r2, r3

  700.     sub     r3, mmsize

  701.     neg     r2

  702.     mova    m2, [ps_m1m1m1m1]

  703. .loop:

  704. %if mmsize == 8

  705.     PSWAPD  m0, [r1 + r3]

  706.     PSWAPD  m1, [r0 + r2]

  707.     pxor    m0, m2

  708. %else

  709.     mova    m0, [r1 + r3]

  710.     mova    m1, [r0 + r2]

  711.     shufps  m0, m0, 0x1b

  712.     shufps  m1, m1, 0x1b

  713.     xorps   m0, m2

  714. %endif

  715.     mova [r0 + r3], m1

  716.     mova [r1 + r2], m0

  717.     sub     r3, mmsize

  718.     add     r2, mmsize

  719.     jl      .loop

  720. %if cpuflag(3dnow)

  721.     femms

  722.     RET

  723. %else

  724.     REP_RET

  725. %endif

  726. %endmacro

  727. 

  728. %if ARCH_X86_32

  729. INIT_MMX 3dnow

  730. IMDCT_CALC_FUNC

  731. INIT_MMX 3dnowext

  732. IMDCT_CALC_FUNC

  733. %endif

  734. 

  735. INIT_XMM sse

  736. IMDCT_CALC_FUNC

  737. 

  738. %if ARCH_X86_32

  739. INIT_MMX 3dnow

  740. %define mulps pfmul

  741. %define addps pfadd

  742. %define subps pfsub

  743. %define unpcklps punpckldq

  744. %define unpckhps punpckhdq

  745. DECL_PASS pass_3dnow, PASS_SMALL 1, [wq], [wq+o1q]

  746. DECL_PASS pass_interleave_3dnow, PASS_BIG 0

  747. %define pass_3dnowext pass_3dnow

  748. %define pass_interleave_3dnowext pass_interleave_3dnow

  749. %endif

  750. 

  751. %ifdef PIC

  752. %define SECTION_REL - $$

  753. %else

  754. %define SECTION_REL

  755. %endif

  756. 

  757. %macro DECL_FFT 1-2 ; nbits, suffix

  758. %ifidn %0, 1

  759. %xdefine fullsuffix SUFFIX

  760. %else

  761. %xdefine fullsuffix %2 %+ SUFFIX

  762. %endif

  763. %xdefine list_of_fft fft4 %+ SUFFIX SECTION_REL, fft8 %+ SUFFIX SECTION_REL

  764. %if %1>=5

  765. %xdefine list_of_fft list_of_fft, fft16 %+ SUFFIX SECTION_REL

  766. %endif

  767. %if %1>=6

  768. %xdefine list_of_fft list_of_fft, fft32 %+ fullsuffix SECTION_REL

  769. %endif

  770. 

  771. %assign n 1<<%1

  772. %rep 17-%1

  773. %assign n2 n/2

  774. %assign n4 n/4

  775. %xdefine list_of_fft list_of_fft, fft %+ n %+ fullsuffix SECTION_REL

  776. 

  777. align 16

  778. fft %+ n %+ fullsuffix:

  779.     call fft %+ n2 %+ SUFFIX

  780.     add r0, n*4 - (n&(-2<<%1))

  781.     call fft %+ n4 %+ SUFFIX

  782.     add r0, n*2 - (n2&(-2<<%1))

  783.     call fft %+ n4 %+ SUFFIX

  784.     sub r0, n*6 + (n2&(-2<<%1))

  785.     lea r1, [cos_ %+ n]

  786.     mov r2d, n4/2

  787.     jmp pass %+ fullsuffix

  788. 

  789. %assign n n*2

  790. %endrep

  791. %undef n

  792. 

  793. align 8

  794. dispatch_tab %+ fullsuffix: pointer list_of_fft

  795. 

  796. section .text

  797. 

  798. ; On x86_32, this function does the register saving and restoring for all of fft.

  799. ; The others pass args in registers and don't spill anything.

  800. cglobal fft_dispatch%2, 2,5,8, zc, nbits

  801.     FFT_DISPATCH fullsuffix, nbits

  802.     RET

  803. %endmacro ; DECL_FFT

  804. 

  805. %if HAVE_AVX

  806. INIT_YMM avx

  807. DECL_FFT 6

  808. DECL_FFT 6, _interleave

  809. %endif

  810. INIT_XMM sse

  811. DECL_FFT 5

  812. DECL_FFT 5, _interleave

  813. %if ARCH_X86_32

  814. INIT_MMX 3dnow

  815. DECL_FFT 4

  816. DECL_FFT 4, _interleave

  817. INIT_MMX 3dnowext

  818. DECL_FFT 4

  819. DECL_FFT 4, _interleave

  820. %endif

  821. 

  822. INIT_XMM sse

  823. %undef mulps

  824. %undef addps

  825. %undef subps

  826. %undef unpcklps

  827. %undef unpckhps

  828. 

  829. %macro PREROTATER 5 ;-2*k, 2*k, input+n4, tcos+n8, tsin+n8

  830. %if mmsize == 8 ; j*2+2-n4, n4-2-j*2, input+n4, tcos+n8, tsin+n8

  831.     PSWAPD     m0, [%3+%2*4]

  832.     movq       m2, [%3+%1*4-8]

  833.     movq       m3, m0

  834.     punpckldq  m0, m2

  835.     punpckhdq  m2, m3

  836.     movd       m1, [%4+%1*2-4] ; tcos[j]

  837.     movd       m3, [%4+%2*2]   ; tcos[n4-j-1]

  838.     punpckldq  m1, [%5+%1*2-4] ; tsin[j]

  839.     punpckldq  m3, [%5+%2*2]   ; tsin[n4-j-1]

  840. 

  841.     mova       m4, m0

  842.     PSWAPD     m5, m1

  843.     pfmul      m0, m1

  844.     pfmul      m4, m5

  845.     mova       m6, m2

  846.     PSWAPD     m5, m3

  847.     pfmul      m2, m3

  848.     pfmul      m6, m5

  849. %if cpuflag(3dnowext)

  850.     pfpnacc    m0, m4

  851.     pfpnacc    m2, m6

  852. %else

  853.     SBUTTERFLY dq, 0, 4, 1

  854.     SBUTTERFLY dq, 2, 6, 3

  855.     pxor       m4, m7

  856.     pxor       m6, m7

  857.     pfadd      m0, m4

  858.     pfadd      m2, m6

  859. %endif

  860. %else

  861.     movaps   xmm0, [%3+%2*4]

  862.     movaps   xmm1, [%3+%1*4-0x10]

  863.     movaps   xmm2, xmm0

  864.     shufps   xmm0, xmm1, 0x88

  865.     shufps   xmm1, xmm2, 0x77

  866.     movlps   xmm4, [%4+%2*2]

  867.     movlps   xmm5, [%5+%2*2+0x0]

  868.     movhps   xmm4, [%4+%1*2-0x8]

  869.     movhps   xmm5, [%5+%1*2-0x8]

  870.     movaps   xmm2, xmm0

  871.     movaps   xmm3, xmm1

  872.     mulps    xmm0, xmm5

  873.     mulps    xmm1, xmm4

  874.     mulps    xmm2, xmm4

  875.     mulps    xmm3, xmm5

  876.     subps    xmm1, xmm0

  877.     addps    xmm2, xmm3

  878.     movaps   xmm0, xmm1

  879.     unpcklps xmm1, xmm2

  880.     unpckhps xmm0, xmm2

  881. %endif

  882. %endmacro

  883. 

  884. %macro CMUL 6 ;j, xmm0, xmm1, 3, 4, 5

  885.     mulps      m6, %3, [%5+%1]

  886.     mulps      m7, %2, [%5+%1]

  887.     mulps      %2, %2, [%6+%1]

  888.     mulps      %3, %3, [%6+%1]

  889.     subps      %2, %2, m6

  890.     addps      %3, %3, m7

  891. %endmacro

  892. 

  893. %macro POSROTATESHUF_AVX 5 ;j, k, z+n8, tcos+n8, tsin+n8

  894. .post:

  895.     vmovaps      ymm1,   [%3+%1*2]

  896.     vmovaps      ymm0,   [%3+%1*2+0x20]

  897.     vmovaps      ymm3,   [%3+%2*2]

  898.     vmovaps      ymm2,   [%3+%2*2+0x20]

  899. 

  900.     CMUL         %1, ymm0, ymm1, %3, %4, %5

  901.     CMUL         %2, ymm2, ymm3, %3, %4, %5

  902.     vshufps      ymm1, ymm1, ymm1, 0x1b

  903.     vshufps      ymm3, ymm3, ymm3, 0x1b

  904.     vperm2f128   ymm1, ymm1, ymm1, 0x01

  905.     vperm2f128   ymm3, ymm3, ymm3, 0x01

  906.     vunpcklps    ymm6, ymm2, ymm1

  907.     vunpckhps    ymm4, ymm2, ymm1

  908.     vunpcklps    ymm7, ymm0, ymm3

  909.     vunpckhps    ymm5, ymm0, ymm3

  910. 

  911.     vextractf128 [%3+%1*2],      ymm7, 0

  912.     vextractf128 [%3+%1*2+0x10], ymm5, 0

  913.     vextractf128 [%3+%1*2+0x20], ymm7, 1

  914.     vextractf128 [%3+%1*2+0x30], ymm5, 1

  915. 

  916.     vextractf128 [%3+%2*2],      ymm6, 0

  917.     vextractf128 [%3+%2*2+0x10], ymm4, 0

  918.     vextractf128 [%3+%2*2+0x20], ymm6, 1

  919.     vextractf128 [%3+%2*2+0x30], ymm4, 1

  920.     sub      %2,   0x20

  921.     add      %1,   0x20

  922.     jl       .post

  923. %endmacro

  924. 

  925. %macro POSROTATESHUF 5 ;j, k, z+n8, tcos+n8, tsin+n8

  926. .post:

  927.     movaps   xmm1, [%3+%1*2]

  928.     movaps   xmm0, [%3+%1*2+0x10]

  929.     CMUL     %1,   xmm0, xmm1, %3, %4, %5

  930.     movaps   xmm5, [%3+%2*2]

  931.     movaps   xmm4, [%3+%2*2+0x10]

  932.     CMUL     %2,   xmm4, xmm5, %3, %4, %5

  933.     shufps   xmm1, xmm1, 0x1b

  934.     shufps   xmm5, xmm5, 0x1b

  935.     movaps   xmm6, xmm4

  936.     unpckhps xmm4, xmm1

  937.     unpcklps xmm6, xmm1

  938.     movaps   xmm2, xmm0

  939.     unpcklps xmm0, xmm5

  940.     unpckhps xmm2, xmm5

  941.     movaps   [%3+%2*2],      xmm6

  942.     movaps   [%3+%2*2+0x10], xmm4

  943.     movaps   [%3+%1*2],      xmm0

  944.     movaps   [%3+%1*2+0x10], xmm2

  945.     sub      %2,   0x10

  946.     add      %1,   0x10

  947.     jl       .post

  948. %endmacro

  949. 

  950. %macro CMUL_3DNOW 6

  951.     mova       m6, [%1+%2*2]

  952.     mova       %3, [%1+%2*2+8]

  953.     mova       %4, m6

  954.     mova       m7, %3

  955.     pfmul      m6, [%5+%2]

  956.     pfmul      %3, [%6+%2]

  957.     pfmul      %4, [%6+%2]

  958.     pfmul      m7, [%5+%2]

  959.     pfsub      %3, m6

  960.     pfadd      %4, m7

  961. %endmacro

  962. 

  963. %macro POSROTATESHUF_3DNOW 5 ;j, k, z+n8, tcos+n8, tsin+n8

  964. .post:

  965.     CMUL_3DNOW %3, %1, m0, m1, %4, %5

  966.     CMUL_3DNOW %3, %2, m2, m3, %4, %5

  967.     movd  [%3+%1*2+ 0], m0

  968.     movd  [%3+%2*2+12], m1

  969.     movd  [%3+%2*2+ 0], m2

  970.     movd  [%3+%1*2+12], m3

  971.     psrlq      m0, 32

  972.     psrlq      m1, 32

  973.     psrlq      m2, 32

  974.     psrlq      m3, 32

  975.     movd  [%3+%1*2+ 8], m0

  976.     movd  [%3+%2*2+ 4], m1

  977.     movd  [%3+%2*2+ 8], m2

  978.     movd  [%3+%1*2+ 4], m3

  979.     sub        %2, 8

  980.     add        %1, 8

  981.     jl         .post

  982. %endmacro

  983. 

  984. %macro DECL_IMDCT 1

  985. cglobal imdct_half, 3,12,8; FFTContext *s, FFTSample *output, const FFTSample *input

  986. %if ARCH_X86_64

  987. %define rrevtab r7

  988. %define rtcos   r8

  989. %define rtsin   r9

  990. %else

  991. %define rrevtab r6

  992. %define rtsin   r6

  993. %define rtcos   r5

  994. %endif

  995.     mov   r3d, [r0+FFTContext.mdctsize]

  996.     add   r2, r3

  997.     shr   r3, 1

  998.     mov   rtcos, [r0+FFTContext.tcos]

  999.     mov   rtsin, [r0+FFTContext.tsin]

  1000.     add   rtcos, r3

  1001.     add   rtsin, r3

  1002. %if ARCH_X86_64 == 0

  1003.     push  rtcos

  1004.     push  rtsin

  1005. %endif

  1006.     shr   r3, 1

  1007.     mov   rrevtab, [r0+FFTContext.revtab]

  1008.     add   rrevtab, r3

  1009. %if ARCH_X86_64 == 0

  1010.     push  rrevtab

  1011. %endif

  1012. 

  1013. %if mmsize == 8

  1014.     sub   r3, 2

  1015. %else

  1016.     sub   r3, 4

  1017. %endif

  1018. %if ARCH_X86_64 || mmsize == 8

  1019.     xor   r4, r4

  1020.     sub   r4, r3

  1021. %endif

  1022. %if notcpuflag(3dnowext) && mmsize == 8

  1023.     movd  m7, [ps_m1m1m1m1]

  1024. %endif

  1025. .pre:

  1026. %if ARCH_X86_64 == 0

  1027. ;unspill

  1028. %if mmsize != 8

  1029.     xor   r4, r4

  1030.     sub   r4, r3

  1031. %endif

  1032.     mov   rtcos, [esp+8]

  1033.     mov   rtsin, [esp+4]

  1034. %endif

  1035. 

  1036.     PREROTATER r4, r3, r2, rtcos, rtsin

  1037. %if mmsize == 8

  1038.     mov    r6, [esp]                ; rrevtab = ptr+n8

  1039.     movzx  r5,  word [rrevtab+r4-2] ; rrevtab[j]

  1040.     movzx  r6,  word [rrevtab+r3]   ; rrevtab[n4-j-1]

  1041.     mova [r1+r5*8], m0

  1042.     mova [r1+r6*8], m2

  1043.     add    r4, 2

  1044.     sub    r3, 2

  1045. %else

  1046. %if ARCH_X86_64

  1047.     movzx  r5,  word [rrevtab+r4-4]

  1048.     movzx  r6,  word [rrevtab+r4-2]

  1049.     movzx  r10, word [rrevtab+r3]

  1050.     movzx  r11, word [rrevtab+r3+2]

  1051.     movlps [r1+r5 *8], xmm0

  1052.     movhps [r1+r6 *8], xmm0

  1053.     movlps [r1+r10*8], xmm1

  1054.     movhps [r1+r11*8], xmm1

  1055.     add    r4, 4

  1056. %else

  1057.     mov    r6, [esp]

  1058.     movzx  r5, word [r6+r4-4]

  1059.     movzx  r4, word [r6+r4-2]

  1060.     movlps [r1+r5*8], xmm0

  1061.     movhps [r1+r4*8], xmm0

  1062.     movzx  r5, word [r6+r3]

  1063.     movzx  r4, word [r6+r3+2]

  1064.     movlps [r1+r5*8], xmm1

  1065.     movhps [r1+r4*8], xmm1

  1066. %endif

  1067.     sub    r3, 4

  1068. %endif

  1069.     jns    .pre

  1070. 

  1071.     mov  r5, r0

  1072.     mov  r6, r1

  1073.     mov  r0, r1

  1074.     mov  r1d, [r5+FFTContext.nbits]

  1075. 

  1076.     FFT_DISPATCH SUFFIX, r1

  1077. 

  1078.     mov  r0d, [r5+FFTContext.mdctsize]

  1079.     add  r6, r0

  1080.     shr  r0, 1

  1081. %if ARCH_X86_64 == 0

  1082. %define rtcos r2

  1083. %define rtsin r3

  1084.     mov  rtcos, [esp+8]

  1085.     mov  rtsin, [esp+4]

  1086. %endif

  1087.     neg  r0

  1088.     mov  r1, -mmsize

  1089.     sub  r1, r0

  1090.     %1 r0, r1, r6, rtcos, rtsin

  1091. %if ARCH_X86_64 == 0

  1092.     add esp, 12

  1093. %endif

  1094. %if mmsize == 8

  1095.     femms

  1096. %endif

  1097.     RET

  1098. %endmacro

  1099. 

  1100. DECL_IMDCT POSROTATESHUF

  1101. 

  1102. %if ARCH_X86_32

  1103. INIT_MMX 3dnow

  1104. DECL_IMDCT POSROTATESHUF_3DNOW

  1105. 

  1106. INIT_MMX 3dnowext

  1107. DECL_IMDCT POSROTATESHUF_3DNOW

  1108. %endif

  1109. 

  1110. INIT_YMM avx

  1111. 

  1112. %if HAVE_AVX

  1113. DECL_IMDCT POSROTATESHUF_AVX

  1114. %endif