falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
44689 Commits
32 Branches
283MB
Tree: c5714b51aa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c5714b51aa'
${ noResults }
FFmpeg/libavcodec/x86/fft.asm

1079 lines
25KB
Raw Blame History 

  1. ;******************************************************************************

  2. ;* FFT transform with SSE/3DNow optimizations

  3. ;* Copyright (c) 2008 Loren Merritt

  4. ;* Copyright (c) 2011 Vitor Sessak

  5. ;*

  6. ;* This algorithm (though not any of the implementation details) is

  7. ;* based on libdjbfft by D. J. Bernstein.

  8. ;*

  9. ;* This file is part of Libav.

  10. ;*

  11. ;* Libav is free software; you can redistribute it and/or

  12. ;* modify it under the terms of the GNU Lesser General Public

  13. ;* License as published by the Free Software Foundation; either

  14. ;* version 2.1 of the License, or (at your option) any later version.

  15. ;*

  16. ;* Libav is distributed in the hope that it will be useful,

  17. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  18. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  19. ;* Lesser General Public License for more details.

  20. ;*

  21. ;* You should have received a copy of the GNU Lesser General Public

  22. ;* License along with Libav; if not, write to the Free Software

  23. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  24. ;******************************************************************************

  25. 

  26. ; These functions are not individually interchangeable with the C versions.

  27. ; While C takes arrays of FFTComplex, SSE/3DNow leave intermediate results

  28. ; in blocks as conventient to the vector size.

  29. ; i.e. {4x real, 4x imaginary, 4x real, ...} (or 2x respectively)

  30. 

  31. %include "libavutil/x86/x86util.asm"

  32. 

  33. %if ARCH_X86_64

  34. %define pointer resq

  35. %else

  36. %define pointer resd

  37. %endif

  38. 

  39. struc FFTContext

  40.     .nbits:    resd 1

  41.     .reverse:  resd 1

  42.     .revtab:   pointer 1

  43.     .tmpbuf:   pointer 1

  44.     .mdctsize: resd 1

  45.     .mdctbits: resd 1

  46.     .tcos:     pointer 1

  47.     .tsin:     pointer 1

  48.     .fftperm:  pointer 1

  49.     .fftcalc:  pointer 1

  50.     .imdctcalc:pointer 1

  51.     .imdcthalf:pointer 1

  52. endstruc

  53. 

  54. SECTION_RODATA 32

  55. 

  56. %define M_SQRT1_2 0.70710678118654752440

  57. %define M_COS_PI_1_8 0.923879532511287

  58. %define M_COS_PI_3_8 0.38268343236509

  59. 

  60. ps_cos16_1: dd 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8, 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8

  61. ps_cos16_2: dd 0, M_COS_PI_3_8, M_SQRT1_2, M_COS_PI_1_8, 0, -M_COS_PI_3_8, -M_SQRT1_2, -M_COS_PI_1_8

  62. 

  63. ps_root2: times 8 dd M_SQRT1_2

  64. ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2

  65. ps_p1p1m1p1: dd 0, 0, 1<<31, 0, 0, 0, 1<<31, 0

  66. 

  67. perm1: dd 0x00, 0x02, 0x03, 0x01, 0x03, 0x00, 0x02, 0x01

  68. perm2: dd 0x00, 0x01, 0x02, 0x03, 0x01, 0x00, 0x02, 0x03

  69. ps_p1p1m1p1root2: dd 1.0, 1.0, -1.0, 1.0, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2

  70. ps_m1m1p1m1p1m1m1m1: dd 1<<31, 1<<31, 0, 1<<31, 0, 1<<31, 1<<31, 1<<31

  71. ps_m1m1m1m1: times 4 dd 1<<31

  72. ps_m1p1: dd 1<<31, 0

  73. 

  74. %assign i 16

  75. %rep 13

  76. cextern cos_ %+ i

  77. %assign i i<<1

  78. %endrep

  79. 

  80. %if ARCH_X86_64

  81.     %define pointer dq

  82. %else

  83.     %define pointer dd

  84. %endif

  85. 

  86. %macro IF0 1+

  87. %endmacro

  88. %macro IF1 1+

  89.     %1

  90. %endmacro

  91. 

  92. SECTION .text

  93. 

  94. %macro T2_3DNOW 4 ; z0, z1, mem0, mem1

  95.     mova     %1, %3

  96.     mova     %2, %1

  97.     pfadd    %1, %4

  98.     pfsub    %2, %4

  99. %endmacro

  100. 

  101. %macro T4_3DNOW 6 ; z0, z1, z2, z3, tmp0, tmp1

  102.     mova     %5, %3

  103.     pfsub    %3, %4

  104.     pfadd    %5, %4 ; {t6,t5}

  105.     pxor     %3, [ps_m1p1] ; {t8,t7}

  106.     mova     %6, %1

  107.     movd [r0+12], %3

  108.     punpckhdq %3, [r0+8]

  109.     pfadd    %1, %5 ; {r0,i0}

  110.     pfsub    %6, %5 ; {r2,i2}

  111.     mova     %4, %2

  112.     pfadd    %2, %3 ; {r1,i1}

  113.     pfsub    %4, %3 ; {r3,i3}

  114.     SWAP     %3, %6

  115. %endmacro

  116. 

  117. ;  in: %1 = {r0,i0,r2,i2,r4,i4,r6,i6}

  118. ;      %2 = {r1,i1,r3,i3,r5,i5,r7,i7}

  119. ;      %3, %4, %5 tmp

  120. ; out: %1 = {r0,r1,r2,r3,i0,i1,i2,i3}

  121. ;      %2 = {r4,r5,r6,r7,i4,i5,i6,i7}

  122. %macro T8_AVX 5

  123.     vsubps     %5, %1, %2       ; v  = %1 - %2

  124.     vaddps     %3, %1, %2       ; w  = %1 + %2

  125.     vmulps     %2, %5, [ps_p1p1m1p1root2]  ; v *= vals1

  126.     vpermilps  %2, %2, [perm1]

  127.     vblendps   %1, %2, %3, 0x33 ; q = {w1,w2,v4,v2,w5,w6,v7,v6}

  128.     vshufps    %5, %3, %2, 0x4e ; r = {w3,w4,v1,v3,w7,w8,v8,v5}

  129.     vsubps     %4, %5, %1       ; s = r - q

  130.     vaddps     %1, %5, %1       ; u = r + q

  131.     vpermilps  %1, %1, [perm2]  ; k  = {u1,u2,u3,u4,u6,u5,u7,u8}

  132.     vshufps    %5, %4, %1, 0xbb

  133.     vshufps    %3, %4, %1, 0xee

  134.     vperm2f128 %3, %3, %5, 0x13

  135.     vxorps     %4, %4, [ps_m1m1p1m1p1m1m1m1]  ; s *= {1,1,-1,-1,1,-1,-1,-1}

  136.     vshufps    %2, %1, %4, 0xdd

  137.     vshufps    %1, %1, %4, 0x88

  138.     vperm2f128 %4, %2, %1, 0x02 ; v  = {k1,k3,s1,s3,k2,k4,s2,s4}

  139.     vperm2f128 %1, %1, %2, 0x13 ; w  = {k6,k8,s6,s8,k5,k7,s5,s7}

  140.     vsubps     %5, %1, %3

  141.     vblendps   %1, %5, %1, 0x55 ; w -= {0,s7,0,k7,0,s8,0,k8}

  142.     vsubps     %2, %4, %1       ; %2 = v - w

  143.     vaddps     %1, %4, %1       ; %1 = v + w

  144. %endmacro

  145. 

  146. ; In SSE mode do one fft4 transforms

  147. ; in:  %1={r0,i0,r2,i2} %2={r1,i1,r3,i3}

  148. ; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}

  149. ;

  150. ; In AVX mode do two fft4 transforms

  151. ; in:  %1={r0,i0,r2,i2,r4,i4,r6,i6} %2={r1,i1,r3,i3,r5,i5,r7,i7}

  152. ; out: %1={r0,r1,r2,r3,r4,r5,r6,r7} %2={i0,i1,i2,i3,i4,i5,i6,i7}

  153. %macro T4_SSE 3

  154.     subps    %3, %1, %2       ; {t3,t4,-t8,t7}

  155.     addps    %1, %1, %2       ; {t1,t2,t6,t5}

  156.     xorps    %3, %3, [ps_p1p1m1p1]

  157.     shufps   %2, %1, %3, 0xbe ; {t6,t5,t7,t8}

  158.     shufps   %1, %1, %3, 0x44 ; {t1,t2,t3,t4}

  159.     subps    %3, %1, %2       ; {r2,i2,r3,i3}

  160.     addps    %1, %1, %2       ; {r0,i0,r1,i1}

  161.     shufps   %2, %1, %3, 0xdd ; {i0,i1,i2,i3}

  162.     shufps   %1, %1, %3, 0x88 ; {r0,r1,r2,r3}

  163. %endmacro

  164. 

  165. ; In SSE mode do one FFT8

  166. ; in:  %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %3={r4,i4,r6,i6} %4={r5,i5,r7,i7}

  167. ; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %1={r4,r5,r6,r7} %2={i4,i5,i6,i7}

  168. ;

  169. ; In AVX mode do two FFT8

  170. ; in:  %1={r0,i0,r2,i2,r8, i8, r10,i10} %2={r1,i1,r3,i3,r9, i9, r11,i11}

  171. ;      %3={r4,i4,r6,i6,r12,i12,r14,i14} %4={r5,i5,r7,i7,r13,i13,r15,i15}

  172. ; out: %1={r0,r1,r2,r3,r8, r9, r10,r11} %2={i0,i1,i2,i3,i8, i9, i10,i11}

  173. ;      %3={r4,r5,r6,r7,r12,r13,r14,r15} %4={i4,i5,i6,i7,i12,i13,i14,i15}

  174. %macro T8_SSE 6

  175.     addps    %6, %3, %4       ; {t1,t2,t3,t4}

  176.     subps    %3, %3, %4       ; {r5,i5,r7,i7}

  177.     shufps   %4, %3, %3, 0xb1 ; {i5,r5,i7,r7}

  178.     mulps    %3, %3, [ps_root2mppm] ; {-r5,i5,r7,-i7}

  179.     mulps    %4, %4, [ps_root2]

  180.     addps    %3, %3, %4       ; {t8,t7,ta,t9}

  181.     shufps   %4, %6, %3, 0x9c ; {t1,t4,t7,ta}

  182.     shufps   %6, %6, %3, 0x36 ; {t3,t2,t9,t8}

  183.     subps    %3, %6, %4       ; {t6,t5,tc,tb}

  184.     addps    %6, %6, %4       ; {t1,t2,t9,ta}

  185.     shufps   %5, %6, %3, 0x8d ; {t2,ta,t6,tc}

  186.     shufps   %6, %6, %3, 0xd8 ; {t1,t9,t5,tb}

  187.     subps    %3, %1, %6       ; {r4,r5,r6,r7}

  188.     addps    %1, %1, %6       ; {r0,r1,r2,r3}

  189.     subps    %4, %2, %5       ; {i4,i5,i6,i7}

  190.     addps    %2, %2, %5       ; {i0,i1,i2,i3}

  191. %endmacro

  192. 

  193. %macro INTERL 5

  194. %if cpuflag(avx)

  195.     vunpckhps      %3, %2, %1

  196.     vunpcklps      %2, %2, %1

  197.     vextractf128   %4(%5), %2, 0

  198.     vextractf128  %4 %+ H(%5), %3, 0

  199.     vextractf128   %4(%5 + 1), %2, 1

  200.     vextractf128  %4 %+ H(%5 + 1), %3, 1

  201. %elif cpuflag(sse)

  202.     mova     %3, %2

  203.     unpcklps %2, %1

  204.     unpckhps %3, %1

  205.     mova  %4(%5), %2

  206.     mova  %4(%5+1), %3

  207. %endif

  208. %endmacro

  209. 

  210. ; scheduled for cpu-bound sizes

  211. %macro PASS_SMALL 3 ; (to load m4-m7), wre, wim

  212. IF%1 mova    m4, Z(4)

  213. IF%1 mova    m5, Z(5)

  214.     mova     m0, %2 ; wre

  215.     mova     m1, %3 ; wim

  216.     mulps    m2, m4, m0 ; r2*wre

  217. IF%1 mova    m6, Z2(6)

  218.     mulps    m3, m5, m1 ; i2*wim

  219. IF%1 mova    m7, Z2(7)

  220.     mulps    m4, m4, m1 ; r2*wim

  221.     mulps    m5, m5, m0 ; i2*wre

  222.     addps    m2, m2, m3 ; r2*wre + i2*wim

  223.     mulps    m3, m1, m7 ; i3*wim

  224.     subps    m5, m5, m4 ; i2*wre - r2*wim

  225.     mulps    m1, m1, m6 ; r3*wim

  226.     mulps    m4, m0, m6 ; r3*wre

  227.     mulps    m0, m0, m7 ; i3*wre

  228.     subps    m4, m4, m3 ; r3*wre - i3*wim

  229.     mova     m3, Z(0)

  230.     addps    m0, m0, m1 ; i3*wre + r3*wim

  231.     subps    m1, m4, m2 ; t3

  232.     addps    m4, m4, m2 ; t5

  233.     subps    m3, m3, m4 ; r2

  234.     addps    m4, m4, Z(0) ; r0

  235.     mova     m6, Z(2)

  236.     mova   Z(4), m3

  237.     mova   Z(0), m4

  238.     subps    m3, m5, m0 ; t4

  239.     subps    m4, m6, m3 ; r3

  240.     addps    m3, m3, m6 ; r1

  241.     mova  Z2(6), m4

  242.     mova   Z(2), m3

  243.     mova     m2, Z(3)

  244.     addps    m3, m5, m0 ; t6

  245.     subps    m2, m2, m1 ; i3

  246.     mova     m7, Z(1)

  247.     addps    m1, m1, Z(3) ; i1

  248.     mova  Z2(7), m2

  249.     mova   Z(3), m1

  250.     subps    m4, m7, m3 ; i2

  251.     addps    m3, m3, m7 ; i0

  252.     mova   Z(5), m4

  253.     mova   Z(1), m3

  254. %endmacro

  255. 

  256. ; scheduled to avoid store->load aliasing

  257. %macro PASS_BIG 1 ; (!interleave)

  258.     mova     m4, Z(4) ; r2

  259.     mova     m5, Z(5) ; i2

  260.     mova     m0, [wq] ; wre

  261.     mova     m1, [wq+o1q] ; wim

  262.     mulps    m2, m4, m0 ; r2*wre

  263.     mova     m6, Z2(6) ; r3

  264.     mulps    m3, m5, m1 ; i2*wim

  265.     mova     m7, Z2(7) ; i3

  266.     mulps    m4, m4, m1 ; r2*wim

  267.     mulps    m5, m5, m0 ; i2*wre

  268.     addps    m2, m2, m3 ; r2*wre + i2*wim

  269.     mulps    m3, m1, m7 ; i3*wim

  270.     mulps    m1, m1, m6 ; r3*wim

  271.     subps    m5, m5, m4 ; i2*wre - r2*wim

  272.     mulps    m4, m0, m6 ; r3*wre

  273.     mulps    m0, m0, m7 ; i3*wre

  274.     subps    m4, m4, m3 ; r3*wre - i3*wim

  275.     mova     m3, Z(0)

  276.     addps    m0, m0, m1 ; i3*wre + r3*wim

  277.     subps    m1, m4, m2 ; t3

  278.     addps    m4, m4, m2 ; t5

  279.     subps    m3, m3, m4 ; r2

  280.     addps    m4, m4, Z(0) ; r0

  281.     mova     m6, Z(2)

  282.     mova   Z(4), m3

  283.     mova   Z(0), m4

  284.     subps    m3, m5, m0 ; t4

  285.     subps    m4, m6, m3 ; r3

  286.     addps    m3, m3, m6 ; r1

  287. IF%1 mova Z2(6), m4

  288. IF%1 mova  Z(2), m3

  289.     mova     m2, Z(3)

  290.     addps    m5, m5, m0 ; t6

  291.     subps    m2, m2, m1 ; i3

  292.     mova     m7, Z(1)

  293.     addps    m1, m1, Z(3) ; i1

  294. IF%1 mova Z2(7), m2

  295. IF%1 mova  Z(3), m1

  296.     subps    m6, m7, m5 ; i2

  297.     addps    m5, m5, m7 ; i0

  298. IF%1 mova  Z(5), m6

  299. IF%1 mova  Z(1), m5

  300. %if %1==0

  301.     INTERL m1, m3, m7, Z, 2

  302.     INTERL m2, m4, m0, Z2, 6

  303. 

  304.     mova     m1, Z(0)

  305.     mova     m2, Z(4)

  306. 

  307.     INTERL m5, m1, m3, Z, 0

  308.     INTERL m6, m2, m7, Z, 4

  309. %endif

  310. %endmacro

  311. 

  312. %macro PUNPCK 3

  313.     mova      %3, %1

  314.     punpckldq %1, %2

  315.     punpckhdq %3, %2

  316. %endmacro

  317. 

  318. %define Z(x) [r0+mmsize*x]

  319. %define Z2(x) [r0+mmsize*x]

  320. %define ZH(x) [r0+mmsize*x+mmsize/2]

  321. 

  322. INIT_YMM avx

  323. 

  324. align 16

  325. fft8_avx:

  326.     mova      m0, Z(0)

  327.     mova      m1, Z(1)

  328.     T8_AVX    m0, m1, m2, m3, m4

  329.     mova      Z(0), m0

  330.     mova      Z(1), m1

  331.     ret

  332. 

  333. 

  334. align 16

  335. fft16_avx:

  336.     mova       m2, Z(2)

  337.     mova       m3, Z(3)

  338.     T4_SSE     m2, m3, m7

  339. 

  340.     mova       m0, Z(0)

  341.     mova       m1, Z(1)

  342.     T8_AVX     m0, m1, m4, m5, m7

  343. 

  344.     mova       m4, [ps_cos16_1]

  345.     mova       m5, [ps_cos16_2]

  346.     vmulps     m6, m2, m4

  347.     vmulps     m7, m3, m5

  348.     vaddps     m7, m7, m6

  349.     vmulps     m2, m2, m5

  350.     vmulps     m3, m3, m4

  351.     vsubps     m3, m3, m2

  352.     vblendps   m2, m7, m3, 0xf0

  353.     vperm2f128 m3, m7, m3, 0x21

  354.     vaddps     m4, m2, m3

  355.     vsubps     m2, m3, m2

  356.     vperm2f128 m2, m2, m2, 0x01

  357.     vsubps     m3, m1, m2

  358.     vaddps     m1, m1, m2

  359.     vsubps     m5, m0, m4

  360.     vaddps     m0, m0, m4

  361.     vextractf128   Z(0), m0, 0

  362.     vextractf128  ZH(0), m1, 0

  363.     vextractf128   Z(1), m0, 1

  364.     vextractf128  ZH(1), m1, 1

  365.     vextractf128   Z(2), m5, 0

  366.     vextractf128  ZH(2), m3, 0

  367.     vextractf128   Z(3), m5, 1

  368.     vextractf128  ZH(3), m3, 1

  369.     ret

  370. 

  371. align 16

  372. fft32_avx:

  373.     call fft16_avx

  374. 

  375.     mova m0, Z(4)

  376.     mova m1, Z(5)

  377. 

  378.     T4_SSE      m0, m1, m4

  379. 

  380.     mova m2, Z(6)

  381.     mova m3, Z(7)

  382. 

  383.     T8_SSE      m0, m1, m2, m3, m4, m6

  384.     ; m0={r0,r1,r2,r3,r8, r9, r10,r11} m1={i0,i1,i2,i3,i8, i9, i10,i11}

  385.     ; m2={r4,r5,r6,r7,r12,r13,r14,r15} m3={i4,i5,i6,i7,i12,i13,i14,i15}

  386. 

  387.     vperm2f128  m4, m0, m2, 0x20

  388.     vperm2f128  m5, m1, m3, 0x20

  389.     vperm2f128  m6, m0, m2, 0x31

  390.     vperm2f128  m7, m1, m3, 0x31

  391. 

  392.     PASS_SMALL 0, [cos_32], [cos_32+32]

  393. 

  394.     ret

  395. 

  396. fft32_interleave_avx:

  397.     call fft32_avx

  398.     mov r2d, 32

  399. .deint_loop:

  400.     mova     m2, Z(0)

  401.     mova     m3, Z(1)

  402.     vunpcklps      m0, m2, m3

  403.     vunpckhps      m1, m2, m3

  404.     vextractf128   Z(0), m0, 0

  405.     vextractf128  ZH(0), m1, 0

  406.     vextractf128   Z(1), m0, 1

  407.     vextractf128  ZH(1), m1, 1

  408.     add r0, mmsize*2

  409.     sub r2d, mmsize/4

  410.     jg .deint_loop

  411.     ret

  412. 

  413. INIT_XMM sse

  414. 

  415. align 16

  416. fft4_avx:

  417. fft4_sse:

  418.     mova     m0, Z(0)

  419.     mova     m1, Z(1)

  420.     T4_SSE   m0, m1, m2

  421.     mova   Z(0), m0

  422.     mova   Z(1), m1

  423.     ret

  424. 

  425. align 16

  426. fft8_sse:

  427.     mova     m0, Z(0)

  428.     mova     m1, Z(1)

  429.     T4_SSE   m0, m1, m2

  430.     mova     m2, Z(2)

  431.     mova     m3, Z(3)

  432.     T8_SSE   m0, m1, m2, m3, m4, m5

  433.     mova   Z(0), m0

  434.     mova   Z(1), m1

  435.     mova   Z(2), m2

  436.     mova   Z(3), m3

  437.     ret

  438. 

  439. align 16

  440. fft16_sse:

  441.     mova     m0, Z(0)

  442.     mova     m1, Z(1)

  443.     T4_SSE   m0, m1, m2

  444.     mova     m2, Z(2)

  445.     mova     m3, Z(3)

  446.     T8_SSE   m0, m1, m2, m3, m4, m5

  447.     mova     m4, Z(4)

  448.     mova     m5, Z(5)

  449.     mova   Z(0), m0

  450.     mova   Z(1), m1

  451.     mova   Z(2), m2

  452.     mova   Z(3), m3

  453.     T4_SSE   m4, m5, m6

  454.     mova     m6, Z2(6)

  455.     mova     m7, Z2(7)

  456.     T4_SSE   m6, m7, m0

  457.     PASS_SMALL 0, [cos_16], [cos_16+16]

  458.     ret

  459. 

  460. 

  461. %macro FFT48_3DNOW 0

  462. align 16

  463. fft4 %+ SUFFIX:

  464.     T2_3DNOW m0, m1, Z(0), Z(1)

  465.     mova     m2, Z(2)

  466.     mova     m3, Z(3)

  467.     T4_3DNOW m0, m1, m2, m3, m4, m5

  468.     PUNPCK   m0, m1, m4

  469.     PUNPCK   m2, m3, m5

  470.     mova   Z(0), m0

  471.     mova   Z(1), m4

  472.     mova   Z(2), m2

  473.     mova   Z(3), m5

  474.     ret

  475. 

  476. align 16

  477. fft8 %+ SUFFIX:

  478.     T2_3DNOW m0, m1, Z(0), Z(1)

  479.     mova     m2, Z(2)

  480.     mova     m3, Z(3)

  481.     T4_3DNOW m0, m1, m2, m3, m4, m5

  482.     mova   Z(0), m0

  483.     mova   Z(2), m2

  484.     T2_3DNOW m4, m5,  Z(4),  Z(5)

  485.     T2_3DNOW m6, m7, Z2(6), Z2(7)

  486.     PSWAPD   m0, m5

  487.     PSWAPD   m2, m7

  488.     pxor     m0, [ps_m1p1]

  489.     pxor     m2, [ps_m1p1]

  490.     pfsub    m5, m0

  491.     pfadd    m7, m2

  492.     pfmul    m5, [ps_root2]

  493.     pfmul    m7, [ps_root2]

  494.     T4_3DNOW m1, m3, m5, m7, m0, m2

  495.     mova   Z(5), m5

  496.     mova  Z2(7), m7

  497.     mova     m0, Z(0)

  498.     mova     m2, Z(2)

  499.     T4_3DNOW m0, m2, m4, m6, m5, m7

  500.     PUNPCK   m0, m1, m5

  501.     PUNPCK   m2, m3, m7

  502.     mova   Z(0), m0

  503.     mova   Z(1), m5

  504.     mova   Z(2), m2

  505.     mova   Z(3), m7

  506.     PUNPCK   m4,  Z(5), m5

  507.     PUNPCK   m6, Z2(7), m7

  508.     mova   Z(4), m4

  509.     mova   Z(5), m5

  510.     mova  Z2(6), m6

  511.     mova  Z2(7), m7

  512.     ret

  513. %endmacro

  514. 

  515. %if ARCH_X86_32

  516. INIT_MMX 3dnowext

  517. FFT48_3DNOW

  518. 

  519. INIT_MMX 3dnow

  520. FFT48_3DNOW

  521. %endif

  522. 

  523. %define Z(x) [zcq + o1q*(x&6) + mmsize*(x&1)]

  524. %define Z2(x) [zcq + o3q + mmsize*(x&1)]

  525. %define ZH(x) [zcq + o1q*(x&6) + mmsize*(x&1) + mmsize/2]

  526. %define Z2H(x) [zcq + o3q + mmsize*(x&1) + mmsize/2]

  527. 

  528. %macro DECL_PASS 2+ ; name, payload

  529. align 16

  530. %1:

  531. DEFINE_ARGS zc, w, n, o1, o3

  532.     lea o3q, [nq*3]

  533.     lea o1q, [nq*8]

  534.     shl o3q, 4

  535. .loop:

  536.     %2

  537.     add zcq, mmsize*2

  538.     add  wq, mmsize

  539.     sub  nd, mmsize/8

  540.     jg .loop

  541.     rep ret

  542. %endmacro

  543. 

  544. %macro FFT_DISPATCH 2; clobbers 5 GPRs, 8 XMMs

  545.     lea r2, [dispatch_tab%1]

  546.     mov r2, [r2 + (%2q-2)*gprsize]

  547. %ifdef PIC

  548.     lea r3, [$$]

  549.     add r2, r3

  550. %endif

  551.     call r2

  552. %endmacro ; FFT_DISPATCH

  553. 

  554. INIT_YMM avx

  555. 

  556. DECL_PASS pass_avx, PASS_BIG 1

  557. DECL_PASS pass_interleave_avx, PASS_BIG 0

  558. 

  559. cglobal fft_calc, 2,5,8

  560.     mov     r3d, [r0 + FFTContext.nbits]

  561.     mov     r0, r1

  562.     mov     r1, r3

  563.     FFT_DISPATCH _interleave %+ SUFFIX, r1

  564.     REP_RET

  565. 

  566. 

  567. INIT_XMM sse

  568. 

  569. DECL_PASS pass_sse, PASS_BIG 1

  570. DECL_PASS pass_interleave_sse, PASS_BIG 0

  571. 

  572. %macro FFT_CALC_FUNC 0

  573. cglobal fft_calc, 2,5,8

  574.     mov     r3d, [r0 + FFTContext.nbits]

  575.     PUSH    r1

  576.     PUSH    r3

  577.     mov     r0, r1

  578.     mov     r1, r3

  579.     FFT_DISPATCH _interleave %+ SUFFIX, r1

  580.     POP     rcx

  581.     POP     r4

  582.     cmp     rcx, 3+(mmsize/16)

  583.     jg      .end

  584.     mov     r2, -1

  585.     add     rcx, 3

  586.     shl     r2, cl

  587.     sub     r4, r2

  588. .loop:

  589. %if mmsize == 8

  590.     PSWAPD  m0, [r4 + r2 + 4]

  591.     mova [r4 + r2 + 4], m0

  592. %else

  593.     movaps   xmm0, [r4 + r2]

  594.     movaps   xmm1, xmm0

  595.     unpcklps xmm0, [r4 + r2 + 16]

  596.     unpckhps xmm1, [r4 + r2 + 16]

  597.     movaps   [r4 + r2],      xmm0

  598.     movaps   [r4 + r2 + 16], xmm1

  599. %endif

  600.     add      r2, mmsize*2

  601.     jl       .loop

  602. .end:

  603. %if cpuflag(3dnow)

  604.     femms

  605.     RET

  606. %else

  607.     REP_RET

  608. %endif

  609. %endmacro

  610. 

  611. %if ARCH_X86_32

  612. INIT_MMX 3dnow

  613. FFT_CALC_FUNC

  614. INIT_MMX 3dnowext

  615. FFT_CALC_FUNC

  616. %endif

  617. INIT_XMM sse

  618. FFT_CALC_FUNC

  619. 

  620. cglobal fft_permute, 2,7,1

  621.     mov     r4,  [r0 + FFTContext.revtab]

  622.     mov     r5,  [r0 + FFTContext.tmpbuf]

  623.     mov     ecx, [r0 + FFTContext.nbits]

  624.     mov     r2, 1

  625.     shl     r2, cl

  626.     xor     r0, r0

  627. %if ARCH_X86_32

  628.     mov     r1, r1m

  629. %endif

  630. .loop:

  631.     movaps  xmm0, [r1 + 8*r0]

  632.     movzx   r6, word [r4 + 2*r0]

  633.     movzx   r3, word [r4 + 2*r0 + 2]

  634.     movlps  [r5 + 8*r6], xmm0

  635.     movhps  [r5 + 8*r3], xmm0

  636.     add     r0, 2

  637.     cmp     r0, r2

  638.     jl      .loop

  639.     shl     r2, 3

  640.     add     r1, r2

  641.     add     r5, r2

  642.     neg     r2

  643. ; nbits >= 2 (FFT4) and sizeof(FFTComplex)=8 => at least 32B

  644. .loopcopy:

  645.     movaps  xmm0, [r5 + r2]

  646.     movaps  xmm1, [r5 + r2 + 16]

  647.     movaps  [r1 + r2], xmm0

  648.     movaps  [r1 + r2 + 16], xmm1

  649.     add     r2, 32

  650.     jl      .loopcopy

  651.     REP_RET

  652. 

  653. %if ARCH_X86_32

  654. INIT_MMX 3dnow

  655. %define mulps pfmul

  656. %define addps pfadd

  657. %define subps pfsub

  658. %define unpcklps punpckldq

  659. %define unpckhps punpckhdq

  660. DECL_PASS pass_3dnow, PASS_SMALL 1, [wq], [wq+o1q]

  661. DECL_PASS pass_interleave_3dnow, PASS_BIG 0

  662. %define pass_3dnowext pass_3dnow

  663. %define pass_interleave_3dnowext pass_interleave_3dnow

  664. %endif

  665. 

  666. %ifdef PIC

  667. %define SECTION_REL - $$

  668. %else

  669. %define SECTION_REL

  670. %endif

  671. 

  672. %macro DECL_FFT 1-2 ; nbits, suffix

  673. %ifidn %0, 1

  674. %xdefine fullsuffix SUFFIX

  675. %else

  676. %xdefine fullsuffix %2 %+ SUFFIX

  677. %endif

  678. %xdefine list_of_fft fft4 %+ SUFFIX SECTION_REL, fft8 %+ SUFFIX SECTION_REL

  679. %if %1>=5

  680. %xdefine list_of_fft list_of_fft, fft16 %+ SUFFIX SECTION_REL

  681. %endif

  682. %if %1>=6

  683. %xdefine list_of_fft list_of_fft, fft32 %+ fullsuffix SECTION_REL

  684. %endif

  685. 

  686. %assign n 1<<%1

  687. %rep 17-%1

  688. %assign n2 n/2

  689. %assign n4 n/4

  690. %xdefine list_of_fft list_of_fft, fft %+ n %+ fullsuffix SECTION_REL

  691. 

  692. align 16

  693. fft %+ n %+ fullsuffix:

  694.     call fft %+ n2 %+ SUFFIX

  695.     add r0, n*4 - (n&(-2<<%1))

  696.     call fft %+ n4 %+ SUFFIX

  697.     add r0, n*2 - (n2&(-2<<%1))

  698.     call fft %+ n4 %+ SUFFIX

  699.     sub r0, n*6 + (n2&(-2<<%1))

  700.     lea r1, [cos_ %+ n]

  701.     mov r2d, n4/2

  702.     jmp pass %+ fullsuffix

  703. 

  704. %assign n n*2

  705. %endrep

  706. %undef n

  707. 

  708. align 8

  709. dispatch_tab %+ fullsuffix: pointer list_of_fft

  710. %endmacro ; DECL_FFT

  711. 

  712. INIT_YMM avx

  713. DECL_FFT 6

  714. DECL_FFT 6, _interleave

  715. INIT_XMM sse

  716. DECL_FFT 5

  717. DECL_FFT 5, _interleave

  718. %if ARCH_X86_32

  719. INIT_MMX 3dnow

  720. DECL_FFT 4

  721. DECL_FFT 4, _interleave

  722. INIT_MMX 3dnowext

  723. DECL_FFT 4

  724. DECL_FFT 4, _interleave

  725. %endif

  726. 

  727. %if CONFIG_MDCT

  728. 

  729. %macro IMDCT_CALC_FUNC 0

  730. cglobal imdct_calc, 3,5,3

  731.     mov     r3d, [r0 + FFTContext.mdctsize]

  732.     mov     r4,  [r0 + FFTContext.imdcthalf]

  733.     add     r1,  r3

  734.     PUSH    r3

  735.     PUSH    r1

  736. %if ARCH_X86_32

  737.     push    r2

  738.     push    r1

  739.     push    r0

  740. %else

  741.     sub     rsp, 8+32*WIN64 ; allocate win64 shadow space

  742. %endif

  743.     call    r4

  744. %if ARCH_X86_32

  745.     add     esp, 12

  746. %else

  747.     add     rsp, 8+32*WIN64

  748. %endif

  749.     POP     r1

  750.     POP     r3

  751.     lea     r0, [r1 + 2*r3]

  752.     mov     r2, r3

  753.     sub     r3, mmsize

  754.     neg     r2

  755.     mova    m2, [ps_m1m1m1m1]

  756. .loop:

  757. %if mmsize == 8

  758.     PSWAPD  m0, [r1 + r3]

  759.     PSWAPD  m1, [r0 + r2]

  760.     pxor    m0, m2

  761. %else

  762.     mova    m0, [r1 + r3]

  763.     mova    m1, [r0 + r2]

  764.     shufps  m0, m0, 0x1b

  765.     shufps  m1, m1, 0x1b

  766.     xorps   m0, m2

  767. %endif

  768.     mova [r0 + r3], m1

  769.     mova [r1 + r2], m0

  770.     sub     r3, mmsize

  771.     add     r2, mmsize

  772.     jl      .loop

  773. %if cpuflag(3dnow)

  774.     femms

  775.     RET

  776. %else

  777.     REP_RET

  778. %endif

  779. %endmacro

  780. 

  781. %if ARCH_X86_32

  782. INIT_MMX 3dnow

  783. IMDCT_CALC_FUNC

  784. INIT_MMX 3dnowext

  785. IMDCT_CALC_FUNC

  786. %endif

  787. 

  788. INIT_XMM sse

  789. IMDCT_CALC_FUNC

  790. 

  791. INIT_XMM sse

  792. %undef mulps

  793. %undef addps

  794. %undef subps

  795. %undef unpcklps

  796. %undef unpckhps

  797. 

  798. %macro PREROTATER 5 ;-2*k, 2*k, input+n4, tcos+n8, tsin+n8

  799. %if mmsize == 8 ; j*2+2-n4, n4-2-j*2, input+n4, tcos+n8, tsin+n8

  800.     PSWAPD     m0, [%3+%2*4]

  801.     movq       m2, [%3+%1*4-8]

  802.     movq       m3, m0

  803.     punpckldq  m0, m2

  804.     punpckhdq  m2, m3

  805.     movd       m1, [%4+%1*2-4] ; tcos[j]

  806.     movd       m3, [%4+%2*2]   ; tcos[n4-j-1]

  807.     punpckldq  m1, [%5+%1*2-4] ; tsin[j]

  808.     punpckldq  m3, [%5+%2*2]   ; tsin[n4-j-1]

  809. 

  810.     mova       m4, m0

  811.     PSWAPD     m5, m1

  812.     pfmul      m0, m1

  813.     pfmul      m4, m5

  814.     mova       m6, m2

  815.     PSWAPD     m5, m3

  816.     pfmul      m2, m3

  817.     pfmul      m6, m5

  818. %if cpuflag(3dnowext)

  819.     pfpnacc    m0, m4

  820.     pfpnacc    m2, m6

  821. %else

  822.     SBUTTERFLY dq, 0, 4, 1

  823.     SBUTTERFLY dq, 2, 6, 3

  824.     pxor       m4, m7

  825.     pxor       m6, m7

  826.     pfadd      m0, m4

  827.     pfadd      m2, m6

  828. %endif

  829. %else

  830.     movaps   xmm0, [%3+%2*4]

  831.     movaps   xmm1, [%3+%1*4-0x10]

  832.     movaps   xmm2, xmm0

  833.     shufps   xmm0, xmm1, 0x88

  834.     shufps   xmm1, xmm2, 0x77

  835.     movlps   xmm4, [%4+%2*2]

  836.     movlps   xmm5, [%5+%2*2+0x0]

  837.     movhps   xmm4, [%4+%1*2-0x8]

  838.     movhps   xmm5, [%5+%1*2-0x8]

  839.     movaps   xmm2, xmm0

  840.     movaps   xmm3, xmm1

  841.     mulps    xmm0, xmm5

  842.     mulps    xmm1, xmm4

  843.     mulps    xmm2, xmm4

  844.     mulps    xmm3, xmm5

  845.     subps    xmm1, xmm0

  846.     addps    xmm2, xmm3

  847.     movaps   xmm0, xmm1

  848.     unpcklps xmm1, xmm2

  849.     unpckhps xmm0, xmm2

  850. %endif

  851. %endmacro

  852. 

  853. %macro CMUL 6 ;j, xmm0, xmm1, 3, 4, 5

  854. %if cpuflag(sse)

  855.     mulps      m6, %3, [%5+%1]

  856.     mulps      m7, %2, [%5+%1]

  857.     mulps      %2, %2, [%6+%1]

  858.     mulps      %3, %3, [%6+%1]

  859.     subps      %2, %2, m6

  860.     addps      %3, %3, m7

  861. %elif cpuflag(3dnow)

  862.     mova       m6, [%1+%2*2]

  863.     mova       %3, [%1+%2*2+8]

  864.     mova       %4, m6

  865.     mova       m7, %3

  866.     pfmul      m6, [%5+%2]

  867.     pfmul      %3, [%6+%2]

  868.     pfmul      %4, [%6+%2]

  869.     pfmul      m7, [%5+%2]

  870.     pfsub      %3, m6

  871.     pfadd      %4, m7

  872. %endif

  873. %endmacro

  874. 

  875. %macro POSROTATESHUF 5 ;j, k, z+n8, tcos+n8, tsin+n8

  876. .post:

  877. %if cpuflag(avx)

  878.     vmovaps      ymm1,   [%3+%1*2]

  879.     vmovaps      ymm0,   [%3+%1*2+0x20]

  880.     vmovaps      ymm3,   [%3+%2*2]

  881.     vmovaps      ymm2,   [%3+%2*2+0x20]

  882. 

  883.     CMUL         %1, ymm0, ymm1, %3, %4, %5

  884.     CMUL         %2, ymm2, ymm3, %3, %4, %5

  885.     vshufps      ymm1, ymm1, ymm1, 0x1b

  886.     vshufps      ymm3, ymm3, ymm3, 0x1b

  887.     vperm2f128   ymm1, ymm1, ymm1, 0x01

  888.     vperm2f128   ymm3, ymm3, ymm3, 0x01

  889.     vunpcklps    ymm6, ymm2, ymm1

  890.     vunpckhps    ymm4, ymm2, ymm1

  891.     vunpcklps    ymm7, ymm0, ymm3

  892.     vunpckhps    ymm5, ymm0, ymm3

  893. 

  894.     vextractf128 [%3+%1*2],      ymm7, 0

  895.     vextractf128 [%3+%1*2+0x10], ymm5, 0

  896.     vextractf128 [%3+%1*2+0x20], ymm7, 1

  897.     vextractf128 [%3+%1*2+0x30], ymm5, 1

  898. 

  899.     vextractf128 [%3+%2*2],      ymm6, 0

  900.     vextractf128 [%3+%2*2+0x10], ymm4, 0

  901.     vextractf128 [%3+%2*2+0x20], ymm6, 1

  902.     vextractf128 [%3+%2*2+0x30], ymm4, 1

  903.     sub      %2,   0x20

  904.     add      %1,   0x20

  905.     jl       .post

  906. %elif cpuflag(sse)

  907.     movaps   xmm1, [%3+%1*2]

  908.     movaps   xmm0, [%3+%1*2+0x10]

  909.     CMUL     %1,   xmm0, xmm1, %3, %4, %5

  910.     movaps   xmm5, [%3+%2*2]

  911.     movaps   xmm4, [%3+%2*2+0x10]

  912.     CMUL     %2,   xmm4, xmm5, %3, %4, %5

  913.     shufps   xmm1, xmm1, 0x1b

  914.     shufps   xmm5, xmm5, 0x1b

  915.     movaps   xmm6, xmm4

  916.     unpckhps xmm4, xmm1

  917.     unpcklps xmm6, xmm1

  918.     movaps   xmm2, xmm0

  919.     unpcklps xmm0, xmm5

  920.     unpckhps xmm2, xmm5

  921.     movaps   [%3+%2*2],      xmm6

  922.     movaps   [%3+%2*2+0x10], xmm4

  923.     movaps   [%3+%1*2],      xmm0

  924.     movaps   [%3+%1*2+0x10], xmm2

  925.     sub      %2,   0x10

  926.     add      %1,   0x10

  927.     jl       .post

  928. %elif cpuflag(3dnow)

  929.     CMUL  %3, %1, m0, m1, %4, %5

  930.     CMUL  %3, %2, m2, m3, %4, %5

  931.     movd  [%3+%1*2+ 0], m0

  932.     movd  [%3+%2*2+12], m1

  933.     movd  [%3+%2*2+ 0], m2

  934.     movd  [%3+%1*2+12], m3

  935.     psrlq      m0, 32

  936.     psrlq      m1, 32

  937.     psrlq      m2, 32

  938.     psrlq      m3, 32

  939.     movd  [%3+%1*2+ 8], m0

  940.     movd  [%3+%2*2+ 4], m1

  941.     movd  [%3+%2*2+ 8], m2

  942.     movd  [%3+%1*2+ 4], m3

  943.     sub        %2, 8

  944.     add        %1, 8

  945.     jl         .post

  946. %endif

  947. %endmacro

  948. 

  949. %macro DECL_IMDCT 0

  950. cglobal imdct_half, 3,12,8; FFTContext *s, FFTSample *output, const FFTSample *input

  951. %if ARCH_X86_64

  952. %define rrevtab r7

  953. %define rtcos   r8

  954. %define rtsin   r9

  955. %else

  956. %define rrevtab r6

  957. %define rtsin   r6

  958. %define rtcos   r5

  959. %endif

  960.     mov   r3d, [r0+FFTContext.mdctsize]

  961.     add   r2, r3

  962.     shr   r3, 1

  963.     mov   rtcos, [r0+FFTContext.tcos]

  964.     mov   rtsin, [r0+FFTContext.tsin]

  965.     add   rtcos, r3

  966.     add   rtsin, r3

  967. %if ARCH_X86_64 == 0

  968.     push  rtcos

  969.     push  rtsin

  970. %endif

  971.     shr   r3, 1

  972.     mov   rrevtab, [r0+FFTContext.revtab]

  973.     add   rrevtab, r3

  974. %if ARCH_X86_64 == 0

  975.     push  rrevtab

  976. %endif

  977. 

  978. %if mmsize == 8

  979.     sub   r3, 2

  980. %else

  981.     sub   r3, 4

  982. %endif

  983. %if ARCH_X86_64 || mmsize == 8

  984.     xor   r4, r4

  985.     sub   r4, r3

  986. %endif

  987. %if notcpuflag(3dnowext) && mmsize == 8

  988.     movd  m7, [ps_m1m1m1m1]

  989. %endif

  990. .pre:

  991. %if ARCH_X86_64 == 0

  992. ;unspill

  993. %if mmsize != 8

  994.     xor   r4, r4

  995.     sub   r4, r3

  996. %endif

  997.     mov   rtcos, [esp+8]

  998.     mov   rtsin, [esp+4]

  999. %endif

  1000. 

  1001.     PREROTATER r4, r3, r2, rtcos, rtsin

  1002. %if mmsize == 8

  1003.     mov    r6, [esp]                ; rrevtab = ptr+n8

  1004.     movzx  r5,  word [rrevtab+r4-2] ; rrevtab[j]

  1005.     movzx  r6,  word [rrevtab+r3]   ; rrevtab[n4-j-1]

  1006.     mova [r1+r5*8], m0

  1007.     mova [r1+r6*8], m2

  1008.     add    r4, 2

  1009.     sub    r3, 2

  1010. %else

  1011. %if ARCH_X86_64

  1012.     movzx  r5,  word [rrevtab+r4-4]

  1013.     movzx  r6,  word [rrevtab+r4-2]

  1014.     movzx  r10, word [rrevtab+r3]

  1015.     movzx  r11, word [rrevtab+r3+2]

  1016.     movlps [r1+r5 *8], xmm0

  1017.     movhps [r1+r6 *8], xmm0

  1018.     movlps [r1+r10*8], xmm1

  1019.     movhps [r1+r11*8], xmm1

  1020.     add    r4, 4

  1021. %else

  1022.     mov    r6, [esp]

  1023.     movzx  r5, word [r6+r4-4]

  1024.     movzx  r4, word [r6+r4-2]

  1025.     movlps [r1+r5*8], xmm0

  1026.     movhps [r1+r4*8], xmm0

  1027.     movzx  r5, word [r6+r3]

  1028.     movzx  r4, word [r6+r3+2]

  1029.     movlps [r1+r5*8], xmm1

  1030.     movhps [r1+r4*8], xmm1

  1031. %endif

  1032.     sub    r3, 4

  1033. %endif

  1034.     jns    .pre

  1035. 

  1036.     mov  r5, r0

  1037.     mov  r6, r1

  1038.     mov  r0, r1

  1039.     mov  r1d, [r5+FFTContext.nbits]

  1040. 

  1041.     FFT_DISPATCH SUFFIX, r1

  1042. 

  1043.     mov  r0d, [r5+FFTContext.mdctsize]

  1044.     add  r6, r0

  1045.     shr  r0, 1

  1046. %if ARCH_X86_64 == 0

  1047. %define rtcos r2

  1048. %define rtsin r3

  1049.     mov  rtcos, [esp+8]

  1050.     mov  rtsin, [esp+4]

  1051. %endif

  1052.     neg  r0

  1053.     mov  r1, -mmsize

  1054.     sub  r1, r0

  1055.     POSROTATESHUF r0, r1, r6, rtcos, rtsin

  1056. %if ARCH_X86_64 == 0

  1057.     add esp, 12

  1058. %endif

  1059. %if mmsize == 8

  1060.     femms

  1061. %endif

  1062.     RET

  1063. %endmacro

  1064. 

  1065. DECL_IMDCT

  1066. 

  1067. %if ARCH_X86_32

  1068. INIT_MMX 3dnow

  1069. DECL_IMDCT

  1070. 

  1071. INIT_MMX 3dnowext

  1072. DECL_IMDCT

  1073. %endif

  1074. 

  1075. INIT_YMM avx

  1076. DECL_IMDCT

  1077. 

  1078. %endif ; CONFIG_MDCT