You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

648 lines
22KB

  1. /*
  2. * Copyright (c) 2014 Janne Grunau <janne-libav@jannau.net>
  3. *
  4. * This file is part of Libav.
  5. *
  6. * Libav is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2.1 of the License, or (at your option) any later version.
  10. *
  11. * Libav is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with Libav; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19. */
  20. #include "libavutil/aarch64/asm.S"
  21. #include "asm-offsets.h"
  22. .macro shuffle a, b, c, d
  23. const shuffle_\a\b\c\d, align=4
  24. .byte (\a * 4), (\a * 4 + 1), (\a * 4 + 2), (\a * 4 + 3)
  25. .byte (\b * 4), (\b * 4 + 1), (\b * 4 + 2), (\b * 4 + 3)
  26. .byte (\c * 4), (\c * 4 + 1), (\c * 4 + 2), (\c * 4 + 3)
  27. .byte (\d * 4), (\d * 4 + 1), (\d * 4 + 2), (\d * 4 + 3)
  28. endconst
  29. .endm
  30. shuffle 0, 2, 1, 3
  31. shuffle 1, 0, 3, 2
  32. shuffle 2, 3, 0, 1
  33. shuffle 3, 1, 2, 0
  34. function fft5_neon
  35. lsl x2, x2, #3
  36. ld1 {v24.2s}, [x1], x2
  37. ld2 {v25.s,v26.s}[0], [x1], x2
  38. ld2 {v25.s,v26.s}[1], [x1], x2
  39. ld2 {v25.s,v26.s}[2], [x1], x2
  40. ld2 {v25.s,v26.s}[3], [x1]
  41. dup v6.4s, v24.s[0]
  42. dup v7.4s, v24.s[1]
  43. faddp v0.4s, v25.4s, v26.4s
  44. // z[][0], z[][3]
  45. fmul v16.4s, v25.4s, v15.s[0] // rr
  46. fmul v17.4s, v25.4s, v15.s[1] // ri
  47. fmul v18.4s, v26.4s, v15.s[0] // ir
  48. fmul v19.4s, v26.4s, v15.s[1] // ii
  49. faddp v0.4s, v0.4s, v0.4s
  50. // z[][1], z[][2]
  51. fmul v20.4s, v25.4s, v15.s[2] // rr
  52. fmul v21.4s, v25.4s, v15.s[3] // ri
  53. fmul v22.4s, v26.4s, v15.s[2] // ir
  54. fmul v23.4s, v26.4s, v15.s[3] // ii
  55. fadd v0.2s, v24.2s, v0.2s // out[0]
  56. // z[0123][0], z[0123][3]
  57. fsub v24.4s, v16.4s, v19.4s // (c).re = rr - ii;
  58. fadd v27.4s, v16.4s, v19.4s // (d).re = rr + ii;
  59. ld1 {v16.16b}, [x11]
  60. ld1 {v19.16b}, [x14]
  61. fadd v28.4s, v17.4s, v18.4s // (c).im = ri + ir;
  62. fsub v31.4s, v18.4s, v17.4s // (d).im = -ri + ir;
  63. ld1 {v17.16b}, [x12]
  64. // z[0123][1], z[0123][2]
  65. fsub v25.4s, v20.4s, v23.4s // (c).re = rr - ii;
  66. fadd v26.4s, v20.4s, v23.4s // (d).re = rr + ii;
  67. ld1 {v18.16b}, [x13]
  68. fadd v29.4s, v21.4s, v22.4s // (c).im = ri + ir;
  69. fsub v30.4s, v22.4s, v21.4s // (d).im = -ri + ir;
  70. //real
  71. tbl v20.16b, {v24.16b}, v16.16b
  72. tbl v21.16b, {v25.16b}, v17.16b
  73. tbl v22.16b, {v26.16b}, v18.16b
  74. tbl v23.16b, {v27.16b}, v19.16b
  75. //imag
  76. tbl v16.16b, {v28.16b}, v16.16b
  77. tbl v17.16b, {v29.16b}, v17.16b
  78. tbl v18.16b, {v30.16b}, v18.16b
  79. tbl v19.16b, {v31.16b}, v19.16b
  80. fadd v6.4s, v6.4s, v20.4s
  81. fadd v22.4s, v22.4s, v23.4s
  82. fadd v7.4s, v7.4s, v16.4s
  83. fadd v18.4s, v18.4s, v19.4s
  84. fadd v21.4s, v21.4s, v22.4s
  85. fadd v17.4s, v17.4s, v18.4s
  86. fadd v6.4s, v6.4s, v21.4s
  87. fadd v7.4s, v7.4s, v17.4s
  88. ret
  89. endfunc
  90. function fft15_neon
  91. mov x8, x1
  92. mov x9, x30
  93. add x2, x3, x3, lsl #1 // 3 * stride
  94. add x1, x8, x3, lsl #3 // in + 1 * stride
  95. bl fft5_neon
  96. mov v1.8b, v0.8b
  97. mov v2.16b, v6.16b
  98. mov v3.16b, v7.16b
  99. add x1, x8, x3, lsl #4 // in + 2 * stride
  100. add x2, x3, x3, lsl #1 // 3 * stride
  101. bl fft5_neon
  102. zip1 v1.4s, v1.4s, v0.4s
  103. mov v4.16b, v6.16b
  104. mov v5.16b, v7.16b
  105. mov x1, x8 // in + 0 * stride
  106. add x2, x3, x3, lsl #1 // 3 * stride
  107. bl fft5_neon
  108. faddp v20.4s, v1.4s, v1.4s
  109. ext v18.16b, v8.16b, v8.16b, #4
  110. ext v19.16b, v9.16b, v9.16b, #4
  111. mov v16.16b, v6.16b
  112. mov v17.16b, v7.16b
  113. fadd v20.2s, v20.2s, v0.2s
  114. uzp1 v18.4s, v18.4s, v10.4s // exp[2,4,6,8].re
  115. uzp1 v19.4s, v19.4s, v11.4s // exp[2,4,6,8].im
  116. st1 {v20.2s}, [x0], #8 // out[0]
  117. fmla v16.4s, v2.4s, v8.4s
  118. fmls v16.4s, v3.4s, v9.4s
  119. fmla v17.4s, v2.4s, v9.4s
  120. fmla v17.4s, v3.4s, v8.4s
  121. fmla v16.4s, v4.4s, v18.4s
  122. fmls v16.4s, v5.4s, v19.4s
  123. fmla v17.4s, v4.4s, v19.4s
  124. fmla v17.4s, v5.4s, v18.4s
  125. zip1 v18.4s, v16.4s, v17.4s
  126. zip2 v19.4s, v16.4s, v17.4s
  127. rev64 v31.4s, v14.4s
  128. trn1 v28.2d, v1.2d, v1.2d
  129. trn2 v29.2d, v1.2d, v1.2d
  130. zip1 v30.2d, v14.2d, v31.2d
  131. zip2 v31.2d, v14.2d, v31.2d
  132. st1 {v18.4s,v19.4s}, [x0], #32 // out[1-4]
  133. fmul v16.4s, v28.4s, v30.4s
  134. fmul v17.4s, v29.4s, v30.4s
  135. fmls v16.4s, v29.4s, v31.4s
  136. fmla v17.4s, v28.4s, v31.4s
  137. faddp v16.4s, v16.4s, v16.4s
  138. faddp v17.4s, v17.4s, v17.4s
  139. zip1 v18.2s, v16.2s, v17.2s
  140. zip2 v19.2s, v16.2s, v17.2s
  141. fadd v18.2s, v18.2s, v0.2s
  142. fadd v0.2s, v19.2s, v0.2s
  143. ext v30.16b, v12.16b, v12.16b, #4
  144. ext v31.16b, v13.16b, v13.16b, #4
  145. mov v16.16b, v6.16b
  146. mov v17.16b, v7.16b
  147. uzp1 v30.4s, v30.4s, v8.4s
  148. uzp1 v31.4s, v31.4s, v9.4s
  149. st1 {v18.2s}, [x0], #8 // out[5]
  150. fmla v16.4s, v2.4s, v10.4s
  151. fmls v16.4s, v3.4s, v11.4s
  152. fmla v17.4s, v2.4s, v11.4s
  153. fmla v17.4s, v3.4s, v10.4s
  154. fmla v16.4s, v4.4s, v30.4s
  155. fmls v16.4s, v5.4s, v31.4s
  156. fmla v17.4s, v4.4s, v31.4s
  157. fmla v17.4s, v5.4s, v30.4s
  158. zip1 v18.4s, v16.4s, v17.4s
  159. zip2 v19.4s, v16.4s, v17.4s
  160. ext v30.16b, v10.16b, v10.16b, #4
  161. ext v31.16b, v11.16b, v11.16b, #4
  162. fmla v6.4s, v2.4s, v12.4s
  163. fmls v6.4s, v3.4s, v13.4s
  164. st1 {v18.4s,v19.4s}, [x0], #32 // out[6-9]
  165. uzp1 v30.4s, v30.4s, v12.4s
  166. uzp1 v31.4s, v31.4s, v13.4s
  167. fmla v7.4s, v2.4s, v13.4s
  168. fmla v7.4s, v3.4s, v12.4s
  169. st1 {v0.2s}, [x0], #8 // out[10]
  170. fmla v6.4s, v4.4s, v30.4s
  171. fmls v6.4s, v5.4s, v31.4s
  172. fmla v7.4s, v4.4s, v31.4s
  173. fmla v7.4s, v5.4s, v30.4s
  174. zip1 v18.4s, v6.4s, v7.4s
  175. zip2 v19.4s, v6.4s, v7.4s
  176. st1 {v18.4s,v19.4s}, [x0], #32 // out[11-14]
  177. ret x9
  178. endfunc
  179. // x0: out, x1: out+len2, x2: exptab, x3: len2
  180. function fft15_pass
  181. ands x6, x3, #3
  182. mov x4, x0
  183. mov x5, x1
  184. b.eq 9f
  185. ld1 {v0.2s}, [x0], #8
  186. ld1 {v1.2s}, [x1], #8
  187. sub x3, x3, x6
  188. subs x6, x6, #1
  189. fadd v2.2s, v0.2s, v1.2s
  190. fsub v3.2s, v0.2s, v1.2s
  191. add x2, x2, #8
  192. st1 {v2.2s}, [x4], #8
  193. st1 {v3.2s}, [x5], #8
  194. b.eq 9f
  195. 1:
  196. subs x6, x6, #1
  197. ldp s4, s5, [x2], #8
  198. ldp s2, s3, [x1], #8
  199. ldp s0, s1, [x0], #8
  200. fmul s6, s2, s4
  201. fmul s7, s2, s5
  202. fmls s6, s3, v5.s[0]
  203. fmla s7, s3, v4.s[0]
  204. fsub s2, s0, s6
  205. fsub s3, s1, s7
  206. fadd s0, s0, s6
  207. fadd s1, s1, s7
  208. stp s2, s3, [x5], #8
  209. stp s0, s1, [x4], #8
  210. b.gt 1b
  211. 9:
  212. ld1 {v4.4s,v5.4s}, [x2], #32
  213. ld2 {v2.4s,v3.4s}, [x1], #32
  214. uzp1 v6.4s, v4.4s, v5.4s
  215. uzp2 v7.4s, v4.4s, v5.4s
  216. ld2 {v0.4s,v1.4s}, [x0], #32
  217. 8:
  218. subs x3, x3, #8
  219. fmul v4.4s, v2.4s, v6.4s
  220. fmul v5.4s, v2.4s, v7.4s
  221. b.lt 4f
  222. ld1 {v18.4s,v19.4s}, [x2], #32
  223. fmls v4.4s, v3.4s, v7.4s
  224. fmla v5.4s, v3.4s, v6.4s
  225. ld2 {v22.4s,v23.4s}, [x1], #32
  226. fsub v2.4s, v0.4s, v4.4s
  227. fadd v0.4s, v0.4s, v4.4s
  228. fsub v3.4s, v1.4s, v5.4s
  229. fadd v1.4s, v1.4s, v5.4s
  230. uzp1 v16.4s, v18.4s, v19.4s
  231. uzp2 v17.4s, v18.4s, v19.4s
  232. st2 {v2.4s,v3.4s}, [x5], #32
  233. st2 {v0.4s,v1.4s}, [x4], #32
  234. ld2 {v20.4s,v21.4s}, [x0], #32
  235. fmul v18.4s, v22.4s, v16.4s
  236. fmul v19.4s, v22.4s, v17.4s
  237. b.eq 0f
  238. ld1 {v4.4s,v5.4s}, [x2], #32
  239. fmls v18.4s, v23.4s, v17.4s
  240. fmla v19.4s, v23.4s, v16.4s
  241. ld2 {v2.4s,v3.4s}, [x1], #32
  242. fsub v22.4s, v20.4s, v18.4s
  243. fadd v20.4s, v20.4s, v18.4s
  244. fsub v23.4s, v21.4s, v19.4s
  245. fadd v21.4s, v21.4s, v19.4s
  246. uzp1 v6.4s, v4.4s, v5.4s
  247. uzp2 v7.4s, v4.4s, v5.4s
  248. st2 {v22.4s,v23.4s}, [x5], #32
  249. st2 {v20.4s,v21.4s}, [x4], #32
  250. ld2 {v0.4s,v1.4s}, [x0], #32
  251. b 8b
  252. 4:
  253. fmls v4.4s, v3.4s, v7.4s
  254. fmla v5.4s, v3.4s, v6.4s
  255. fsub v2.4s, v0.4s, v4.4s
  256. fadd v0.4s, v0.4s, v4.4s
  257. fsub v3.4s, v1.4s, v5.4s
  258. fadd v1.4s, v1.4s, v5.4s
  259. st2 {v2.4s,v3.4s}, [x5], #32
  260. st2 {v0.4s,v1.4s}, [x4], #32
  261. ret
  262. 0:
  263. fmls v18.4s, v23.4s, v17.4s
  264. fmla v19.4s, v23.4s, v16.4s
  265. fsub v22.4s, v20.4s, v18.4s
  266. fadd v20.4s, v20.4s, v18.4s
  267. fsub v23.4s, v21.4s, v19.4s
  268. fadd v21.4s, v21.4s, v19.4s
  269. st2 {v22.4s,v23.4s}, [x5], #32
  270. st2 {v20.4s,v21.4s}, [x4], #32
  271. ret
  272. endfunc
  273. function fft30_neon, align=6
  274. sub sp, sp, #0x20
  275. stp x20, x21, [sp]
  276. stp x22, x30, [sp, #0x10]
  277. mov x21, x1
  278. mov x22, x2
  279. mov x20, x4
  280. mov x0, x21
  281. mov x1, x22
  282. lsl x3, x20, #1
  283. bl fft15_neon
  284. add x0, x21, #15*8
  285. add x1, x22, x20, lsl #3
  286. lsl x3, x20, #1
  287. bl fft15_neon
  288. ldr x2, [x10, #(CELT_EXPTAB + 8)] // s->exptab[1]
  289. add x0, x21, #0
  290. add x1, x21, #15*8
  291. mov x3, #15
  292. ldp x20, x21, [sp]
  293. ldp x22, x30, [sp, #0x10]
  294. add sp, sp, #0x20
  295. b fft15_pass
  296. endfunc
  297. .macro def_fft n, n2
  298. function fft\n\()_neon, align=6
  299. sub sp, sp, #0x30
  300. stp x20, x21, [sp]
  301. stp x22, x30, [sp, #0x10]
  302. stp x23, x24, [sp, #0x20]
  303. mov x21, x1
  304. mov x22, x2
  305. mov x23, x3
  306. mov x20, x4
  307. sub x3, x3, #1
  308. lsl x4, x4, #1
  309. bl fft\n2\()_neon
  310. add x1, x21, #(\n2 * 8)
  311. add x2, x22, x20, lsl #3
  312. sub x3, x23, #1
  313. lsl x4, x20, #1
  314. bl fft\n2\()_neon
  315. add x5, x10, #CELT_EXPTAB
  316. mov x0, x21
  317. ldr x2, [x5, x23, lsl #3] // s->exptab[N]
  318. add x1, x21, #(\n2 * 8)
  319. mov x3, #\n2
  320. ldp x20, x21, [sp]
  321. ldp x22, x30, [sp, #0x10]
  322. ldp x23, x24, [sp, #0x20]
  323. add sp, sp, #0x30
  324. b fft15_pass
  325. endfunc
  326. .endm
  327. def_fft 60, 30
  328. def_fft 120, 60
  329. def_fft 240, 120
  330. def_fft 480, 240
  331. def_fft 960, 480
  332. function fft_b15_calc_neon
  333. sub sp, sp, #0x50
  334. ldr x8, [x0, #CELT_EXPTAB] // s->exptab[0]
  335. movrel x6, fact5
  336. movrel x11, shuffle_0213
  337. movrel x12, shuffle_1032
  338. movrel x13, shuffle_2301
  339. movrel x14, shuffle_3120
  340. add x8, x8, #8
  341. movrel x5, fft_tab_neon
  342. stp x20, x30, [sp]
  343. stp d8, d9, [sp, #0x10]
  344. stp d10, d11, [sp, #0x20]
  345. stp d12, d13, [sp, #0x30]
  346. stp d14, d15, [sp, #0x40]
  347. ld1 {v15.4s}, [x6]
  348. ld1 {v0.4s,v1.4s}, [x8], #32
  349. ld1 {v6.2s}, [x8], #8
  350. ld1 {v2.4s,v3.4s}, [x8], #32
  351. ld1 {v7.2s}, [x8], #8
  352. ld1 {v4.4s,v5.4s}, [x8], #32
  353. uzp1 v8.4s, v0.4s, v1.4s // exp[ 1 - 4].re
  354. uzp2 v9.4s, v0.4s, v1.4s // exp[ 1 - 4].im
  355. uzp1 v10.4s, v2.4s, v3.4s // exp[ 6 - 9].re
  356. uzp2 v11.4s, v2.4s, v3.4s // exp[ 6 - 9].im
  357. uzp1 v12.4s, v4.4s, v5.4s // exp[11 - 14].re
  358. uzp2 v13.4s, v4.4s, v5.4s // exp[11 - 14].im
  359. zip1 v14.4s, v6.4s, v7.4s // exp[5,10].re/exp[5,10].im
  360. add x5, x5, x3, lsl #3
  361. ldr x5, [x5]
  362. mov x10, x0
  363. blr x5
  364. ldp x20, x30, [sp]
  365. ldp d8, d9, [sp, #0x10]
  366. ldp d10, d11, [sp, #0x20]
  367. ldp d12, d13, [sp, #0x30]
  368. ldp d14, d15, [sp, #0x40]
  369. add sp, sp, #0x50
  370. ret
  371. endfunc
  372. const fft_tab_neon, relocate=1
  373. .quad fft15_neon
  374. .quad fft30_neon
  375. .quad fft60_neon
  376. .quad fft120_neon
  377. .quad fft240_neon
  378. .quad fft480_neon
  379. .quad fft960_neon
  380. endconst
  381. function ff_celt_imdct_half_neon, export=1
  382. sub sp, sp, #0x20
  383. stp x21, x30, [sp]
  384. str s0, [sp, #0x10]
  385. ldp w5, w6, [x0, #CELT_LEN2] // CELT_LEN4
  386. mov x10, x0
  387. mov x21, x1
  388. sub w5, w5, #1
  389. lsl x7, x3, #3 // 2 * stride * sizeof(float)
  390. sub x8, xzr, x3, lsl #3 // -2 * stride * sizeof(float)
  391. mul x5, x5, x3
  392. ldp x9, x10, [x0, #CELT_TMP] // CELT_TWIDDLE
  393. ldr w3, [x0, #CELT_FFT_N]
  394. add x5, x2, x5, lsl #2
  395. mov x11, x9
  396. sub w6, w6, #4
  397. ld1 {v0.s}[0], [x5], x8
  398. ld1 {v1.s}[0], [x2], x7
  399. ld1 {v4.4s,v5.4s}, [x10], #32
  400. ld1 {v0.s}[1], [x5], x8
  401. ld1 {v1.s}[1], [x2], x7
  402. uzp1 v2.4s, v4.4s, v5.4s
  403. ld1 {v0.s}[2], [x5], x8
  404. ld1 {v1.s}[2], [x2], x7
  405. uzp2 v3.4s, v4.4s, v5.4s
  406. ld1 {v0.s}[3], [x5], x8
  407. ld1 {v1.s}[3], [x2], x7
  408. 1:
  409. subs w6, w6, #4
  410. ld1 {v20.s}[0], [x5], x8
  411. ld1 {v21.s}[0], [x2], x7
  412. ld1 {v4.4s,v5.4s}, [x10], #32
  413. fmul v6.4s, v0.4s, v2.4s
  414. fmul v7.4s, v0.4s, v3.4s
  415. ld1 {v20.s}[1], [x5], x8
  416. ld1 {v21.s}[1], [x2], x7
  417. fmls v6.4s, v1.4s, v3.4s
  418. fmla v7.4s, v1.4s, v2.4s
  419. ld1 {v20.s}[2], [x5], x8
  420. ld1 {v21.s}[2], [x2], x7
  421. uzp1 v2.4s, v4.4s, v5.4s
  422. uzp2 v3.4s, v4.4s, v5.4s
  423. ld1 {v20.s}[3], [x5], x8
  424. ld1 {v21.s}[3], [x2], x7
  425. zip1 v4.4s, v6.4s, v7.4s
  426. zip2 v5.4s, v6.4s, v7.4s
  427. fmul v6.4s, v20.4s, v2.4s
  428. fmul v7.4s, v20.4s, v3.4s
  429. st1 {v4.4s,v5.4s}, [x9], #32
  430. fmls v6.4s, v21.4s, v3.4s
  431. fmla v7.4s, v21.4s, v2.4s
  432. b.eq 3f
  433. subs w6, w6, #4
  434. ld1 {v4.4s,v5.4s}, [x10], #32
  435. ld1 {v0.s}[0], [x5], x8
  436. ld1 {v1.s}[0], [x2], x7
  437. uzp1 v2.4s, v4.4s, v5.4s
  438. ld1 {v0.s}[1], [x5], x8
  439. ld1 {v1.s}[1], [x2], x7
  440. uzp2 v3.4s, v4.4s, v5.4s
  441. ld1 {v0.s}[2], [x5], x8
  442. ld1 {v1.s}[2], [x2], x7
  443. zip1 v4.4s, v6.4s, v7.4s
  444. zip2 v5.4s, v6.4s, v7.4s
  445. ld1 {v0.s}[3], [x5], x8
  446. ld1 {v1.s}[3], [x2], x7
  447. st1 {v4.4s,v5.4s}, [x9], #32
  448. b.gt 1b
  449. fmul v6.4s, v0.4s, v2.4s
  450. fmul v7.4s, v0.4s, v3.4s
  451. fmls v6.4s, v1.4s, v3.4s
  452. fmla v7.4s, v1.4s, v2.4s
  453. 3:
  454. zip1 v4.4s, v6.4s, v7.4s
  455. zip2 v5.4s, v6.4s, v7.4s
  456. st1 {v4.4s,v5.4s}, [x9], #32
  457. mov x2, x11
  458. mov x4, #1
  459. bl fft_b15_calc_neon
  460. ldr w5, [x10, #CELT_LEN4]
  461. ldr x6, [x10, #CELT_TWIDDLE]
  462. ldr s31, [sp, #0x10]
  463. add x1, x21, x5, lsl #2
  464. add x3, x6, x5, lsl #2
  465. sub x0, x1, #16
  466. sub x2, x3, #16
  467. mov x8, #-16
  468. mov x7, #16
  469. mov x10, x0
  470. mov x11, x1
  471. sub w5, w5, #4
  472. ld1 {v0.4s}, [x0], x8
  473. ld1 {v1.4s}, [x1], x7
  474. ld1 {v2.4s}, [x2], x8
  475. ld1 {v3.4s}, [x3], x7
  476. uzp1 v4.4s, v0.4s, v1.4s // z[-i-2, -i-1, +i, i+1].re
  477. uzp2 v6.4s, v0.4s, v1.4s // z[-i-2, -i-1, +i, i+1].im
  478. uzp1 v5.4s, v2.4s, v3.4s // twidlle_exptab[-i-2, -i-1, +i, i+1].re
  479. uzp2 v7.4s, v2.4s, v3.4s // twidlle_exptab[-i-2, -i-1, +i, i+1].im
  480. fmul v1.4s, v6.4s, v5.4s
  481. fmul v0.4s, v6.4s, v7.4s
  482. 2:
  483. subs w5, w5, #4
  484. ld1 {v20.4s}, [x0], x8
  485. fmla v1.4s, v4.4s, v7.4s
  486. fmls v0.4s, v4.4s, v5.4s
  487. ld1 {v21.4s}, [x1], x7
  488. ext v1.16b, v1.16b, v1.16b, #8
  489. fmul v0.4s, v0.4s, v31.s[0]
  490. ld1 {v2.4s}, [x2], x8
  491. rev64 v1.4s, v1.4s
  492. fmul v1.4s, v1.4s, v31.s[0]
  493. ld1 {v3.4s}, [x3], x7
  494. zip1 v5.4s, v0.4s, v1.4s
  495. zip2 v7.4s, v0.4s, v1.4s
  496. uzp1 v4.4s, v20.4s, v21.4s // z[-i-2, -i-1, +i, i+1].re
  497. uzp2 v6.4s, v20.4s, v21.4s // z[-i-2, -i-1, +i, i+1].im
  498. st1 {v5.4s}, [x10], x8
  499. st1 {v7.4s}, [x11], x7
  500. uzp1 v5.4s, v2.4s, v3.4s // twidlle_exptab[-i-2, -i-1, +i, i+1].re
  501. uzp2 v7.4s, v2.4s, v3.4s // twidlle_exptab[-i-2, -i-1, +i, i+1].im
  502. fmul v1.4s, v6.4s, v5.4s
  503. fmul v0.4s, v6.4s, v7.4s
  504. b.gt 2b
  505. fmla v1.4s, v4.4s, v7.4s
  506. fmls v0.4s, v4.4s, v5.4s
  507. ext v1.16b, v1.16b, v1.16b, #8
  508. fmul v0.4s, v0.4s, v31.s[0]
  509. rev64 v1.4s, v1.4s
  510. fmul v1.4s, v1.4s, v31.s[0]
  511. zip1 v5.4s, v0.4s, v1.4s
  512. zip2 v7.4s, v0.4s, v1.4s
  513. st1 {v5.4s}, [x10], x8
  514. st1 {v7.4s}, [x11], x7
  515. ldp x21, x30, [sp]
  516. add sp, sp, #0x20
  517. ret
  518. endfunc
  519. // [0] = exp(2 * i * pi / 5), [1] = exp(2 * i * pi * 2 / 5)
  520. const fact5, align=4
  521. .float 0.30901699437494745, 0.95105651629515353
  522. .float -0.80901699437494734, 0.58778525229247325
  523. endconst