DISTRHO
/
JUCE
mirror of https://github.com/DISTRHO/JUCE
1
0
Fork 0
Code Releases 1 Activity
The JUCE cross-platform C++ framework, with DISTRHO/KXStudio specific changes
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
8865 Commits
11 Branches
1.4GB
Tree: bfd3afa6cf
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfd3afa6cf'
${ noResults }
JUCE/modules/juce_dsp/native/juce_neon_SIMDNativeOps.h

467 lines
32KB
Raw Blame History 

  1. /*

  2.   ==============================================================================

  3. 

  4.    This file is part of the JUCE library.

  5.    Copyright (c) 2017 - ROLI Ltd.

  6. 

  7.    JUCE is an open source library subject to commercial or open-source

  8.    licensing.

  9. 

  10.    By using JUCE, you agree to the terms of both the JUCE 5 End-User License

  11.    Agreement and JUCE 5 Privacy Policy (both updated and effective as of the

  12.    27th April 2017).

  13. 

  14.    End User License Agreement: www.juce.com/juce-5-licence

  15.    Privacy Policy: www.juce.com/juce-5-privacy-policy

  16. 

  17.    Or: You may also use this code under the terms of the GPL v3 (see

  18.    www.gnu.org/licenses).

  19. 

  20.    JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER

  21.    EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE

  22.    DISCLAIMED.

  23. 

  24.   ==============================================================================

  25. */

  26. 

  27. namespace juce

  28. {

  29. namespace dsp

  30. {

  31. 

  32. #ifndef DOXYGEN

  33. 

  34. #ifdef _MSC_VER

  35.  #define DECLARE_NEON_SIMD_CONST(type, name) \

  36.     static __declspec(align(16)) const type name [16 / sizeof (type)]

  37. 

  38.  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \

  39.     __declspec(align(16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]

  40. 

  41. #else

  42.  #define DECLARE_NEON_SIMD_CONST(type, name) \

  43.     static const type name [16 / sizeof (type)] __attribute__((aligned(16)))

  44. 

  45.  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \

  46.     const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__((aligned(16)))

  47. 

  48. #endif

  49. 

  50. template <typename type>

  51. struct SIMDNativeOps;

  52. 

  53. //==============================================================================

  54. /** Unsigned 32-bit integer NEON intrinsics.

  55. 

  56.     @tags{DSP}

  57. */

  58. template <>

  59. struct SIMDNativeOps<uint32_t>

  60. {

  61.     //==============================================================================

  62.     typedef uint32x4_t vSIMDType;

  63.     typedef SIMDFallbackOps<uint32_t, vSIMDType> fb;

  64. 

  65.     //==============================================================================

  66.     DECLARE_NEON_SIMD_CONST (uint32_t, kAllBitsSet);

  67. 

  68.     //==============================================================================

  69.     static forcedinline vSIMDType expand (uint32_t s) noexcept                   { return vdupq_n_u32 (s); }

  70.     static forcedinline vSIMDType load (const uint32_t* a) noexcept              { return vld1q_u32 (a); }

  71.     static forcedinline void store (vSIMDType value, uint32_t* a) noexcept       { vst1q_u32 (a, value); }

  72.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_u32 (a, b); }

  73.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_u32 (a, b); }

  74.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_u32 (a, b); }

  75.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_u32 (a, b); }

  76.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_u32  (a, b); }

  77.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_u32 (a, b); }

  78.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u32 (b, a); }

  79.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_u32 ((uint32_t*) kAllBitsSet)); }

  80.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_u32 (a, b); }

  81.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_u32 (a, b); }

  82.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_u32 (a, b); }

  83.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (sum (notEqual (a, b)) == 0); }

  84.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  85.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_u32 (a, b); }

  86.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_u32 (a, b); }

  87.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_u32 (a, b, c); }

  88.     static forcedinline uint32_t sum (vSIMDType a) noexcept

  89.     {

  90.         auto rr = vadd_u32 (vget_high_u32 (a), vget_low_u32 (a));

  91.         return vget_lane_u32 (vpadd_u32 (rr, rr), 0);

  92.     }

  93. };

  94. 

  95. //==============================================================================

  96. /** Signed 32-bit integer NEON intrinsics.

  97. 

  98.     @tags{DSP}

  99. */

  100. template <>

  101. struct SIMDNativeOps<int32_t>

  102. {

  103.     //==============================================================================

  104.     typedef int32x4_t vSIMDType;

  105.     typedef SIMDFallbackOps<int32_t, vSIMDType> fb;

  106. 

  107.     //==============================================================================

  108.     DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);

  109. 

  110.     //==============================================================================

  111.     static forcedinline vSIMDType expand (int32_t s) noexcept                    { return vdupq_n_s32 (s); }

  112.     static forcedinline vSIMDType load (const int32_t* a) noexcept               { return vld1q_s32 (a); }

  113.     static forcedinline void store (vSIMDType value, int32_t* a) noexcept        { vst1q_s32 (a, value); }

  114.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_s32 (a, b); }

  115.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_s32 (a, b); }

  116.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_s32 (a, b); }

  117.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_s32 (a, b); }

  118.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_s32 (a, b); }

  119.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_s32 (a, b); }

  120.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s32 (b, a); }

  121.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_s32 ((int32_t*) kAllBitsSet)); }

  122.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_s32 (a, b); }

  123.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_s32 (a, b); }

  124.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_s32 (a, b); }

  125.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (sum (notEqual (a, b)) == 0); }

  126.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  127.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_s32 (a, b); }

  128.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_s32 (a, b); }

  129.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_s32 (a, b, c); }

  130.     static forcedinline int32_t sum (vSIMDType a) noexcept

  131.     {

  132.         auto rr = vadd_s32 (vget_high_s32 (a), vget_low_s32 (a));

  133.         rr = vpadd_s32 (rr, rr);

  134.         return vget_lane_s32 (rr, 0);

  135.     }

  136. };

  137. 

  138. //==============================================================================

  139. /** Signed 8-bit integer NEON intrinsics.

  140. 

  141.     @tags{DSP}

  142. */

  143. template <>

  144. struct SIMDNativeOps<int8_t>

  145. {

  146.     //==============================================================================

  147.     typedef int8x16_t vSIMDType;

  148.     typedef SIMDFallbackOps<int8_t, vSIMDType> fb;

  149. 

  150.     //==============================================================================

  151.     DECLARE_NEON_SIMD_CONST (int8_t, kAllBitsSet);

  152. 

  153.     //==============================================================================

  154.     static forcedinline vSIMDType expand (int8_t s) noexcept                     { return vdupq_n_s8 (s); }

  155.     static forcedinline vSIMDType load (const int8_t* a) noexcept                { return vld1q_s8 (a); }

  156.     static forcedinline void store (vSIMDType value, int8_t* a) noexcept         { vst1q_s8 (a, value); }

  157.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_s8 (a, b); }

  158.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_s8 (a, b); }

  159.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_s8 (a, b); }

  160.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_s8 (a, b); }

  161.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_s8 (a, b); }

  162.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_s8 (a, b); }

  163.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s8 (b, a); }

  164.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_s8 ((int8_t*) kAllBitsSet)); }

  165.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_s8 (a, b); }

  166.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_s8 (a, b); }

  167.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_s8 (a, b); }

  168.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  169.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_s8 (a, b); }

  170.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_s8 (a, b); }

  171.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<int32_t>::sum (notEqual (a, b)) == 0); }

  172.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_s8 (a, b, c); }

  173.     static forcedinline int8_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  174. };

  175. 

  176. //==============================================================================

  177. /** Unsigned 8-bit integer NEON intrinsics.

  178. 

  179.     @tags{DSP}

  180. */

  181. template <>

  182. struct SIMDNativeOps<uint8_t>

  183. {

  184.     //==============================================================================

  185.     typedef uint8x16_t vSIMDType;

  186.     typedef SIMDFallbackOps<uint8_t, vSIMDType> fb;

  187. 

  188.     //==============================================================================

  189.     DECLARE_NEON_SIMD_CONST (uint8_t, kAllBitsSet);

  190. 

  191.     //==============================================================================

  192.     static forcedinline vSIMDType expand (uint8_t s) noexcept                    { return vdupq_n_u8 (s); }

  193.     static forcedinline vSIMDType load (const uint8_t* a) noexcept               { return vld1q_u8 (a); }

  194.     static forcedinline void store (vSIMDType value, uint8_t* a) noexcept        { vst1q_u8 (a, value); }

  195.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_u8 (a, b); }

  196.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_u8 (a, b); }

  197.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_u8 (a, b); }

  198.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_u8 (a, b); }

  199.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_u8 (a, b); }

  200.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_u8 (a, b); }

  201.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u8 (b, a); }

  202.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_u8 ((uint8_t*) kAllBitsSet)); }

  203.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_u8 (a, b); }

  204.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_u8 (a, b); }

  205.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_u8 (a, b); }

  206.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  207.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_u8 (a, b); }

  208.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_u8 (a, b); }

  209.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<uint32_t>::sum (notEqual (a, b)) == 0); }

  210.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_u8 (a, b, c); }

  211.     static forcedinline uint8_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  212. };

  213. 

  214. //==============================================================================

  215. /** Signed 16-bit integer NEON intrinsics.

  216. 

  217.     @tags{DSP}

  218. */

  219. template <>

  220. struct SIMDNativeOps<int16_t>

  221. {

  222.     //==============================================================================

  223.     typedef int16x8_t vSIMDType;

  224.     typedef SIMDFallbackOps<int16_t, vSIMDType> fb;

  225. 

  226.     //==============================================================================

  227.     DECLARE_NEON_SIMD_CONST (int16_t, kAllBitsSet);

  228. 

  229.     //==============================================================================

  230.     static forcedinline vSIMDType expand (int16_t s) noexcept                    { return vdupq_n_s16 (s); }

  231.     static forcedinline vSIMDType load (const int16_t* a) noexcept               { return vld1q_s16 (a); }

  232.     static forcedinline void store (vSIMDType value, int16_t* a) noexcept        { vst1q_s16 (a, value); }

  233.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_s16 (a, b); }

  234.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_s16 (a, b); }

  235.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_s16 (a, b); }

  236.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_s16 (a, b); }

  237.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_s16 (a, b); }

  238.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_s16 (a, b); }

  239.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s16 (b, a); }

  240.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_s16 ((int16_t*) kAllBitsSet)); }

  241.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_s16 (a, b); }

  242.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_s16 (a, b); }

  243.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_s16 (a, b); }

  244.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  245.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_s16 (a, b); }

  246.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_s16 (a, b); }

  247.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<int32_t>::sum (notEqual (a, b)) == 0); }

  248.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_s16 (a, b, c); }

  249.     static forcedinline int16_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  250. };

  251. 

  252. 

  253. //==============================================================================

  254. /** Unsigned 16-bit integer NEON intrinsics.

  255. 

  256.     @tags{DSP}

  257. */

  258. template <>

  259. struct SIMDNativeOps<uint16_t>

  260. {

  261.     //==============================================================================

  262.     typedef uint16x8_t vSIMDType;

  263.     typedef SIMDFallbackOps<uint16_t, vSIMDType> fb;

  264. 

  265.     //==============================================================================

  266.     DECLARE_NEON_SIMD_CONST (uint16_t, kAllBitsSet);

  267. 

  268.     //==============================================================================

  269.     static forcedinline vSIMDType expand (uint16_t s) noexcept                   { return vdupq_n_u16 (s); }

  270.     static forcedinline vSIMDType load (const uint16_t* a) noexcept              { return vld1q_u16 (a); }

  271.     static forcedinline void store (vSIMDType value, uint16_t* a) noexcept       { vst1q_u16 (a, value); }

  272.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_u16 (a, b); }

  273.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_u16 (a, b); }

  274.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_u16 (a, b); }

  275.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_u16 (a, b); }

  276.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_u16 (a, b); }

  277.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_u16 (a, b); }

  278.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u16 (b, a); }

  279.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_u16 ((uint16_t*) kAllBitsSet)); }

  280.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_u16 (a, b); }

  281.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_u16 (a, b); }

  282.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_u16 (a, b); }

  283.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  284.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_u16 (a, b); }

  285.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_u16 (a, b); }

  286.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<uint32_t>::sum (notEqual (a, b)) == 0); }

  287.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_u16 (a, b, c); }

  288.     static forcedinline uint16_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  289. };

  290. 

  291. //==============================================================================

  292. /** Signed 64-bit integer NEON intrinsics.

  293. 

  294.     @tags{DSP}

  295. */

  296. template <>

  297. struct SIMDNativeOps<int64_t>

  298. {

  299.     //==============================================================================

  300.     typedef int64x2_t vSIMDType;

  301.     typedef SIMDFallbackOps<int64_t, vSIMDType> fb;

  302. 

  303.     //==============================================================================

  304.     DECLARE_NEON_SIMD_CONST (int64_t, kAllBitsSet);

  305. 

  306.     //==============================================================================

  307.     static forcedinline vSIMDType expand (int64_t s) noexcept                    { return vdupq_n_s64 (s); }

  308.     static forcedinline vSIMDType load (const int64_t* a) noexcept               { return vld1q_s64 (a); }

  309.     static forcedinline void store (vSIMDType value, int64_t* a) noexcept        { vst1q_s64 (a, value); }

  310.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_s64 (a, b); }

  311.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_s64 (a, b); }

  312.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return fb::mul (a, b); }

  313.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_s64 (a, b); }

  314.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_s64 (a, b); }

  315.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_s64 (a, b); }

  316.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s64 (b, a); }

  317.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_s64 ((int64_t*) kAllBitsSet)); }

  318.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return fb::min (a, b); }

  319.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return fb::max (a, b); }

  320.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return fb::equal (a, b); }

  321.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return fb::notEqual (a, b); }

  322.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return fb::greaterThan (a, b); }

  323.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return fb::greaterThanOrEqual (a, b); }

  324.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<int32_t>::sum (notEqual (a, b)) == 0); }

  325.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return fb::multiplyAdd (a, b, c); }

  326.     static forcedinline int64_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  327. };

  328. 

  329. 

  330. //==============================================================================

  331. /** Unsigned 64-bit integer NEON intrinsics.

  332. 

  333.     @tags{DSP}

  334. */

  335. template <>

  336. struct SIMDNativeOps<uint64_t>

  337. {

  338.     //==============================================================================

  339.     typedef uint64x2_t vSIMDType;

  340.     typedef SIMDFallbackOps<uint64_t, vSIMDType> fb;

  341. 

  342.     //==============================================================================

  343.     DECLARE_NEON_SIMD_CONST (uint64_t, kAllBitsSet);

  344. 

  345.     //==============================================================================

  346.     static forcedinline vSIMDType expand (uint64_t s) noexcept                   { return vdupq_n_u64 (s); }

  347.     static forcedinline vSIMDType load (const uint64_t* a) noexcept              { return vld1q_u64 (a); }

  348.     static forcedinline void store (vSIMDType value, uint64_t* a) noexcept       { vst1q_u64 (a, value); }

  349.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_u64 (a, b); }

  350.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_u64 (a, b); }

  351.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return fb::mul (a, b); }

  352.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return vandq_u64 (a, b); }

  353.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return vorrq_u64 (a, b); }

  354.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return veorq_u64 (a, b); }

  355.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u64 (b, a); }

  356.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return bit_notand (a, vld1q_u64 ((uint64_t*) kAllBitsSet)); }

  357.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return fb::min (a, b); }

  358.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return fb::max (a, b); }

  359.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return fb::equal (a, b); }

  360.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return fb::notEqual (a, b); }

  361.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return fb::greaterThan (a, b); }

  362.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return fb::greaterThanOrEqual (a, b); }

  363.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<uint32_t>::sum (notEqual (a, b)) == 0); }

  364.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return fb::multiplyAdd (a, b, c); }

  365.     static forcedinline uint64_t sum (vSIMDType a) noexcept { return fb::sum (a); }

  366. };

  367. 

  368.     //==============================================================================

  369. /** Single-precision floating point NEON intrinsics.

  370. 

  371.     @tags{DSP}

  372. */

  373. template <>

  374. struct SIMDNativeOps<float>

  375. {

  376.     //==============================================================================

  377.     typedef float32x4_t vSIMDType;

  378.     typedef uint32x4_t vMaskType;

  379.     typedef SIMDFallbackOps<float, vSIMDType> fb;

  380. 

  381.     //==============================================================================

  382.     DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);

  383.     DECLARE_NEON_SIMD_CONST (int32_t, kEvenHighBit);

  384.     DECLARE_NEON_SIMD_CONST (float, kOne);

  385. 

  386.     //==============================================================================

  387.     static forcedinline vSIMDType expand (float s) noexcept                          { return vdupq_n_f32 (s); }

  388.     static forcedinline vSIMDType load (const float* a) noexcept                     { return vld1q_f32 (a); }

  389.     static forcedinline void store (vSIMDType value, float* a) noexcept              { vst1q_f32 (a, value); }

  390.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return vaddq_f32 (a, b); }

  391.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return vsubq_f32 (a, b); }

  392.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return vmulq_f32 (a, b); }

  393.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return (vSIMDType) vandq_u32 ((vMaskType) a, (vMaskType) b); }

  394.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return (vSIMDType) vorrq_u32 ((vMaskType) a, (vMaskType) b); }

  395.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return (vSIMDType) veorq_u32 ((vMaskType) a, (vMaskType) b); }

  396.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vbicq_u32 ((vMaskType) b, (vMaskType) a); }

  397.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                   { return bit_notand (a, vld1q_f32 ((float*) kAllBitsSet)); }

  398.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return vminq_f32 (a, b); }

  399.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return vmaxq_f32 (a, b); }

  400.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vceqq_f32 (a, b); }

  401.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return bit_not (equal (a, b)); }

  402.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return (vSIMDType) vcgtq_f32 (a, b); }

  403.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return (vSIMDType) vcgeq_f32 (a, b); }

  404.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return (SIMDNativeOps<uint32_t>::sum (notEqual (a, b)) == 0); }

  405.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_f32 (a, b, c); }

  406.     static forcedinline vSIMDType dupeven (vSIMDType a) noexcept { return fb::shuffle<(0 << 0) | (0 << 2) | (2 << 4) | (2 << 6)>     (a); }

  407.     static forcedinline vSIMDType dupodd  (vSIMDType a) noexcept { return fb::shuffle<(1 << 0) | (1 << 2) | (3 << 4) | (3 << 6)>     (a); }

  408.     static forcedinline vSIMDType swapevenodd (vSIMDType a) noexcept { return fb::shuffle<(1 << 0) | (0 << 2) | (3 << 4) | (2 << 6)> (a); }

  409.     static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept { return add (fb::shuffle<(2 << 0) | (3 << 2) | (0 << 4) | (1 << 6)> (a), a); }

  410. 

  411.     //==============================================================================

  412.     static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept

  413.     {

  414.         vSIMDType rr_ir = mul (a, dupeven (b));

  415.         vSIMDType ii_ri = mul (swapevenodd (a), dupodd (b));

  416.         return add (rr_ir, bit_xor (ii_ri, vld1q_f32 ((float*) kEvenHighBit)));

  417.     }

  418. 

  419.     static forcedinline float sum (vSIMDType a) noexcept

  420.     {

  421.         auto rr = vadd_f32 (vget_high_f32 (a), vget_low_f32 (a));

  422.         return vget_lane_f32 (vpadd_f32 (rr, rr), 0);

  423.     }

  424. };

  425. 

  426. //==============================================================================

  427. /** Double-precision floating point NEON intrinsics does not exist in NEON

  428.     so we need to emulate this.

  429. 

  430.     @tags{DSP}

  431. */

  432. template <>

  433. struct SIMDNativeOps<double>

  434. {

  435.     //==============================================================================

  436.     typedef struct { double values [2]; } vSIMDType;

  437.     typedef SIMDFallbackOps<double, vSIMDType> fb;

  438. 

  439.     static forcedinline vSIMDType expand (double s) noexcept                     { return fb::expand (s); }

  440.     static forcedinline vSIMDType load (const double* a) noexcept                { return fb::load (a); }

  441.     static forcedinline void store (vSIMDType value, double* a) noexcept         { fb::store (value, a); }

  442.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept        { return fb::add (a, b); }

  443.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept        { return fb::sub (a, b); }

  444.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept        { return fb::mul (a, b); }

  445.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept    { return fb::bit_and (a, b); }

  446.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept    { return fb::bit_or  (a, b); }

  447.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept    { return fb::bit_xor (a, b); }

  448.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return fb::bit_notand (a, b); }

  449.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                 { return fb::bit_not (a); }

  450.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                    { return fb::min (a, b); }

  451.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                    { return fb::max (a, b); }

  452.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                  { return fb::equal (a, b); }

  453.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept               { return fb::notEqual (a, b); }

  454.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept            { return fb::greaterThan (a, b); }

  455.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept     { return fb::greaterThanOrEqual (a, b); }

  456.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept               { return fb::allEqual (a, b); }

  457.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }

  458.     static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept { return fb::cmplxmul (a, b); }

  459.     static forcedinline double sum (vSIMDType a) noexcept { return fb::sum (a); }

  460.     static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept { return a; }

  461. };

  462. 

  463. #endif

  464. 

  465. } // namespace dsp

  466. } // namespace juce