DISTRHO
/
JUCE
mirror of https://github.com/DISTRHO/JUCE
1
0
Fork 0
Code Releases 1 Activity
The JUCE cross-platform C++ framework, with DISTRHO/KXStudio specific changes
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
13953 Commits
9 Branches
3.6GB
Tree: 158220ddfa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '158220ddfa'
${ noResults }
JUCE/modules/juce_dsp/native/juce_SIMDNativeOps_neon.h

540 lines
42KB
Raw Blame History 

  1. /*

  2.   ==============================================================================

  3. 

  4.    This file is part of the JUCE library.

  5.    Copyright (c) 2022 - Raw Material Software Limited

  6. 

  7.    JUCE is an open source library subject to commercial or open-source

  8.    licensing.

  9. 

  10.    By using JUCE, you agree to the terms of both the JUCE 7 End-User License

  11.    Agreement and JUCE Privacy Policy.

  12. 

  13.    End User License Agreement: www.juce.com/juce-7-licence

  14.    Privacy Policy: www.juce.com/juce-privacy-policy

  15. 

  16.    Or: You may also use this code under the terms of the GPL v3 (see

  17.    www.gnu.org/licenses).

  18. 

  19.    JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER

  20.    EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE

  21.    DISCLAIMED.

  22. 

  23.   ==============================================================================

  24. */

  25. 

  26. namespace juce::dsp

  27. {

  28. 

  29. #ifndef DOXYGEN

  30. 

  31. JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wignored-attributes")

  32. 

  33. #ifdef _MSC_VER

  34.  #define DECLARE_NEON_SIMD_CONST(type, name) \

  35.     static __declspec (align (16)) const type name [16 / sizeof (type)]

  36. 

  37.  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \

  38.     __declspec (align (16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]

  39. 

  40. #else

  41.  #define DECLARE_NEON_SIMD_CONST(type, name) \

  42.     static const type name [16 / sizeof (type)] __attribute__ ((aligned (16)))

  43. 

  44.  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \

  45.     const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__ ((aligned (16)))

  46. 

  47. #endif

  48. 

  49. template <typename type>

  50. struct SIMDNativeOps;

  51. 

  52. //==============================================================================

  53. /** Unsigned 32-bit integer NEON intrinsics.

  54. 

  55.     @tags{DSP}

  56. */

  57. template <>

  58. struct SIMDNativeOps<uint32_t>

  59. {

  60.     //==============================================================================

  61.     using vSIMDType = uint32x4_t;

  62.     using fb = SIMDFallbackOps<uint32_t, vSIMDType>;

  63. 

  64.     //==============================================================================

  65.     DECLARE_NEON_SIMD_CONST (uint32_t, kAllBitsSet);

  66. 

  67.     //==============================================================================

  68.     static forcedinline vSIMDType expand (uint32_t s) noexcept                                  { return vdupq_n_u32 (s); }

  69.     static forcedinline vSIMDType load (const uint32_t* a) noexcept                             { return vld1q_u32 (a); }

  70.     static forcedinline void      store (vSIMDType value, uint32_t* a) noexcept                 { vst1q_u32 (a, value); }

  71.     static forcedinline uint32_t  get (vSIMDType v, size_t i) noexcept                          { return fb::get (v, i); }

  72.     static forcedinline vSIMDType set (vSIMDType v, size_t i, uint32_t s) noexcept              { return fb::set (v, i, s); }

  73.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                       { return vaddq_u32 (a, b); }

  74.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                       { return vsubq_u32 (a, b); }

  75.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                       { return vmulq_u32 (a, b); }

  76.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                   { return vandq_u32 (a, b); }

  77.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                   { return vorrq_u32  (a, b); }

  78.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                   { return veorq_u32 (a, b); }

  79.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept                { return vbicq_u32 (b, a); }

  80.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                                { return bit_notand (a, vld1q_u32 ((uint32_t*) kAllBitsSet)); }

  81.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                       { return vminq_u32 (a, b); }

  82.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                       { return vmaxq_u32 (a, b); }

  83.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                     { return (vSIMDType) vceqq_u32 (a, b); }

  84.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                  { return (sum (notEqual (a, b)) == 0); }

  85.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                  { return bit_not (equal (a, b)); }

  86.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept               { return (vSIMDType) vcgtq_u32 (a, b); }

  87.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept        { return (vSIMDType) vcgeq_u32 (a, b); }

  88.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_u32 (a, b, c); }

  89.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                               { return a; }

  90. 

  91.     static forcedinline uint32_t sum (vSIMDType a) noexcept

  92.     {

  93.         auto rr = vadd_u32 (vget_high_u32 (a), vget_low_u32 (a));

  94.         return vget_lane_u32 (vpadd_u32 (rr, rr), 0);

  95.     }

  96. };

  97. 

  98. //==============================================================================

  99. /** Signed 32-bit integer NEON intrinsics.

  100. 

  101.     @tags{DSP}

  102. */

  103. template <>

  104. struct SIMDNativeOps<int32_t>

  105. {

  106.     //==============================================================================

  107.     using vSIMDType = int32x4_t;

  108.     using fb = SIMDFallbackOps<int32_t, vSIMDType>;

  109. 

  110.     //==============================================================================

  111.     DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);

  112. 

  113.     //==============================================================================

  114.     static forcedinline vSIMDType expand (int32_t s) noexcept                                   { return vdupq_n_s32 (s); }

  115.     static forcedinline vSIMDType load (const int32_t* a) noexcept                              { return vld1q_s32 (a); }

  116.     static forcedinline void      store (vSIMDType value, int32_t* a) noexcept                  { vst1q_s32 (a, value); }

  117.     static forcedinline int32_t   get (vSIMDType v, size_t i) noexcept                          { return fb::get (v, i); }

  118.     static forcedinline vSIMDType set (vSIMDType v, size_t i, int32_t s) noexcept               { return fb::set (v, i, s); }

  119.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                       { return vaddq_s32 (a, b); }

  120.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                       { return vsubq_s32 (a, b); }

  121.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                       { return vmulq_s32 (a, b); }

  122.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                   { return vandq_s32 (a, b); }

  123.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                   { return vorrq_s32 (a, b); }

  124.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                   { return veorq_s32 (a, b); }

  125.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept                { return vbicq_s32 (b, a); }

  126.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                                { return bit_notand (a, vld1q_s32 ((int32_t*) kAllBitsSet)); }

  127.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                       { return vminq_s32 (a, b); }

  128.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                       { return vmaxq_s32 (a, b); }

  129.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                     { return (vSIMDType) vceqq_s32 (a, b); }

  130.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                  { return (sum (notEqual (a, b)) == 0); }

  131.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                  { return bit_not (equal (a, b)); }

  132.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept               { return (vSIMDType) vcgtq_s32 (a, b); }

  133.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept        { return (vSIMDType) vcgeq_s32 (a, b); }

  134.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return vmlaq_s32 (a, b, c); }

  135.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                               { return a; }

  136. 

  137.     static forcedinline int32_t sum (vSIMDType a) noexcept

  138.     {

  139.         auto rr = vadd_s32 (vget_high_s32 (a), vget_low_s32 (a));

  140.         rr = vpadd_s32 (rr, rr);

  141.         return vget_lane_s32 (rr, 0);

  142.     }

  143. };

  144. 

  145. //==============================================================================

  146. /** Signed 8-bit integer NEON intrinsics.

  147. 

  148.     @tags{DSP}

  149. */

  150. template <>

  151. struct SIMDNativeOps<int8_t>

  152. {

  153.     //==============================================================================

  154.     using vSIMDType = int8x16_t;

  155.     using fb = SIMDFallbackOps<int8_t, vSIMDType>;

  156. 

  157.     //==============================================================================

  158.     DECLARE_NEON_SIMD_CONST (int8_t, kAllBitsSet);

  159. 

  160.     //==============================================================================

  161.     static forcedinline vSIMDType expand (int8_t s) noexcept                                   { return vdupq_n_s8 (s); }

  162.     static forcedinline vSIMDType load (const int8_t* a) noexcept                              { return vld1q_s8 (a); }

  163.     static forcedinline void      store (vSIMDType value, int8_t* a) noexcept                  { vst1q_s8 (a, value); }

  164.     static forcedinline int8_t    get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  165.     static forcedinline vSIMDType set (vSIMDType v, size_t i, int8_t s) noexcept               { return fb::set (v, i, s); }

  166.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_s8 (a, b); }

  167.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_s8 (a, b); }

  168.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_s8 (a, b); }

  169.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return vandq_s8 (a, b); }

  170.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return vorrq_s8 (a, b); }

  171.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return veorq_s8 (a, b); }

  172.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return vbicq_s8 (b, a); }

  173.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_s8 ((int8_t*) kAllBitsSet)); }

  174.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_s8 (a, b); }

  175.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_s8 (a, b); }

  176.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_s8 (a, b); }

  177.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  178.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_s8 (a, b); }

  179.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_s8 (a, b); }

  180.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }

  181.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_s8 (a, b, c); }

  182.     static forcedinline int8_t    sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  183.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return a; }

  184. };

  185. 

  186. //==============================================================================

  187. /** Unsigned 8-bit integer NEON intrinsics.

  188. 

  189.     @tags{DSP}

  190. */

  191. template <>

  192. struct SIMDNativeOps<uint8_t>

  193. {

  194.     //==============================================================================

  195.     using vSIMDType = uint8x16_t;

  196.     using fb = SIMDFallbackOps<uint8_t, vSIMDType>;

  197. 

  198.     //==============================================================================

  199.     DECLARE_NEON_SIMD_CONST (uint8_t, kAllBitsSet);

  200. 

  201.     //==============================================================================

  202.     static forcedinline vSIMDType expand (uint8_t s) noexcept                                  { return vdupq_n_u8 (s); }

  203.     static forcedinline vSIMDType load (const uint8_t* a) noexcept                             { return vld1q_u8 (a); }

  204.     static forcedinline void      store (vSIMDType value, uint8_t* a) noexcept                 { vst1q_u8 (a, value); }

  205.     static forcedinline uint8_t   get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  206.     static forcedinline vSIMDType set (vSIMDType v, size_t i, uint8_t s) noexcept              { return fb::set (v, i, s); }

  207.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_u8 (a, b); }

  208.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_u8 (a, b); }

  209.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_u8 (a, b); }

  210.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return vandq_u8 (a, b); }

  211.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return vorrq_u8 (a, b); }

  212.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return veorq_u8 (a, b); }

  213.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return vbicq_u8 (b, a); }

  214.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_u8 ((uint8_t*) kAllBitsSet)); }

  215.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_u8 (a, b); }

  216.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_u8 (a, b); }

  217.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_u8 (a, b); }

  218.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  219.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_u8 (a, b); }

  220.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_u8 (a, b); }

  221.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }

  222.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_u8 (a, b, c); }

  223.     static forcedinline uint8_t   sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  224.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return a; }

  225. };

  226. 

  227. //==============================================================================

  228. /** Signed 16-bit integer NEON intrinsics.

  229. 

  230.     @tags{DSP}

  231. */

  232. template <>

  233. struct SIMDNativeOps<int16_t>

  234. {

  235.     //==============================================================================

  236.     using vSIMDType = int16x8_t;

  237.     using fb = SIMDFallbackOps<int16_t, vSIMDType>;

  238. 

  239.     //==============================================================================

  240.     DECLARE_NEON_SIMD_CONST (int16_t, kAllBitsSet);

  241. 

  242.     //==============================================================================

  243.     static forcedinline vSIMDType expand (int16_t s) noexcept                                  { return vdupq_n_s16 (s); }

  244.     static forcedinline vSIMDType load (const int16_t* a) noexcept                             { return vld1q_s16 (a); }

  245.     static forcedinline void      store (vSIMDType value, int16_t* a) noexcept                 { vst1q_s16 (a, value); }

  246.     static forcedinline int16_t   get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  247.     static forcedinline vSIMDType set (vSIMDType v, size_t i, int16_t s) noexcept              { return fb::set (v, i, s); }

  248.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_s16 (a, b); }

  249.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_s16 (a, b); }

  250.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_s16 (a, b); }

  251.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return vandq_s16 (a, b); }

  252.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return vorrq_s16 (a, b); }

  253.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return veorq_s16 (a, b); }

  254.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return vbicq_s16 (b, a); }

  255.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_s16 ((int16_t*) kAllBitsSet)); }

  256.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_s16 (a, b); }

  257.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_s16 (a, b); }

  258.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_s16 (a, b); }

  259.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  260.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_s16 (a, b); }

  261.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_s16 (a, b); }

  262.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }

  263.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_s16 (a, b, c); }

  264.     static forcedinline int16_t   sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  265.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return a; }

  266. };

  267. 

  268. 

  269. //==============================================================================

  270. /** Unsigned 16-bit integer NEON intrinsics.

  271. 

  272.     @tags{DSP}

  273. */

  274. template <>

  275. struct SIMDNativeOps<uint16_t>

  276. {

  277.     //==============================================================================

  278.     using vSIMDType = uint16x8_t;

  279.     using fb = SIMDFallbackOps<uint16_t, vSIMDType>;

  280. 

  281.     //==============================================================================

  282.     DECLARE_NEON_SIMD_CONST (uint16_t, kAllBitsSet);

  283. 

  284.     //==============================================================================

  285.     static forcedinline vSIMDType expand (uint16_t s) noexcept                                 { return vdupq_n_u16 (s); }

  286.     static forcedinline vSIMDType load (const uint16_t* a) noexcept                            { return vld1q_u16 (a); }

  287.     static forcedinline void      store (vSIMDType value, uint16_t* a) noexcept                { vst1q_u16 (a, value); }

  288.     static forcedinline uint16_t  get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  289.     static forcedinline vSIMDType set (vSIMDType v, size_t i, uint16_t s) noexcept             { return fb::set (v, i, s); }

  290.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_u16 (a, b); }

  291.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_u16 (a, b); }

  292.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_u16 (a, b); }

  293.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return vandq_u16 (a, b); }

  294.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return vorrq_u16 (a, b); }

  295.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return veorq_u16 (a, b); }

  296.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return vbicq_u16 (b, a); }

  297.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_u16 ((uint16_t*) kAllBitsSet)); }

  298.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_u16 (a, b); }

  299.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_u16 (a, b); }

  300.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_u16 (a, b); }

  301.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  302.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_u16 (a, b); }

  303.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_u16 (a, b); }

  304.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }

  305.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_u16 (a, b, c); }

  306.     static forcedinline uint16_t  sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  307.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return a; }

  308. };

  309. 

  310. //==============================================================================

  311. /** Signed 64-bit integer NEON intrinsics.

  312. 

  313.     @tags{DSP}

  314. */

  315. template <>

  316. struct SIMDNativeOps<int64_t>

  317. {

  318.     //==============================================================================

  319.     using vSIMDType = int64x2_t;

  320.     using fb = SIMDFallbackOps<int64_t, vSIMDType>;

  321. 

  322.     //==============================================================================

  323.     DECLARE_NEON_SIMD_CONST (int64_t, kAllBitsSet);

  324. 

  325.     //==============================================================================

  326.     static forcedinline vSIMDType expand (int64_t s) noexcept                                  { return vdupq_n_s64 (s); }

  327.     static forcedinline vSIMDType load (const int64_t* a) noexcept                             { return vld1q_s64 (a); }

  328.     static forcedinline void      store (vSIMDType value, int64_t* a) noexcept                 { vst1q_s64 (a, value); }

  329.     static forcedinline int64_t   get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  330.     static forcedinline vSIMDType set (vSIMDType v, size_t i, int64_t s) noexcept              { return fb::set (v, i, s); }

  331.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_s64 (a, b); }

  332.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_s64 (a, b); }

  333.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return fb::mul (a, b); }

  334.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return vandq_s64 (a, b); }

  335.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return vorrq_s64 (a, b); }

  336.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return veorq_s64 (a, b); }

  337.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return vbicq_s64 (b, a); }

  338.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_s64 ((int64_t*) kAllBitsSet)); }

  339.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return fb::min (a, b); }

  340.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return fb::max (a, b); }

  341.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return fb::equal (a, b); }

  342.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return fb::notEqual (a, b); }

  343.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return fb::greaterThan (a, b); }

  344.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return fb::greaterThanOrEqual (a, b); }

  345.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }

  346.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }

  347.     static forcedinline int64_t   sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  348.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return a; }

  349. };

  350. 

  351. 

  352. //==============================================================================

  353. /** Unsigned 64-bit integer NEON intrinsics.

  354. 

  355.     @tags{DSP}

  356. */

  357. template <>

  358. struct SIMDNativeOps<uint64_t>

  359. {

  360.     //==============================================================================

  361.     using vSIMDType = uint64x2_t;

  362.     using fb = SIMDFallbackOps<uint64_t, vSIMDType>;

  363. 

  364.     //==============================================================================

  365.     DECLARE_NEON_SIMD_CONST (uint64_t, kAllBitsSet);

  366. 

  367.     //==============================================================================

  368.     static forcedinline vSIMDType expand (uint64_t s) noexcept                                  { return vdupq_n_u64 (s); }

  369.     static forcedinline vSIMDType load (const uint64_t* a) noexcept                             { return vld1q_u64 (a); }

  370.     static forcedinline void      store (vSIMDType value, uint64_t* a) noexcept                 { vst1q_u64 (a, value); }

  371.     static forcedinline uint64_t  get (vSIMDType v, size_t i) noexcept                          { return fb::get (v, i); }

  372.     static forcedinline vSIMDType set (vSIMDType v, size_t i, uint64_t s) noexcept              { return fb::set (v, i, s); }

  373.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                       { return vaddq_u64 (a, b); }

  374.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                       { return vsubq_u64 (a, b); }

  375.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                       { return fb::mul (a, b); }

  376.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                   { return vandq_u64 (a, b); }

  377.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                   { return vorrq_u64 (a, b); }

  378.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                   { return veorq_u64 (a, b); }

  379.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept                { return vbicq_u64 (b, a); }

  380.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                                { return bit_notand (a, vld1q_u64 ((uint64_t*) kAllBitsSet)); }

  381.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                       { return fb::min (a, b); }

  382.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                       { return fb::max (a, b); }

  383.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                     { return fb::equal (a, b); }

  384.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                  { return fb::notEqual (a, b); }

  385.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept               { return fb::greaterThan (a, b); }

  386.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept        { return fb::greaterThanOrEqual (a, b); }

  387.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                  { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }

  388.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept  { return fb::multiplyAdd (a, b, c); }

  389.     static forcedinline uint64_t  sum (vSIMDType a) noexcept                                    { return fb::sum (a); }

  390.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                               { return a; }

  391. };

  392. 

  393.     //==============================================================================

  394. /** Single-precision floating point NEON intrinsics.

  395. 

  396.     @tags{DSP}

  397. */

  398. template <>

  399. struct SIMDNativeOps<float>

  400. {

  401.     //==============================================================================

  402.     using vSIMDType = float32x4_t;

  403.     using vMaskType = uint32x4_t;

  404.     using fb = SIMDFallbackOps<float, vSIMDType>;

  405. 

  406.     //==============================================================================

  407.     DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);

  408.     DECLARE_NEON_SIMD_CONST (int32_t, kEvenHighBit);

  409.     DECLARE_NEON_SIMD_CONST (float, kOne);

  410. 

  411.     //==============================================================================

  412.     static forcedinline vSIMDType expand (float s) noexcept                                    { return vdupq_n_f32 (s); }

  413.     static forcedinline vSIMDType load (const float* a) noexcept                               { return vld1q_f32 (a); }

  414.     static forcedinline float     get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  415.     static forcedinline vSIMDType set (vSIMDType v, size_t i, float s) noexcept                { return fb::set (v, i, s); }

  416.     static forcedinline void      store (vSIMDType value, float* a) noexcept                   { vst1q_f32 (a, value); }

  417.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_f32 (a, b); }

  418.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_f32 (a, b); }

  419.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_f32 (a, b); }

  420.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vandq_u32 ((vMaskType) a, (vMaskType) b); }

  421.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vorrq_u32 ((vMaskType) a, (vMaskType) b); }

  422.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) veorq_u32 ((vMaskType) a, (vMaskType) b); }

  423.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return (vSIMDType) vbicq_u32 ((vMaskType) b, (vMaskType) a); }

  424.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_f32 ((float*) kAllBitsSet)); }

  425.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_f32 (a, b); }

  426.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_f32 (a, b); }

  427.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_f32 (a, b); }

  428.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  429.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_f32 (a, b); }

  430.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_f32 (a, b); }

  431.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }

  432.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_f32 (a, b, c); }

  433.     static forcedinline vSIMDType dupeven (vSIMDType a) noexcept                               { return fb::shuffle<(0 << 0) | (0 << 2) | (2 << 4) | (2 << 6)>     (a); }

  434.     static forcedinline vSIMDType dupodd  (vSIMDType a) noexcept                               { return fb::shuffle<(1 << 0) | (1 << 2) | (3 << 4) | (3 << 6)>     (a); }

  435.     static forcedinline vSIMDType swapevenodd (vSIMDType a) noexcept                           { return fb::shuffle<(1 << 0) | (0 << 2) | (3 << 4) | (2 << 6)> (a); }

  436.     static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept                            { return add (fb::shuffle<(2 << 0) | (3 << 2) | (0 << 4) | (1 << 6)> (a), a); }

  437.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return vcvtq_f32_s32 (vcvtq_s32_f32 (a)); }

  438. 

  439.     //==============================================================================

  440.     static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept

  441.     {

  442.         vSIMDType rr_ir = mul (a, dupeven (b));

  443.         vSIMDType ii_ri = mul (swapevenodd (a), dupodd (b));

  444.         return add (rr_ir, bit_xor (ii_ri, vld1q_f32 ((float*) kEvenHighBit)));

  445.     }

  446. 

  447.     static forcedinline float sum (vSIMDType a) noexcept

  448.     {

  449.         auto rr = vadd_f32 (vget_high_f32 (a), vget_low_f32 (a));

  450.         return vget_lane_f32 (vpadd_f32 (rr, rr), 0);

  451.     }

  452. };

  453. 

  454. //==============================================================================

  455. /** Double-precision floating point NEON intrinsics does not exist in NEON

  456.     so we need to emulate this.

  457. 

  458.     @tags{DSP}

  459. */

  460. #if JUCE_64BIT

  461. template <>

  462. struct SIMDNativeOps<double>

  463. {

  464.     //==============================================================================

  465.     using vSIMDType = float64x2_t;

  466.     using vMaskType = uint64x2_t;

  467.     using fb = SIMDFallbackOps<double, vSIMDType>;

  468. 

  469.     //==============================================================================

  470.     DECLARE_NEON_SIMD_CONST (int64_t, kAllBitsSet);

  471.     DECLARE_NEON_SIMD_CONST (double, kOne);

  472. 

  473.     //==============================================================================

  474.     static forcedinline vSIMDType expand (double s) noexcept                                   { return vdupq_n_f64 (s); }

  475.     static forcedinline vSIMDType load (const double* a) noexcept                              { return vld1q_f64 (a); }

  476.     static forcedinline double    get (vSIMDType v, size_t i) noexcept                         { return fb::get (v, i); }

  477.     static forcedinline vSIMDType set (vSIMDType v, size_t i, double s) noexcept               { return fb::set (v, i, s); }

  478.     static forcedinline void      store (vSIMDType value, double* a) noexcept                  { vst1q_f64 (a, value); }

  479.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return vaddq_f64 (a, b); }

  480.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return vsubq_f64 (a, b); }

  481.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return vmulq_f64 (a, b); }

  482.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vandq_u64 ((vMaskType) a, (vMaskType) b); }

  483.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) vorrq_u64 ((vMaskType) a, (vMaskType) b); }

  484.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return (vSIMDType) veorq_u64 ((vMaskType) a, (vMaskType) b); }

  485.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return (vSIMDType) vbicq_u64 ((vMaskType) b, (vMaskType) a); }

  486.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return bit_notand (a, vld1q_f64 ((double*) kAllBitsSet)); }

  487.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return vminq_f64 (a, b); }

  488.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return vmaxq_f64 (a, b); }

  489.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return (vSIMDType) vceqq_f64 (a, b); }

  490.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return bit_not (equal (a, b)); }

  491.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return (vSIMDType) vcgtq_f64 (a, b); }

  492.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return (vSIMDType) vcgeq_f64 (a, b); }

  493.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }

  494.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_f64 (a, b, c); }

  495.     static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept                 { return fb::cmplxmul (a, b); }

  496.     static forcedinline double    sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  497.     static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept                            { return a; }

  498.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return vcvtq_f64_s64 (vcvtq_s64_f64 (a)); }

  499. };

  500. #else

  501. template <>

  502. struct SIMDNativeOps<double>

  503. {

  504.     //==============================================================================

  505.     using vSIMDType = struct { double v[2]; };

  506.     using fb = SIMDFallbackOps<double, vSIMDType>;

  507. 

  508.     static forcedinline vSIMDType expand (double s) noexcept                                   { return {{s, s}}; }

  509.     static forcedinline vSIMDType load (const double* a) noexcept                              { return {{a[0], a[1]}}; }

  510.     static forcedinline void      store (vSIMDType v, double* a) noexcept                      { a[0] = v.v[0]; a[1] = v.v[1]; }

  511.     static forcedinline double    get (vSIMDType v, size_t i) noexcept                         { return v.v[i]; }

  512.     static forcedinline vSIMDType set (vSIMDType v, size_t i, double s) noexcept               { v.v[i] = s; return v; }

  513.     static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept                      { return {{a.v[0] + b.v[0], a.v[1] + b.v[1]}}; }

  514.     static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept                      { return {{a.v[0] - b.v[0], a.v[1] - b.v[1]}}; }

  515.     static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept                      { return {{a.v[0] * b.v[0], a.v[1] * b.v[1]}}; }

  516.     static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept                  { return fb::bit_and (a, b); }

  517.     static forcedinline vSIMDType bit_or  (vSIMDType a, vSIMDType b) noexcept                  { return fb::bit_or  (a, b); }

  518.     static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept                  { return fb::bit_xor (a, b); }

  519.     static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept               { return fb::bit_notand (a, b); }

  520.     static forcedinline vSIMDType bit_not (vSIMDType a) noexcept                               { return fb::bit_not (a); }

  521.     static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept                      { return fb::min (a, b); }

  522.     static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept                      { return fb::max (a, b); }

  523.     static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept                    { return fb::equal (a, b); }

  524.     static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept                 { return fb::notEqual (a, b); }

  525.     static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept              { return fb::greaterThan (a, b); }

  526.     static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept       { return fb::greaterThanOrEqual (a, b); }

  527.     static forcedinline bool      allEqual (vSIMDType a, vSIMDType b) noexcept                 { return fb::allEqual (a, b); }

  528.     static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }

  529.     static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept                 { return fb::cmplxmul (a, b); }

  530.     static forcedinline double    sum (vSIMDType a) noexcept                                   { return fb::sum (a); }

  531.     static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept                            { return a; }

  532.     static forcedinline vSIMDType truncate (vSIMDType a) noexcept                              { return fb::truncate (a); }

  533. };

  534. #endif // JUCE_64BIT

  535. #endif // #ifndef DOXYGEN

  536. 

  537. JUCE_END_IGNORE_WARNINGS_GCC_LIKE

  538. 

  539. } // namespace juce::dsp