DISTRHO
/
JUCE
mirror of https://github.com/DISTRHO/JUCE
1
0
Fork 0
Code Releases 1 Activity
The JUCE cross-platform C++ framework, with DISTRHO/KXStudio specific changes
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
13953 Commits
9 Branches
3.6GB
Tree: 158220ddfa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '158220ddfa'
${ noResults }
JUCE/modules/juce_dsp/native/juce_SIMDNativeOps_sse.h

729 lines
52KB
Raw Blame History 

  1. /*

  2.   ==============================================================================

  3. 

  4.    This file is part of the JUCE library.

  5.    Copyright (c) 2022 - Raw Material Software Limited

  6. 

  7.    JUCE is an open source library subject to commercial or open-source

  8.    licensing.

  9. 

  10.    By using JUCE, you agree to the terms of both the JUCE 7 End-User License

  11.    Agreement and JUCE Privacy Policy.

  12. 

  13.    End User License Agreement: www.juce.com/juce-7-licence

  14.    Privacy Policy: www.juce.com/juce-privacy-policy

  15. 

  16.    Or: You may also use this code under the terms of the GPL v3 (see

  17.    www.gnu.org/licenses).

  18. 

  19.    JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER

  20.    EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE

  21.    DISCLAIMED.

  22. 

  23.   ==============================================================================

  24. */

  25. 

  26. namespace juce::dsp

  27. {

  28. 

  29. #ifndef DOXYGEN

  30. 

  31. JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wignored-attributes")

  32. 

  33. #ifdef _MSC_VER

  34.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  35.     static __declspec (align (16)) const type name [16 / sizeof (type)]

  36. 

  37.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  38.     __declspec (align (16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]

  39. 

  40. #else

  41.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  42.     static const type name [16 / sizeof (type)] __attribute__ ((aligned (16)))

  43. 

  44.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  45.     const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__ ((aligned (16)))

  46. 

  47. #endif

  48. 

  49. template <typename type>

  50. struct SIMDNativeOps;

  51. 

  52. //==============================================================================

  53. /** Single-precision floating point SSE intrinsics.

  54. 

  55.     @tags{DSP}

  56. */

  57. template <>

  58. struct SIMDNativeOps<float>

  59. {

  60.     //==============================================================================

  61.     using vSIMDType = __m128;

  62. 

  63.     //==============================================================================

  64.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  65.     DECLARE_SSE_SIMD_CONST (int32_t, kEvenHighBit);

  66.     DECLARE_SSE_SIMD_CONST (float, kOne);

  67. 

  68.     //==============================================================================

  69.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE expand (float s) noexcept                            { return _mm_load1_ps (&s); }

  70.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE load (const float* a) noexcept                       { return _mm_load_ps (a); }

  71.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128 value, float* dest) noexcept             { _mm_store_ps (dest, value); }

  72.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE add (__m128 a, __m128 b) noexcept                    { return _mm_add_ps (a, b); }

  73.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE sub (__m128 a, __m128 b) noexcept                    { return _mm_sub_ps (a, b); }

  74.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE mul (__m128 a, __m128 b) noexcept                    { return _mm_mul_ps (a, b); }

  75.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_and (__m128 a, __m128 b) noexcept                { return _mm_and_ps (a, b); }

  76.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_or  (__m128 a, __m128 b) noexcept                { return _mm_or_ps  (a, b); }

  77.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_xor (__m128 a, __m128 b) noexcept                { return _mm_xor_ps (a, b); }

  78.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_notand (__m128 a, __m128 b) noexcept             { return _mm_andnot_ps (a, b); }

  79.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_not (__m128 a) noexcept                          { return bit_notand (a, _mm_loadu_ps ((float*) kAllBitsSet)); }

  80.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE min (__m128 a, __m128 b) noexcept                    { return _mm_min_ps (a, b); }

  81.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE max (__m128 a, __m128 b) noexcept                    { return _mm_max_ps (a, b); }

  82.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE equal (__m128 a, __m128 b) noexcept                  { return _mm_cmpeq_ps (a, b); }

  83.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE notEqual (__m128 a, __m128 b) noexcept               { return _mm_cmpneq_ps (a, b); }

  84.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThan (__m128 a, __m128 b) noexcept            { return _mm_cmpgt_ps (a, b); }

  85.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128 a, __m128 b) noexcept     { return _mm_cmpge_ps (a, b); }

  86.     static forcedinline bool   JUCE_VECTOR_CALLTYPE allEqual (__m128 a, __m128 b ) noexcept              { return (_mm_movemask_ps (equal (a, b)) == 0xf); }

  87.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE multiplyAdd (__m128 a, __m128 b, __m128 c) noexcept  { return _mm_add_ps (a, _mm_mul_ps (b, c)); }

  88.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupeven (__m128 a) noexcept                          { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 2, 0, 0)); }

  89.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupodd (__m128 a) noexcept                           { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (3, 3, 1, 1)); }

  90.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE swapevenodd (__m128 a) noexcept                      { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 3, 0, 1)); }

  91.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE oddevensum (__m128 a) noexcept                       { return _mm_add_ps (_mm_shuffle_ps (a, a, _MM_SHUFFLE (1, 0, 3, 2)), a); }

  92.     static forcedinline float  JUCE_VECTOR_CALLTYPE get (__m128 v, size_t i) noexcept                    { return SIMDFallbackOps<float, __m128>::get (v, i); }

  93.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE set (__m128 v, size_t i, float s) noexcept           { return SIMDFallbackOps<float, __m128>::set (v, i, s); }

  94.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE truncate (__m128 a) noexcept                         { return _mm_cvtepi32_ps (_mm_cvttps_epi32 (a)); }

  95. 

  96.     //==============================================================================

  97.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE cmplxmul (__m128 a, __m128 b) noexcept

  98.     {

  99.         __m128 rr_ir = mul (a, dupeven (b));

  100.         __m128 ii_ri = mul (swapevenodd (a), dupodd (b));

  101.         return add (rr_ir, bit_xor (ii_ri, _mm_loadu_ps ((float*) kEvenHighBit)));

  102.     }

  103. 

  104.     static forcedinline float JUCE_VECTOR_CALLTYPE sum (__m128 a) noexcept

  105.     {

  106.        #if defined (__SSE4__)

  107.         const auto retval = _mm_dp_ps (a, _mm_loadu_ps (kOne), 0xff);

  108.        #elif defined (__SSE3__)

  109.         const auto shuffled = _mm_movehdup_ps (a);

  110.         const auto sums = _mm_add_ps (a, shuffled);

  111.         const auto retval = _mm_add_ss (sums, _mm_movehl_ps (shuffled, sums));

  112.        #else

  113.         auto retval = _mm_add_ps (_mm_shuffle_ps (a, a, 0x4e), a);

  114.         retval = _mm_add_ps (retval, _mm_shuffle_ps (retval, retval, 0xb1));

  115.        #endif

  116.         return _mm_cvtss_f32 (retval);

  117.     }

  118. };

  119. 

  120. //==============================================================================

  121. /** Double-precision floating point SSE intrinsics.

  122. 

  123.     @tags{DSP}

  124. */

  125. template <>

  126. struct SIMDNativeOps<double>

  127. {

  128.     //==============================================================================

  129.     using vSIMDType = __m128d;

  130. 

  131.     //==============================================================================

  132.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  133.     DECLARE_SSE_SIMD_CONST (int64_t, kEvenHighBit);

  134.     DECLARE_SSE_SIMD_CONST (double, kOne);

  135. 

  136.     //==============================================================================

  137.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const double* a) noexcept                       { return load (a); }

  138.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return _mm_castsi128_pd (_mm_load_si128 (reinterpret_cast<const __m128i*> (a))); }

  139.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE expand (double s) noexcept                              { return _mm_load1_pd (&s); }

  140.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE load (const double* a) noexcept                         { return _mm_load_pd (a); }

  141.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128d value, double* dest) noexcept               { _mm_store_pd (dest, value); }

  142.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE add (__m128d a, __m128d b) noexcept                     { return _mm_add_pd (a, b); }

  143.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE sub (__m128d a, __m128d b) noexcept                     { return _mm_sub_pd (a, b); }

  144.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE mul (__m128d a, __m128d b) noexcept                     { return _mm_mul_pd (a, b); }

  145.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_and (__m128d a, __m128d b) noexcept                 { return _mm_and_pd (a, b); }

  146.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_or  (__m128d a, __m128d b) noexcept                 { return _mm_or_pd  (a, b); }

  147.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_xor (__m128d a, __m128d b) noexcept                 { return _mm_xor_pd (a, b); }

  148.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_notand (__m128d a, __m128d b) noexcept              { return _mm_andnot_pd (a, b); }

  149.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_not (__m128d a) noexcept                            { return bit_notand (a, vconst (kAllBitsSet)); }

  150.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE min (__m128d a, __m128d b) noexcept                     { return _mm_min_pd (a, b); }

  151.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE max (__m128d a, __m128d b) noexcept                     { return _mm_max_pd (a, b); }

  152.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE equal (__m128d a, __m128d b) noexcept                   { return _mm_cmpeq_pd (a, b); }

  153.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE notEqual (__m128d a, __m128d b) noexcept                { return _mm_cmpneq_pd (a, b); }

  154.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThan (__m128d a, __m128d b) noexcept             { return _mm_cmpgt_pd (a, b); }

  155.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128d a, __m128d b) noexcept      { return _mm_cmpge_pd (a, b); }

  156.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128d a, __m128d b ) noexcept               { return (_mm_movemask_pd (equal (a, b)) == 0x3); }

  157.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE multiplyAdd (__m128d a, __m128d b, __m128d c) noexcept  { return _mm_add_pd (a, _mm_mul_pd (b, c)); }

  158.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupeven (__m128d a) noexcept                            { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 0)); }

  159.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupodd (__m128d a) noexcept                             { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (1, 1)); }

  160.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE swapevenodd (__m128d a) noexcept                        { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 1)); }

  161.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE oddevensum (__m128d a) noexcept                         { return a; }

  162.     static forcedinline double  JUCE_VECTOR_CALLTYPE get (__m128d v, size_t i) noexcept                      { return SIMDFallbackOps<double, __m128d>::get (v, i); }

  163.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE set (__m128d v, size_t i, double s) noexcept            { return SIMDFallbackOps<double, __m128d>::set (v, i, s); }

  164.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE truncate (__m128d a) noexcept                           { return _mm_cvtepi32_pd (_mm_cvttpd_epi32 (a)); }

  165. 

  166.     //==============================================================================

  167.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE cmplxmul (__m128d a, __m128d b) noexcept

  168.     {

  169.         __m128d rr_ir = mul (a, dupeven (b));

  170.         __m128d ii_ri = mul (swapevenodd (a), dupodd (b));

  171.         return add (rr_ir, bit_xor (ii_ri, vconst (kEvenHighBit)));

  172.     }

  173. 

  174.     static forcedinline double JUCE_VECTOR_CALLTYPE sum (__m128d a) noexcept

  175.     {

  176.        #if defined (__SSE4__)

  177.         __m128d retval = _mm_dp_pd (a, vconst (kOne), 0xff);

  178.        #elif defined (__SSE3__)

  179.         __m128d retval = _mm_hadd_pd (a, a);

  180.        #else

  181.         __m128d retval = _mm_add_pd (_mm_shuffle_pd (a, a, 0x01), a);

  182.        #endif

  183.         return _mm_cvtsd_f64 (retval);

  184.     }

  185. };

  186. 

  187. //==============================================================================

  188. /** Signed 8-bit integer SSE intrinsics.

  189. 

  190.     @tags{DSP}

  191. */

  192. template <>

  193. struct SIMDNativeOps<int8_t>

  194. {

  195.     //==============================================================================

  196.     using vSIMDType = __m128i;

  197. 

  198.     //==============================================================================

  199.     DECLARE_SSE_SIMD_CONST (int8_t, kAllBitsSet);

  200. 

  201.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int8_t* a) noexcept                       { return load (a); }

  202.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int8_t* a) noexcept                         { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  203.     static forcedinline void    JUCE_VECTOR_CALLTYPE store (__m128i v, int8_t* p) noexcept                   { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  204.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int8_t s) noexcept                              { return _mm_set1_epi8 (s); }

  205.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  206.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  207.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  208.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  209.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  210.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  211.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  212.    #if defined (__SSE4__)

  213.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi8 (a, b); }

  214.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi8 (a, b); }

  215.    #else

  216.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  217.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  218.    #endif

  219.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  220.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (a, b); }

  221.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  222.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  223.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  224.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  225.     static forcedinline int8_t  JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<int8_t, __m128i>::get (v, i); }

  226.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, int8_t s) noexcept            { return SIMDFallbackOps<int8_t, __m128i>::set (v, i, s); }

  227.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                           { return a; }

  228. 

  229.     //==============================================================================

  230.     static forcedinline int8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  231.     {

  232.        #ifdef __SSSE3__

  233.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  234.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  235. 

  236.         for (int i = 0; i < 3; ++i)

  237.         {

  238.             lo = _mm_hadd_epi16 (lo, lo);

  239.             hi = _mm_hadd_epi16 (hi, hi);

  240.         }

  241. 

  242.         return static_cast<int8_t> ((_mm_cvtsi128_si32 (lo) & 0xff) + (_mm_cvtsi128_si32 (hi) & 0xff));

  243.        #else

  244.         return SIMDFallbackOps<int8_t, __m128i>::sum (a);

  245.        #endif

  246.     }

  247. 

  248.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  249.     {

  250.         // unpack and multiply

  251.         __m128i even = _mm_mullo_epi16 (a, b);

  252.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  253. 

  254.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  255.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  256.     }

  257. };

  258. 

  259. //==============================================================================

  260. /** Unsigned 8-bit integer SSE intrinsics.

  261. 

  262.     @tags{DSP}

  263. */

  264. template <>

  265. struct SIMDNativeOps<uint8_t>

  266. {

  267.     //==============================================================================

  268.     using vSIMDType = __m128i;

  269. 

  270.     //==============================================================================

  271.     DECLARE_SSE_SIMD_CONST (uint8_t, kHighBit);

  272.     DECLARE_SSE_SIMD_CONST (uint8_t, kAllBitsSet);

  273. 

  274.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint8_t* a) noexcept                      { return load (a); }

  275.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  276.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint8_t* a) noexcept                        { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  277.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i v, uint8_t* p) noexcept                     { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  278.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint8_t s) noexcept                             { return _mm_set1_epi8 ((int8_t) s); }

  279.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  280.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  281.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  282.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  283.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  284.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  285.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  286.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu8 (a, b); }

  287.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu8 (a, b); }

  288.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  289.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (ssign (a), ssign (b)); }

  290.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  291.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  292.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  293.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  294.     static forcedinline uint8_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<uint8_t, __m128i>::get (v, i); }

  295.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, uint8_t s) noexcept           { return SIMDFallbackOps<uint8_t, __m128i>::set (v, i, s); }

  296.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                           { return a; }

  297. 

  298.     //==============================================================================

  299.     static forcedinline uint8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  300.     {

  301.        #ifdef __SSSE3__

  302.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  303.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  304. 

  305.         for (int i = 0; i < 3; ++i)

  306.         {

  307.             lo = _mm_hadd_epi16 (lo, lo);

  308.             hi = _mm_hadd_epi16 (hi, hi);

  309.         }

  310. 

  311.         return static_cast<uint8_t> ((static_cast<uint32_t> (_mm_cvtsi128_si32 (lo)) & 0xffu)

  312.                                    + (static_cast<uint32_t> (_mm_cvtsi128_si32 (hi)) & 0xffu));

  313.        #else

  314.         return SIMDFallbackOps<uint8_t, __m128i>::sum (a);

  315.        #endif

  316.     }

  317. 

  318.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  319.     {

  320.         // unpack and multiply

  321.         __m128i even = _mm_mullo_epi16 (a, b);

  322.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  323. 

  324.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  325.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  326.     }

  327. };

  328. 

  329. //==============================================================================

  330. /** Signed 16-bit integer SSE intrinsics.

  331. 

  332.     @tags{DSP}

  333. */

  334. template <>

  335. struct SIMDNativeOps<int16_t>

  336. {

  337.     //==============================================================================

  338.     using vSIMDType = __m128i;

  339. 

  340.     //==============================================================================

  341.     DECLARE_SSE_SIMD_CONST (int16_t, kAllBitsSet);

  342. 

  343.     //==============================================================================

  344.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int16_t* a) noexcept                      { return load (a); }

  345.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int16_t* a) noexcept                        { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  346.     static forcedinline void    JUCE_VECTOR_CALLTYPE store (__m128i v, int16_t* p) noexcept                  { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  347.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int16_t s) noexcept                             { return _mm_set1_epi16 (s); }

  348.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  349.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  350.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  351.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  352.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  353.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  354.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  355.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  356.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi16 (a, b); }

  357.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi16 (a, b); }

  358.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  359.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (a, b); }

  360.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  361.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  362.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  363.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  364.     static forcedinline int16_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<int16_t, __m128i>::get (v, i); }

  365.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, int16_t s) noexcept           { return SIMDFallbackOps<int16_t, __m128i>::set (v, i, s); }

  366.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                           { return a; }

  367. 

  368.     //==============================================================================

  369.     static forcedinline int16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  370.     {

  371.        #ifdef __SSSE3__

  372.         __m128i tmp = _mm_hadd_epi16 (a, a);

  373.         tmp = _mm_hadd_epi16 (tmp, tmp);

  374.         tmp = _mm_hadd_epi16 (tmp, tmp);

  375. 

  376.         return static_cast<int16_t> (_mm_cvtsi128_si32 (tmp) & 0xffff);

  377.        #else

  378.         return SIMDFallbackOps<int16_t, __m128i>::sum (a);

  379.        #endif

  380.     }

  381. };

  382. 

  383. //==============================================================================

  384. /** Unsigned 16-bit integer SSE intrinsics.

  385. 

  386.     @tags{DSP}

  387. */

  388. template <>

  389. struct SIMDNativeOps<uint16_t>

  390. {

  391.     //==============================================================================

  392.     using vSIMDType = __m128i;

  393. 

  394.     //==============================================================================

  395.     DECLARE_SSE_SIMD_CONST (uint16_t, kHighBit);

  396.     DECLARE_SSE_SIMD_CONST (uint16_t, kAllBitsSet);

  397. 

  398.     //==============================================================================

  399.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE vconst (const uint16_t* a) noexcept                     { return load (a); }

  400.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  401.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE load (const uint16_t* a) noexcept                       { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  402.     static forcedinline void     JUCE_VECTOR_CALLTYPE store (__m128i v, uint16_t* p) noexcept                 { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  403.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE expand (uint16_t s) noexcept                            { return _mm_set1_epi16 ((int16_t) s); }

  404.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  405.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  406.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  407.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  408.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  409.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  410.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  411.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  412.    #if defined (__SSE4__)

  413.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu16 (a, b); }

  414.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu16 (a, b); }

  415.    #else

  416.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  417.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  418.    #endif

  419.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  420.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (ssign (a), ssign (b)); }

  421.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  422.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  423.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  424.     static forcedinline bool     JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  425.     static forcedinline uint16_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<uint16_t, __m128i>::get (v, i); }

  426.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, uint16_t s) noexcept          { return SIMDFallbackOps<uint16_t, __m128i>::set (v, i, s); }

  427.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                            { return a; }

  428. 

  429.     //==============================================================================

  430.     static forcedinline uint16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  431.     {

  432.        #ifdef __SSSE3__

  433.         __m128i tmp = _mm_hadd_epi16 (a, a);

  434.         tmp = _mm_hadd_epi16 (tmp, tmp);

  435.         tmp = _mm_hadd_epi16 (tmp, tmp);

  436. 

  437.         return static_cast<uint16_t> (static_cast<uint32_t> (_mm_cvtsi128_si32 (tmp)) & 0xffffu);

  438.        #else

  439.         return SIMDFallbackOps<uint16_t, __m128i>::sum (a);

  440.        #endif

  441.     }

  442. };

  443. 

  444. //==============================================================================

  445. /** Signed 32-bit integer SSE intrinsics.

  446. 

  447.     @tags{DSP}

  448. */

  449. template <>

  450. struct SIMDNativeOps<int32_t>

  451. {

  452.     //==============================================================================

  453.     using vSIMDType = __m128i;

  454. 

  455.     //==============================================================================

  456.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  457. 

  458.     //==============================================================================

  459.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int32_t* a) noexcept                      { return load (a); }

  460.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int32_t* a) noexcept                        { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  461.     static forcedinline void    JUCE_VECTOR_CALLTYPE store (__m128i v, int32_t* p) noexcept                  { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  462.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int32_t s) noexcept                             { return _mm_set1_epi32 (s); }

  463.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  464.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  465.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  466.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  467.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  468.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  469.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  470.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  471.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (a, b); }

  472.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  473.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  474.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  475.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  476.     static forcedinline int32_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<int32_t, __m128i>::get (v, i); }

  477.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, int32_t s) noexcept           { return SIMDFallbackOps<int32_t, __m128i>::set (v, i, s); }

  478.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                           { return a; }

  479. 

  480.     //==============================================================================

  481.     static forcedinline int32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  482.     {

  483.        #ifdef __SSSE3__

  484.         __m128i tmp = _mm_hadd_epi32 (a, a);

  485.         return _mm_cvtsi128_si32 (_mm_hadd_epi32 (tmp, tmp));

  486.        #else

  487.         return SIMDFallbackOps<int32_t, __m128i>::sum (a);

  488.        #endif

  489.     }

  490. 

  491.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  492.     {

  493.        #if defined (__SSE4_1__)

  494.         return _mm_mullo_epi32 (a, b);

  495.        #else

  496.         __m128i even = _mm_mul_epu32 (a,b);

  497.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  498.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32 (even, _MM_SHUFFLE (0,0,2,0)),

  499.                                    _mm_shuffle_epi32 (odd,  _MM_SHUFFLE (0,0,2,0)));

  500.        #endif

  501.     }

  502. 

  503.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  504.     {

  505.        #if defined (__SSE4_1__)

  506.         return _mm_min_epi32 (a, b);

  507.        #else

  508.         __m128i lt = greaterThan (b, a);

  509.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  510.        #endif

  511.     }

  512. 

  513.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  514.     {

  515.        #if defined (__SSE4_1__)

  516.         return _mm_max_epi32 (a, b);

  517.        #else

  518.         __m128i gt = greaterThan (a, b);

  519.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  520.        #endif

  521.     }

  522. };

  523. 

  524. //==============================================================================

  525. /** Unsigned 32-bit integer SSE intrinsics.

  526. 

  527.     @tags{DSP}

  528. */

  529. template <>

  530. struct SIMDNativeOps<uint32_t>

  531. {

  532.     //==============================================================================

  533.     using vSIMDType = __m128i;

  534. 

  535.     //==============================================================================

  536.     DECLARE_SSE_SIMD_CONST (uint32_t, kAllBitsSet);

  537.     DECLARE_SSE_SIMD_CONST (uint32_t, kHighBit);

  538. 

  539.     //==============================================================================

  540.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE vconst (const uint32_t* a) noexcept                     { return load (a); }

  541.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  542.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE load (const uint32_t* a) noexcept                       { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  543.     static forcedinline void     JUCE_VECTOR_CALLTYPE store (__m128i v, uint32_t* p) noexcept                 { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  544.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE expand (uint32_t s) noexcept                            { return _mm_set1_epi32 ((int32_t) s); }

  545.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  546.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  547.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  548.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  549.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  550.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  551.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  552.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  553.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (ssign (a), ssign (b)); }

  554.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  555.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  556.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  557.     static forcedinline bool     JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  558.     static forcedinline uint32_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<uint32_t, __m128i>::get (v, i); }

  559.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, uint32_t s) noexcept          { return SIMDFallbackOps<uint32_t, __m128i>::set (v, i, s); }

  560.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                            { return a; }

  561. 

  562.     //==============================================================================

  563.     static forcedinline uint32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  564.     {

  565.        #ifdef __SSSE3__

  566.         __m128i tmp = _mm_hadd_epi32 (a, a);

  567.         return static_cast<uint32_t> (_mm_cvtsi128_si32 (_mm_hadd_epi32 (tmp, tmp)));

  568.        #else

  569.         return SIMDFallbackOps<uint32_t, __m128i>::sum (a);

  570.        #endif

  571.     }

  572. 

  573.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  574.     {

  575.        #if defined (__SSE4_1__)

  576.         return _mm_mullo_epi32 (a, b);

  577.        #else

  578.         __m128i even = _mm_mul_epu32 (a,b);

  579.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  580.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32 (even, _MM_SHUFFLE (0,0,2,0)),

  581.                                    _mm_shuffle_epi32 (odd,  _MM_SHUFFLE (0,0,2,0)));

  582.        #endif

  583.     }

  584. 

  585.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  586.     {

  587.        #if defined (__SSE4_1__)

  588.         return _mm_min_epi32 (a, b);

  589.        #else

  590.         __m128i lt = greaterThan (b, a);

  591.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  592.        #endif

  593.     }

  594. 

  595.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  596.     {

  597.        #if defined (__SSE4_1__)

  598.         return _mm_max_epi32 (a, b);

  599.        #else

  600.         __m128i gt = greaterThan (a, b);

  601.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  602.        #endif

  603.     }

  604. };

  605. 

  606. //==============================================================================

  607. /** Signed 64-bit integer SSE intrinsics.

  608. 

  609.     @tags{DSP}

  610. */

  611. template <>

  612. struct SIMDNativeOps<int64_t>

  613. {

  614.     //==============================================================================

  615.     using vSIMDType = __m128i;

  616. 

  617.     //==============================================================================

  618.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  619. 

  620.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return load (a); }

  621.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int64_t s) noexcept                             { return _mm_set1_epi64x (s); }

  622.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int64_t* a) noexcept                        { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  623.     static forcedinline void    JUCE_VECTOR_CALLTYPE store (__m128i v, int64_t* p) noexcept                  { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  624.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  625.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  626.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  627.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  628.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  629.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  630.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  631.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  632.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  633.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  634.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  635.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  636.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  637.     static forcedinline int64_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<int64_t, __m128i>::get (v, i); }

  638.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, int64_t s) noexcept           { return SIMDFallbackOps<int64_t, __m128i>::set (v, i, s); }

  639.     static forcedinline int64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept                                { return SIMDFallbackOps<int64_t, __m128i>::sum (a); }

  640.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return SIMDFallbackOps<int64_t, __m128i>::mul (a, b); }

  641.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                           { return a; }

  642. 

  643.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  644.     {

  645.        #if defined (__SSE4_1__)

  646.         return _mm_cmpeq_epi64 (a, b);

  647.        #else

  648.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  649.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  650.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  651.        #endif

  652.     }

  653. 

  654.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  655.     {

  656.        #if defined (__SSE4_2__)

  657.         return _mm_cmpgt_epi64 (a, b);

  658.        #else

  659.         return SIMDFallbackOps<int64_t, __m128i>::greaterThan (a, b);

  660.        #endif

  661.     }

  662. };

  663. 

  664. //==============================================================================

  665. /** Unsigned 64-bit integer SSE intrinsics.

  666. 

  667.     @tags{DSP}

  668. */

  669. template <>

  670. struct SIMDNativeOps<uint64_t>

  671. {

  672.     //==============================================================================

  673.     using vSIMDType = __m128i;

  674. 

  675.     //==============================================================================

  676.     DECLARE_SSE_SIMD_CONST (uint64_t, kAllBitsSet);

  677.     DECLARE_SSE_SIMD_CONST (uint64_t, kHighBit);

  678. 

  679.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE vconst (const uint64_t* a) noexcept                     { return load (a); }

  680.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE expand (uint64_t s) noexcept                            { return _mm_set1_epi64x ((int64_t) s); }

  681.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  682.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE load (const uint64_t* a) noexcept                       { return _mm_load_si128 (reinterpret_cast<const __m128i*> (a)); }

  683.     static forcedinline void     JUCE_VECTOR_CALLTYPE store (__m128i v, uint64_t* p) noexcept                 { _mm_store_si128 (reinterpret_cast<__m128i*> (p), v); }

  684.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  685.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  686.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  687.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  688.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  689.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  690.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  691.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  692.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  693.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  694.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  695.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  696.     static forcedinline bool     JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  697.     static forcedinline uint64_t JUCE_VECTOR_CALLTYPE get (__m128i v, size_t i) noexcept                      { return SIMDFallbackOps<uint64_t, __m128i>::get (v, i); }

  698.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE set (__m128i v, size_t i, uint64_t s) noexcept          { return SIMDFallbackOps<uint64_t, __m128i>::set (v, i, s); }

  699.     static forcedinline uint64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept                                { return SIMDFallbackOps<uint64_t, __m128i>::sum (a); }

  700.     static forcedinline __m128i  JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return SIMDFallbackOps<uint64_t, __m128i>::mul (a, b); }

  701.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE truncate (__m128i a) noexcept                            { return a; }

  702. 

  703.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  704.     {

  705.        #if defined (__SSE4_1__)

  706.         return _mm_cmpeq_epi64 (a, b);

  707.        #else

  708.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  709.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  710.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  711.        #endif

  712.     }

  713. 

  714.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  715.     {

  716.        #if defined (__SSE4_2__)

  717.         return _mm_cmpgt_epi64 (ssign (a), ssign (b));

  718.        #else

  719.         return SIMDFallbackOps<uint64_t, __m128i>::greaterThan (a, b);

  720.        #endif

  721.     }

  722. };

  723. 

  724. #endif

  725. 

  726. JUCE_END_IGNORE_WARNINGS_GCC_LIKE

  727. 

  728. } // namespace juce::dsp