DISTRHO
/
JUCE
mirror of https://github.com/DISTRHO/JUCE
1
0
Fork 0
Code Releases 1 Activity
The JUCE cross-platform C++ framework, with DISTRHO/KXStudio specific changes
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
9036 Commits
11 Branches
450MB
Tree: bb78ed2006
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bb78ed2006'
${ noResults }
JUCE/modules/juce_dsp/native/juce_sse_SIMDNativeOps.h

881 lines
50KB
Raw Blame History 

  1. /*

  2.   ==============================================================================

  3. 

  4.    This file is part of the JUCE library.

  5.    Copyright (c) 2017 - ROLI Ltd.

  6. 

  7.    JUCE is an open source library subject to commercial or open-source

  8.    licensing.

  9. 

  10.    By using JUCE, you agree to the terms of both the JUCE 5 End-User License

  11.    Agreement and JUCE 5 Privacy Policy (both updated and effective as of the

  12.    27th April 2017).

  13. 

  14.    End User License Agreement: www.juce.com/juce-5-licence

  15.    Privacy Policy: www.juce.com/juce-5-privacy-policy

  16. 

  17.    Or: You may also use this code under the terms of the GPL v3 (see

  18.    www.gnu.org/licenses).

  19. 

  20.    JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER

  21.    EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE

  22.    DISCLAIMED.

  23. 

  24.   ==============================================================================

  25. */

  26. 

  27. namespace juce

  28. {

  29. namespace dsp

  30. {

  31. 

  32. #ifndef DOXYGEN

  33. 

  34. #if JUCE_GCC && (__GNUC__ >= 6)

  35.  #pragma GCC diagnostic push

  36.  #pragma GCC diagnostic ignored "-Wignored-attributes"

  37. #endif

  38. 

  39. #ifdef _MSC_VER

  40.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  41.     static __declspec(align(16)) const type name [16 / sizeof (type)]

  42. 

  43.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  44.     __declspec(align(16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]

  45. 

  46. #else

  47.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  48.     static const type name [16 / sizeof (type)] __attribute__((aligned(16)))

  49. 

  50.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  51.     const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__((aligned(16)))

  52. 

  53. #endif

  54. 

  55. template <typename type>

  56. struct SIMDNativeOps;

  57. 

  58. //==============================================================================

  59. /** Single-precision floating point SSE intrinsics.

  60. 

  61.     @tags{DSP}

  62. */

  63. template <>

  64. struct SIMDNativeOps<float>

  65. {

  66.     //==============================================================================

  67.     typedef __m128 vSIMDType;

  68. 

  69.     //==============================================================================

  70.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  71.     DECLARE_SSE_SIMD_CONST (int32_t, kEvenHighBit);

  72.     DECLARE_SSE_SIMD_CONST (float, kOne);

  73. 

  74.     //==============================================================================

  75.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE expand (float s) noexcept                            { return _mm_load1_ps (&s); }

  76.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE load (const float* a) noexcept                       { return _mm_load_ps (a); }

  77.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128 value, float* dest) noexcept             { _mm_store_ps (dest, value); }

  78.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE add (__m128 a, __m128 b) noexcept                    { return _mm_add_ps (a, b); }

  79.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE sub (__m128 a, __m128 b) noexcept                    { return _mm_sub_ps (a, b); }

  80.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE mul (__m128 a, __m128 b) noexcept                    { return _mm_mul_ps (a, b); }

  81.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_and (__m128 a, __m128 b) noexcept                { return _mm_and_ps (a, b); }

  82.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_or  (__m128 a, __m128 b) noexcept                { return _mm_or_ps  (a, b); }

  83.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_xor (__m128 a, __m128 b) noexcept                { return _mm_xor_ps (a, b); }

  84.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_notand (__m128 a, __m128 b) noexcept             { return _mm_andnot_ps (a, b); }

  85.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_not (__m128 a) noexcept                          { return bit_notand (a, _mm_loadu_ps ((float*) kAllBitsSet)); }

  86.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE min (__m128 a, __m128 b) noexcept                    { return _mm_min_ps (a, b); }

  87.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE max (__m128 a, __m128 b) noexcept                    { return _mm_max_ps (a, b); }

  88.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE equal (__m128 a, __m128 b) noexcept                  { return _mm_cmpeq_ps (a, b); }

  89.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE notEqual (__m128 a, __m128 b) noexcept               { return _mm_cmpneq_ps (a, b); }

  90.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThan (__m128 a, __m128 b) noexcept            { return _mm_cmpgt_ps (a, b); }

  91.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128 a, __m128 b) noexcept     { return _mm_cmpge_ps (a, b); }

  92.     static forcedinline bool   JUCE_VECTOR_CALLTYPE allEqual (__m128 a, __m128 b ) noexcept              { return (_mm_movemask_ps (equal (a, b)) == 0xf); }

  93.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE multiplyAdd (__m128 a, __m128 b, __m128 c) noexcept  { return _mm_add_ps (a, _mm_mul_ps (b, c)); }

  94.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupeven (__m128 a) noexcept                          { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 2, 0, 0)); }

  95.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupodd (__m128 a) noexcept                           { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (3, 3, 1, 1)); }

  96.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE swapevenodd (__m128 a) noexcept                      { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 3, 0, 1)); }

  97.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE oddevensum (__m128 a) noexcept                       { return _mm_add_ps (_mm_shuffle_ps (a, a, _MM_SHUFFLE (1, 0, 3, 2)), a); }

  98. 

  99.     //==============================================================================

  100.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE cmplxmul (__m128 a, __m128 b) noexcept

  101.     {

  102.         __m128 rr_ir = mul (a, dupeven (b));

  103.         __m128 ii_ri = mul (swapevenodd (a), dupodd (b));

  104.         return add (rr_ir, bit_xor (ii_ri, _mm_loadu_ps ((float*) kEvenHighBit)));

  105.     }

  106. 

  107.     static forcedinline float JUCE_VECTOR_CALLTYPE sum (__m128 a) noexcept

  108.     {

  109.        #if defined(__SSE4__)

  110.         __m128 retval = _mm_dp_ps (a, _mm_loadu_ps (kOne), 0xff);

  111.        #elif defined(__SSE3__)

  112.         __m128 retval = _mm_hadd_ps (_mm_hadd_ps (a, a), a);

  113.        #else

  114.         __m128 retval = _mm_add_ps (_mm_shuffle_ps (a, a, 0x4e), a);

  115.         retval = _mm_add_ps (retval, _mm_shuffle_ps (retval, retval, 0xb1));

  116.        #endif

  117.         return ((float*) &retval) [0];

  118.     }

  119. };

  120. 

  121. //==============================================================================

  122. /** Double-precision floating point SSE intrinsics.

  123. 

  124.     @tags{DSP}

  125. */

  126. template <>

  127. struct SIMDNativeOps<double>

  128. {

  129.     //==============================================================================

  130.     typedef __m128d vSIMDType;

  131. 

  132.     //==============================================================================

  133.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  134.     DECLARE_SSE_SIMD_CONST (int64_t, kEvenHighBit);

  135.     DECLARE_SSE_SIMD_CONST (double, kOne);

  136. 

  137.     //==============================================================================

  138.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const double* a) noexcept                       { return *reinterpret_cast<const __m128d*> (a); }

  139.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return *reinterpret_cast<const __m128d*> (a); }

  140.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE expand (double s) noexcept                              { return _mm_load1_pd (&s); }

  141.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE load (const double* a) noexcept                         { return _mm_load_pd (a); }

  142.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128d value, double* dest) noexcept               { _mm_store_pd (dest, value); }

  143.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE add (__m128d a, __m128d b) noexcept                     { return _mm_add_pd (a, b); }

  144.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE sub (__m128d a, __m128d b) noexcept                     { return _mm_sub_pd (a, b); }

  145.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE mul (__m128d a, __m128d b) noexcept                     { return _mm_mul_pd (a, b); }

  146.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_and (__m128d a, __m128d b) noexcept                 { return _mm_and_pd (a, b); }

  147.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_or  (__m128d a, __m128d b) noexcept                 { return _mm_or_pd  (a, b); }

  148.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_xor (__m128d a, __m128d b) noexcept                 { return _mm_xor_pd (a, b); }

  149.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_notand (__m128d a, __m128d b) noexcept              { return _mm_andnot_pd (a, b); }

  150.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_not (__m128d a) noexcept                            { return bit_notand (a, vconst (kAllBitsSet)); }

  151.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE min (__m128d a, __m128d b) noexcept                     { return _mm_min_pd (a, b); }

  152.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE max (__m128d a, __m128d b) noexcept                     { return _mm_max_pd (a, b); }

  153.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE equal (__m128d a, __m128d b) noexcept                   { return _mm_cmpeq_pd (a, b); }

  154.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE notEqual (__m128d a, __m128d b) noexcept                { return _mm_cmpneq_pd (a, b); }

  155.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThan (__m128d a, __m128d b) noexcept             { return _mm_cmpgt_pd (a, b); }

  156.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128d a, __m128d b) noexcept      { return _mm_cmpge_pd (a, b); }

  157.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128d a, __m128d b ) noexcept               { return (_mm_movemask_pd (equal (a, b)) == 0x3); }

  158.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE multiplyAdd (__m128d a, __m128d b, __m128d c) noexcept  { return _mm_add_pd (a, _mm_mul_pd (b, c)); }

  159.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupeven (__m128d a) noexcept                            { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 0)); }

  160.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupodd (__m128d a) noexcept                             { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (1, 1)); }

  161.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE swapevenodd (__m128d a) noexcept                        { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 1)); }

  162.     static forcedinline __m128d oddevensum (__m128d a) noexcept                         { return a; }

  163. 

  164.     //==============================================================================

  165.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE cmplxmul (__m128d a, __m128d b) noexcept

  166.     {

  167.         __m128d rr_ir = mul (a, dupeven (b));

  168.         __m128d ii_ri = mul (swapevenodd (a), dupodd (b));

  169.         return add (rr_ir, bit_xor (ii_ri, vconst (kEvenHighBit)));

  170.     }

  171. 

  172.     static forcedinline double JUCE_VECTOR_CALLTYPE sum (__m128d a) noexcept

  173.     {

  174.        #if defined(__SSE4__)

  175.         __m128d retval = _mm_dp_pd (a, vconst (kOne), 0xff);

  176.        #elif defined(__SSE3__)

  177.         __m128d retval = _mm_hadd_pd (a, a);

  178.        #else

  179.         __m128d retval = _mm_add_pd (_mm_shuffle_pd (a, a, 0x01), a);

  180.        #endif

  181.         return ((double*) &retval) [0];

  182.     }

  183. };

  184. 

  185. //==============================================================================

  186. /** Signed 8-bit integer SSE intrinsics.

  187. 

  188.     @tags{DSP}

  189. */

  190. template <>

  191. struct SIMDNativeOps<int8_t>

  192. {

  193.     //==============================================================================

  194.     typedef __m128i vSIMDType;

  195. 

  196.     //==============================================================================

  197.     DECLARE_SSE_SIMD_CONST (int8_t, kAllBitsSet);

  198. 

  199.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int8_t* a) noexcept                       { return *reinterpret_cast<const __m128i*> (a); }

  200.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int8_t s) noexcept                              { return _mm_set1_epi8 (s); }

  201.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  202.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  203.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  204.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  205.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  206.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  207.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  208.    #if defined(__SSE4__)

  209.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi8 (a, b); }

  210.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi8 (a, b); }

  211.    #else

  212.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  213.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  214.    #endif

  215.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  216.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (a, b); }

  217.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  218.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  219.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  220.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  221. 

  222.     //==============================================================================

  223.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int8_t* a) noexcept

  224.     {

  225.         const auto* b = reinterpret_cast<const char*> (a);

  226.         return _mm_set_epi8 (b[15], b[14], b[13], b[12], b[11], b[10], b[9], b[8],

  227.                              b[7],  b[6],  b[5],  b[4],  b[3],  b[2],  b[1], b[0]);

  228.     }

  229. 

  230.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int8_t* dest) noexcept

  231.     {

  232.         SIMDFallbackOps<int8_t, __m128i>::store (value, dest);

  233.     }

  234. 

  235.     static forcedinline int8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  236.     {

  237.        #ifdef __SSSE3__

  238.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  239.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  240. 

  241.         for (int i = 0; i < 3; ++i)

  242.         {

  243.             lo = _mm_hadd_epi16 (lo, lo);

  244.             hi = _mm_hadd_epi16 (hi, hi);

  245.         }

  246. 

  247.         const int8_t* lo_ptr = reinterpret_cast<const int8_t*> (&lo);

  248.         const int8_t* hi_ptr = reinterpret_cast<const int8_t*> (&hi);

  249. 

  250.         return lo_ptr[0] + hi_ptr[0];

  251.        #else

  252.         int8_t sum = 0;

  253.         const int8_t* src = reinterpret_cast<const int8_t*> (&a);

  254. 

  255.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int8_t)); ++i)

  256.             sum += src [i];

  257. 

  258.         return sum;

  259.        #endif

  260.     }

  261. 

  262.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  263.     {

  264.         // unpack and multiply

  265.         __m128i even = _mm_mullo_epi16 (a, b);

  266.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  267. 

  268.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  269.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  270.     }

  271. };

  272. 

  273. //==============================================================================

  274. /** Unsigned 8-bit integer SSE intrinsics.

  275. 

  276.     @tags{DSP}

  277. */

  278. template <>

  279. struct SIMDNativeOps<uint8_t>

  280. {

  281.     //==============================================================================

  282.     typedef __m128i vSIMDType;

  283. 

  284.     //==============================================================================

  285.     DECLARE_SSE_SIMD_CONST (uint8_t, kHighBit);

  286.     DECLARE_SSE_SIMD_CONST (uint8_t, kAllBitsSet);

  287. 

  288.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint8_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  289.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  290.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint8_t s) noexcept                             { return _mm_set1_epi8 ((int8_t) s); }

  291.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  292.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  293.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  294.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  295.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  296.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  297.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  298.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu8 (a, b); }

  299.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu8 (a, b); }

  300.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  301.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (ssign (a), ssign (b)); }

  302.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  303.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  304.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  305.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  306. 

  307.     //==============================================================================

  308.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint8_t* a) noexcept

  309.     {

  310.         const auto* b = reinterpret_cast<const char*> (a);

  311.         return _mm_set_epi8 (b[15], b[14], b[13], b[12], b[11], b[10], b[9], b[8],

  312.                              b[7],  b[6],  b[5],  b[4],  b[3],  b[2],  b[1], b[0]);

  313.     }

  314. 

  315.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint8_t* dest) noexcept

  316.     {

  317.         SIMDFallbackOps<uint8_t, __m128i>::store (value, dest);

  318.     }

  319. 

  320.     static forcedinline uint8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  321.     {

  322.        #ifdef __SSSE3__

  323.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  324.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  325. 

  326.         for (int i = 0; i < 3; ++i)

  327.         {

  328.             lo = _mm_hadd_epi16 (lo, lo);

  329.             hi = _mm_hadd_epi16 (hi, hi);

  330.         }

  331. 

  332.         const uint8_t* lo_ptr = reinterpret_cast<const uint8_t*> (&lo);

  333.         const uint8_t* hi_ptr = reinterpret_cast<const uint8_t*> (&hi);

  334. 

  335.         return lo_ptr[0] + hi_ptr[0];

  336.        #else

  337.         uint8_t sum = 0;

  338.         const uint8_t* src = reinterpret_cast<const uint8_t*> (&a);

  339. 

  340.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int8_t)); ++i)

  341.             sum += src [i];

  342. 

  343.         return sum;

  344.        #endif

  345.     }

  346. 

  347.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  348.     {

  349.         // unpack and multiply

  350.         __m128i even = _mm_mullo_epi16 (a, b);

  351.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  352. 

  353.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  354.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  355.     }

  356. };

  357. 

  358. //==============================================================================

  359. /** Signed 16-bit integer SSE intrinsics.

  360. 

  361.     @tags{DSP}

  362. */

  363. template <>

  364. struct SIMDNativeOps<int16_t>

  365. {

  366.     //==============================================================================

  367.     typedef __m128i vSIMDType;

  368. 

  369.     //==============================================================================

  370.     DECLARE_SSE_SIMD_CONST (int16_t, kAllBitsSet);

  371. 

  372.     //==============================================================================

  373.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int16_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  374.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int16_t s) noexcept                             { return _mm_set1_epi16 (s); }

  375.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  376.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  377.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  378.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  379.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  380.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  381.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  382.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  383.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi16 (a, b); }

  384.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi16 (a, b); }

  385.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  386.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (a, b); }

  387.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  388.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  389.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  390.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  391. 

  392.     //==============================================================================

  393.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int16_t* a) noexcept

  394.     {

  395.         return _mm_set_epi16 (a[7], a[6], a[5], a[4], a[3], a[2], a[1], a[0]);

  396.     }

  397. 

  398.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int16_t* dest) noexcept

  399.     {

  400.         SIMDFallbackOps<int16_t, __m128i>::store (value, dest);

  401.     }

  402. 

  403.     static forcedinline int16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  404.     {

  405.        #ifdef __SSSE3__

  406.         __m128i tmp = _mm_hadd_epi16 (a, a);

  407.         tmp = _mm_hadd_epi16 (tmp, tmp);

  408.         tmp = _mm_hadd_epi16 (tmp, tmp);

  409.         return *reinterpret_cast<int16_t*> (&tmp);

  410. 

  411.        #else

  412.         int16_t sum = 0;

  413.         const int16_t* src = reinterpret_cast<const int16_t*> (&a);

  414. 

  415.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int16_t)); ++i)

  416.             sum += src [i];

  417. 

  418.         return sum;

  419.        #endif

  420.     }

  421. };

  422. 

  423. //==============================================================================

  424. /** Unsigned 16-bit integer SSE intrinsics.

  425. 

  426.     @tags{DSP}

  427. */

  428. template <>

  429. struct SIMDNativeOps<uint16_t>

  430. {

  431.     //==============================================================================

  432.     typedef __m128i vSIMDType;

  433. 

  434.     //==============================================================================

  435.     DECLARE_SSE_SIMD_CONST (uint16_t, kHighBit);

  436.     DECLARE_SSE_SIMD_CONST (uint16_t, kAllBitsSet);

  437. 

  438.     //==============================================================================

  439.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint16_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  440.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  441.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint16_t s) noexcept                            { return _mm_set1_epi16 ((int16_t) s); }

  442.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  443.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  444.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  445.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  446.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  447.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  448.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  449.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  450.    #if defined(__SSE4__)

  451.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu16 (a, b); }

  452.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu16 (a, b); }

  453.    #else

  454.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  455.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  456.    #endif

  457.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  458.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (ssign (a), ssign (b)); }

  459.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  460.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  461.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  462.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  463. 

  464.     //==============================================================================

  465.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint16_t* a) noexcept

  466.     {

  467.         const auto* b = reinterpret_cast<const int16_t*> (a);

  468.         return _mm_set_epi16 (b[7], b[6], b[5], b[4], b[3], b[2], b[1], b[0]);

  469.     }

  470. 

  471.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint16_t* dest) noexcept

  472.     {

  473.         SIMDFallbackOps<uint16_t, __m128i>::store (value, dest);

  474.     }

  475. 

  476.     static forcedinline uint16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  477.     {

  478.        #ifdef __SSSE3__

  479.         __m128i tmp = _mm_hadd_epi16 (a, a);

  480.         tmp = _mm_hadd_epi16 (tmp, tmp);

  481.         tmp = _mm_hadd_epi16 (tmp, tmp);

  482.         return *reinterpret_cast<uint16_t*> (&tmp);

  483.        #else

  484.         uint16_t sum = 0;

  485.         const uint16_t* src = reinterpret_cast<const uint16_t*> (&a);

  486. 

  487.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(uint16_t)); ++i)

  488.             sum += src [i];

  489. 

  490.         return sum;

  491.        #endif

  492.     }

  493. };

  494. 

  495. //==============================================================================

  496. /** Signed 32-bit integer SSE intrinsics.

  497. 

  498.     @tags{DSP}

  499. */

  500. template <>

  501. struct SIMDNativeOps<int32_t>

  502. {

  503.     //==============================================================================

  504.     typedef __m128i vSIMDType;

  505. 

  506.     //==============================================================================

  507.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  508. 

  509.     //==============================================================================

  510.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int32_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  511.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int32_t s) noexcept                             { return _mm_set1_epi32 (s); }

  512.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int32_t* a) noexcept                        { return _mm_set_epi32 (a[3], a[2], a[1], a[0]); }

  513.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  514.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  515.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  516.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  517.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  518.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  519.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  520.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  521.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (a, b); }

  522.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  523.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  524.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  525.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  526. 

  527.     //==============================================================================

  528.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int32_t* dest) noexcept

  529.     {

  530.         SIMDFallbackOps<int32_t, __m128i>::store (value, dest);

  531.     }

  532. 

  533.     static forcedinline int32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  534.     {

  535.        #ifdef __SSSE3__

  536.         __m128i tmp = _mm_hadd_epi32 (a, a);

  537.         tmp = _mm_hadd_epi32 (tmp, tmp);

  538.         return *reinterpret_cast<int32_t*> (&tmp);

  539.        #else

  540.         int32_t sum = 0;

  541.         const int32_t* src = reinterpret_cast<const int32_t*> (&a);

  542. 

  543.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int32_t)); ++i)

  544.             sum += src [i];

  545. 

  546.         return sum;

  547.        #endif

  548.     }

  549. 

  550.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  551.     {

  552.        #if defined(__SSE4_1__)

  553.         return _mm_mullo_epi32 (a, b);

  554.        #else

  555.         __m128i even = _mm_mul_epu32 (a,b);

  556.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  557.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32(even, _MM_SHUFFLE (0,0,2,0)),

  558.                                    _mm_shuffle_epi32(odd,  _MM_SHUFFLE (0,0,2,0)));

  559.        #endif

  560.     }

  561. 

  562.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  563.     {

  564.        #if defined(__SSE4_1__)

  565.         return _mm_min_epi32 (a, b);

  566.        #else

  567.         __m128i lt = greaterThan (b, a);

  568.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  569.        #endif

  570.     }

  571. 

  572.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  573.     {

  574.        #if defined(__SSE4_1__)

  575.         return _mm_max_epi32 (a, b);

  576.        #else

  577.         __m128i gt = greaterThan (a, b);

  578.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  579.        #endif

  580.     }

  581. };

  582. 

  583. //==============================================================================

  584. /** Unsigned 32-bit integer SSE intrinsics.

  585. 

  586.     @tags{DSP}

  587. */

  588. template <>

  589. struct SIMDNativeOps<uint32_t>

  590. {

  591.     //==============================================================================

  592.     typedef __m128i vSIMDType;

  593. 

  594.     //==============================================================================

  595.     DECLARE_SSE_SIMD_CONST (uint32_t, kAllBitsSet);

  596.     DECLARE_SSE_SIMD_CONST (uint32_t, kHighBit);

  597. 

  598.     //==============================================================================

  599.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint32_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  600.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  601.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint32_t s) noexcept                            { return _mm_set1_epi32 ((int32_t) s); }

  602.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  603.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  604.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  605.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  606.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  607.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  608.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  609.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  610.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (ssign (a), ssign (b)); }

  611.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  612.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  613.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  614.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  615. 

  616.     //==============================================================================

  617.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint32_t* a) noexcept

  618.     {

  619.         const auto* b = reinterpret_cast<const int32_t*> (a);

  620.         return _mm_set_epi32 (b[3], b[2], b[1], b[0]);

  621.     }

  622. 

  623.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint32_t* dest) noexcept

  624.     {

  625.         SIMDFallbackOps<uint32_t, __m128i>::store (value, dest);

  626.     }

  627. 

  628.     static forcedinline uint32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  629.     {

  630.        #ifdef __SSSE3__

  631.         __m128i tmp = _mm_hadd_epi32 (a, a);

  632.         tmp = _mm_hadd_epi32 (tmp, tmp);

  633.         return *reinterpret_cast<uint32_t*> (&tmp);

  634.        #else

  635.         uint32_t sum = 0;

  636.         const uint32_t* src = reinterpret_cast<const uint32_t*> (&a);

  637. 

  638.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(uint32_t)); ++i)

  639.             sum += src [i];

  640. 

  641.         return sum;

  642.        #endif

  643.     }

  644. 

  645.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  646.     {

  647.        #if defined(__SSE4_1__)

  648.         return _mm_mullo_epi32 (a, b);

  649.        #else

  650.         __m128i even = _mm_mul_epu32 (a,b);

  651.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  652.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32(even, _MM_SHUFFLE (0,0,2,0)),

  653.                                    _mm_shuffle_epi32(odd,  _MM_SHUFFLE (0,0,2,0)));

  654.        #endif

  655.     }

  656. 

  657.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  658.     {

  659.        #if defined(__SSE4_1__)

  660.         return _mm_min_epi32 (a, b);

  661.        #else

  662.         __m128i lt = greaterThan (b, a);

  663.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  664.        #endif

  665.     }

  666. 

  667.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  668.     {

  669.        #if defined(__SSE4_1__)

  670.         return _mm_max_epi32 (a, b);

  671.        #else

  672.         __m128i gt = greaterThan (a, b);

  673.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  674.        #endif

  675.     }

  676. };

  677. 

  678. //==============================================================================

  679. /** Signed 64-bit integer SSE intrinsics.

  680. 

  681.     @tags{DSP}

  682. */

  683. template <>

  684. struct SIMDNativeOps<int64_t>

  685. {

  686.     //==============================================================================

  687.     typedef __m128i vSIMDType;

  688. 

  689.     //==============================================================================

  690.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  691. 

  692.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int64_t s) noexcept

  693.     {

  694.         __m128i retval;

  695.         int64_t* ptr = reinterpret_cast<int64_t*> (&retval);

  696.         ptr[0] = ptr[1] = s;

  697.         return retval;

  698.     }

  699. 

  700.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int64_t* a) noexcept                        { return _mm_set_epi64x (a[1], a[0]); }

  701.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  702.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  703.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  704.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  705.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  706.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  707.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  708.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  709.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  710.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  711.     static forcedinline __m128i greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  712.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  713.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  714.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  715. 

  716.     //==============================================================================

  717.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int64_t* dest) noexcept

  718.     {

  719.         SIMDFallbackOps<int64_t, __m128i>::store (value, dest);

  720.     }

  721. 

  722.     static forcedinline int64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  723.     {

  724.         const int64_t* ptr = reinterpret_cast<const int64_t*> (&a);

  725.         return ptr[0] + ptr[1];

  726.     }

  727. 

  728.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  729.     {

  730.         __m128i retval;

  731. 

  732.         const int64_t* aptr = reinterpret_cast<const int64_t*> (&a);

  733.         const int64_t* bptr = reinterpret_cast<const int64_t*> (&b);

  734.         int64_t* dst =  reinterpret_cast<int64_t*> (&retval);

  735. 

  736.         dst[0] = aptr[0] * bptr[0];

  737.         dst[1] = aptr[1] * bptr[1];

  738. 

  739.         return retval;

  740.     }

  741. 

  742.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  743.     {

  744.        #if defined(__SSE4_1__)

  745.         return _mm_cmpeq_epi64 (a, b);

  746.        #else

  747.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  748.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  749.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  750.        #endif

  751.     }

  752. 

  753.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  754.     {

  755.        #if defined(__SSE4_1__) && !defined(__clang__)

  756.         return _mm_cmpgt_epi64 (a, b);

  757.        #else

  758.         __m128i retval;

  759. 

  760.         const int64_t* aptr = reinterpret_cast<const int64_t*> (&a);

  761.         const int64_t* bptr = reinterpret_cast<const int64_t*> (&b);

  762.         int64_t* dst =  reinterpret_cast<int64_t*> (&retval);

  763. 

  764.         dst[0] = aptr[0] > bptr[0] ? -1LL : 0;

  765.         dst[1] = aptr[1] > bptr[1] ? -1LL : 0;

  766. 

  767.         return retval;

  768.        #endif

  769.     }

  770. };

  771. 

  772. //==============================================================================

  773. /** Unsigned 64-bit integer SSE intrinsics.

  774. 

  775.     @tags{DSP}

  776. */

  777. template <>

  778. struct SIMDNativeOps<uint64_t>

  779. {

  780.     //==============================================================================

  781.     typedef __m128i vSIMDType;

  782. 

  783.     //==============================================================================

  784.     DECLARE_SSE_SIMD_CONST (uint64_t, kAllBitsSet);

  785.     DECLARE_SSE_SIMD_CONST (uint64_t, kHighBit);

  786. 

  787.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint64_t s) noexcept

  788.     {

  789.          __m128i retval;

  790.         uint64_t* ptr = reinterpret_cast<uint64_t*> (&retval);

  791.         ptr[0] = ptr[1] = s;

  792.         return retval;

  793.     }

  794. 

  795.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint64_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  796.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  797.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  798.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  799.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  800.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  801.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  802.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  803.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  804.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  805.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  806.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  807.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  808.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  809.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  810. 

  811.     //==============================================================================

  812.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint64_t* a) noexcept

  813.     {

  814.         const auto* b = reinterpret_cast<const int64_t*> (a);

  815.         return _mm_set_epi64x (b[1], b[0]);

  816.     }

  817. 

  818.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint64_t* dest) noexcept

  819.     {

  820.         SIMDFallbackOps<uint64_t, __m128i>::store (value, dest);

  821.     }

  822. 

  823.     static forcedinline uint64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  824.     {

  825.         const uint64_t* ptr = reinterpret_cast<const uint64_t*> (&a);

  826.         return ptr[0] + ptr[1];

  827.     }

  828. 

  829.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  830.     {

  831.         __m128i retval;

  832. 

  833.         const uint64_t* aptr = reinterpret_cast<const uint64_t*> (&a);

  834.         const uint64_t* bptr = reinterpret_cast<const uint64_t*> (&b);

  835.         uint64_t* dst =  reinterpret_cast<uint64_t*> (&retval);

  836. 

  837.         dst[0] = aptr[0] * bptr[0];

  838.         dst[1] = aptr[1] * bptr[1];

  839. 

  840.         return retval;

  841.     }

  842. 

  843.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  844.     {

  845.        #if defined(__SSE4_1__)

  846.         return _mm_cmpeq_epi64 (a, b);

  847.        #else

  848.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  849.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  850.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  851.        #endif

  852.     }

  853. 

  854.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  855.     {

  856.        #if defined(__SSE4_1__) && !defined(__clang__)

  857.         return _mm_cmpgt_epi64 (a, b);

  858.        #else

  859.         __m128i retval;

  860. 

  861.         const uint64_t* aptr = reinterpret_cast<const uint64_t*> (&a);

  862.         const uint64_t* bptr = reinterpret_cast<const uint64_t*> (&b);

  863.         uint64_t* dst =  reinterpret_cast<uint64_t*> (&retval);

  864. 

  865.         dst[0] = aptr[0] > bptr[0] ? (uint64_t) -1LL : 0;

  866.         dst[1] = aptr[1] > bptr[1] ? (uint64_t) -1LL : 0;

  867. 

  868.         return retval;

  869.        #endif

  870.     }

  871. };

  872. 

  873. #endif

  874. 

  875. #if JUCE_GCC && (__GNUC__ >= 6)

  876.  #pragma GCC diagnostic pop

  877. #endif

  878. 

  879. } // namespace dsp

  880. } // namespace juce