DISTRHO
/
JUCE
mirror of https://github.com/DISTRHO/JUCE
1
0
Fork 0
Code Releases 1 Activity
The JUCE cross-platform C++ framework, with DISTRHO/KXStudio specific changes
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
9013 Commits
11 Branches
3.9GB
Tree: c7c26fa9f0
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c7c26fa9f0'
${ noResults }
JUCE/modules/juce_dsp/native/juce_sse_SIMDNativeOps.h

872 lines
50KB
Raw Blame History 

  1. /*

  2.   ==============================================================================

  3. 

  4.    This file is part of the JUCE library.

  5.    Copyright (c) 2017 - ROLI Ltd.

  6. 

  7.    JUCE is an open source library subject to commercial or open-source

  8.    licensing.

  9. 

  10.    By using JUCE, you agree to the terms of both the JUCE 5 End-User License

  11.    Agreement and JUCE 5 Privacy Policy (both updated and effective as of the

  12.    27th April 2017).

  13. 

  14.    End User License Agreement: www.juce.com/juce-5-licence

  15.    Privacy Policy: www.juce.com/juce-5-privacy-policy

  16. 

  17.    Or: You may also use this code under the terms of the GPL v3 (see

  18.    www.gnu.org/licenses).

  19. 

  20.    JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER

  21.    EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE

  22.    DISCLAIMED.

  23. 

  24.   ==============================================================================

  25. */

  26. 

  27. namespace juce

  28. {

  29. namespace dsp

  30. {

  31. 

  32. #ifndef DOXYGEN

  33. 

  34. #ifdef _MSC_VER

  35.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  36.     static __declspec(align(16)) const type name [16 / sizeof (type)]

  37. 

  38.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  39.     __declspec(align(16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]

  40. 

  41. #else

  42.  #define DECLARE_SSE_SIMD_CONST(type, name) \

  43.     static const type name [16 / sizeof (type)] __attribute__((aligned(16)))

  44. 

  45.  #define DEFINE_SSE_SIMD_CONST(type, class_type, name) \

  46.     const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__((aligned(16)))

  47. 

  48. #endif

  49. 

  50. template <typename type>

  51. struct SIMDNativeOps;

  52. 

  53. //==============================================================================

  54. /** Single-precision floating point SSE intrinsics.

  55. 

  56.     @tags{DSP}

  57. */

  58. template <>

  59. struct SIMDNativeOps<float>

  60. {

  61.     //==============================================================================

  62.     typedef __m128 vSIMDType;

  63. 

  64.     //==============================================================================

  65.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  66.     DECLARE_SSE_SIMD_CONST (int32_t, kEvenHighBit);

  67.     DECLARE_SSE_SIMD_CONST (float, kOne);

  68. 

  69.     //==============================================================================

  70.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE expand (float s) noexcept                            { return _mm_load1_ps (&s); }

  71.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE load (const float* a) noexcept                       { return _mm_load_ps (a); }

  72.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128 value, float* dest) noexcept             { _mm_store_ps (dest, value); }

  73.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE add (__m128 a, __m128 b) noexcept                    { return _mm_add_ps (a, b); }

  74.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE sub (__m128 a, __m128 b) noexcept                    { return _mm_sub_ps (a, b); }

  75.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE mul (__m128 a, __m128 b) noexcept                    { return _mm_mul_ps (a, b); }

  76.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_and (__m128 a, __m128 b) noexcept                { return _mm_and_ps (a, b); }

  77.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_or  (__m128 a, __m128 b) noexcept                { return _mm_or_ps  (a, b); }

  78.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_xor (__m128 a, __m128 b) noexcept                { return _mm_xor_ps (a, b); }

  79.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_notand (__m128 a, __m128 b) noexcept             { return _mm_andnot_ps (a, b); }

  80.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE bit_not (__m128 a) noexcept                          { return bit_notand (a, _mm_loadu_ps ((float*) kAllBitsSet)); }

  81.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE min (__m128 a, __m128 b) noexcept                    { return _mm_min_ps (a, b); }

  82.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE max (__m128 a, __m128 b) noexcept                    { return _mm_max_ps (a, b); }

  83.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE equal (__m128 a, __m128 b) noexcept                  { return _mm_cmpeq_ps (a, b); }

  84.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE notEqual (__m128 a, __m128 b) noexcept               { return _mm_cmpneq_ps (a, b); }

  85.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThan (__m128 a, __m128 b) noexcept            { return _mm_cmpgt_ps (a, b); }

  86.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128 a, __m128 b) noexcept     { return _mm_cmpge_ps (a, b); }

  87.     static forcedinline bool   JUCE_VECTOR_CALLTYPE allEqual (__m128 a, __m128 b ) noexcept              { return (_mm_movemask_ps (equal (a, b)) == 0xf); }

  88.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE multiplyAdd (__m128 a, __m128 b, __m128 c) noexcept  { return _mm_add_ps (a, _mm_mul_ps (b, c)); }

  89.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupeven (__m128 a) noexcept                          { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 2, 0, 0)); }

  90.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE dupodd (__m128 a) noexcept                           { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (3, 3, 1, 1)); }

  91.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE swapevenodd (__m128 a) noexcept                      { return _mm_shuffle_ps (a, a, _MM_SHUFFLE (2, 3, 0, 1)); }

  92.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE oddevensum (__m128 a) noexcept                       { return _mm_add_ps (_mm_shuffle_ps (a, a, _MM_SHUFFLE (1, 0, 3, 2)), a); }

  93. 

  94.     //==============================================================================

  95.     static forcedinline __m128 JUCE_VECTOR_CALLTYPE cmplxmul (__m128 a, __m128 b) noexcept

  96.     {

  97.         __m128 rr_ir = mul (a, dupeven (b));

  98.         __m128 ii_ri = mul (swapevenodd (a), dupodd (b));

  99.         return add (rr_ir, bit_xor (ii_ri, _mm_loadu_ps ((float*) kEvenHighBit)));

  100.     }

  101. 

  102.     static forcedinline float JUCE_VECTOR_CALLTYPE sum (__m128 a) noexcept

  103.     {

  104.        #if defined(__SSE4__)

  105.         __m128 retval = _mm_dp_ps (a, _mm_loadu_ps (kOne), 0xff);

  106.        #elif defined(__SSE3__)

  107.         __m128 retval = _mm_hadd_ps (_mm_hadd_ps (a, a), a);

  108.        #else

  109.         __m128 retval = _mm_add_ps (_mm_shuffle_ps (a, a, 0x4e), a);

  110.         retval = _mm_add_ps (retval, _mm_shuffle_ps (retval, retval, 0xb1));

  111.        #endif

  112.         return ((float*) &retval) [0];

  113.     }

  114. };

  115. 

  116. //==============================================================================

  117. /** Double-precision floating point SSE intrinsics.

  118. 

  119.     @tags{DSP}

  120. */

  121. template <>

  122. struct SIMDNativeOps<double>

  123. {

  124.     //==============================================================================

  125.     typedef __m128d vSIMDType;

  126. 

  127.     //==============================================================================

  128.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  129.     DECLARE_SSE_SIMD_CONST (int64_t, kEvenHighBit);

  130.     DECLARE_SSE_SIMD_CONST (double, kOne);

  131. 

  132.     //==============================================================================

  133.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const double* a) noexcept                       { return *reinterpret_cast<const __m128d*> (a); }

  134.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return *reinterpret_cast<const __m128d*> (a); }

  135.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE expand (double s) noexcept                              { return _mm_load1_pd (&s); }

  136.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE load (const double* a) noexcept                         { return _mm_load_pd (a); }

  137.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128d value, double* dest) noexcept               { _mm_store_pd (dest, value); }

  138.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE add (__m128d a, __m128d b) noexcept                     { return _mm_add_pd (a, b); }

  139.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE sub (__m128d a, __m128d b) noexcept                     { return _mm_sub_pd (a, b); }

  140.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE mul (__m128d a, __m128d b) noexcept                     { return _mm_mul_pd (a, b); }

  141.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_and (__m128d a, __m128d b) noexcept                 { return _mm_and_pd (a, b); }

  142.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_or  (__m128d a, __m128d b) noexcept                 { return _mm_or_pd  (a, b); }

  143.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_xor (__m128d a, __m128d b) noexcept                 { return _mm_xor_pd (a, b); }

  144.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_notand (__m128d a, __m128d b) noexcept              { return _mm_andnot_pd (a, b); }

  145.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE bit_not (__m128d a) noexcept                            { return bit_notand (a, vconst (kAllBitsSet)); }

  146.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE min (__m128d a, __m128d b) noexcept                     { return _mm_min_pd (a, b); }

  147.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE max (__m128d a, __m128d b) noexcept                     { return _mm_max_pd (a, b); }

  148.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE equal (__m128d a, __m128d b) noexcept                   { return _mm_cmpeq_pd (a, b); }

  149.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE notEqual (__m128d a, __m128d b) noexcept                { return _mm_cmpneq_pd (a, b); }

  150.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThan (__m128d a, __m128d b) noexcept             { return _mm_cmpgt_pd (a, b); }

  151.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128d a, __m128d b) noexcept      { return _mm_cmpge_pd (a, b); }

  152.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128d a, __m128d b ) noexcept               { return (_mm_movemask_pd (equal (a, b)) == 0x3); }

  153.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE multiplyAdd (__m128d a, __m128d b, __m128d c) noexcept  { return _mm_add_pd (a, _mm_mul_pd (b, c)); }

  154.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupeven (__m128d a) noexcept                            { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 0)); }

  155.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE dupodd (__m128d a) noexcept                             { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (1, 1)); }

  156.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE swapevenodd (__m128d a) noexcept                        { return _mm_shuffle_pd (a, a, _MM_SHUFFLE2 (0, 1)); }

  157.     static forcedinline __m128d oddevensum (__m128d a) noexcept                         { return a; }

  158. 

  159.     //==============================================================================

  160.     static forcedinline __m128d JUCE_VECTOR_CALLTYPE cmplxmul (__m128d a, __m128d b) noexcept

  161.     {

  162.         __m128d rr_ir = mul (a, dupeven (b));

  163.         __m128d ii_ri = mul (swapevenodd (a), dupodd (b));

  164.         return add (rr_ir, bit_xor (ii_ri, vconst (kEvenHighBit)));

  165.     }

  166. 

  167.     static forcedinline double JUCE_VECTOR_CALLTYPE sum (__m128d a) noexcept

  168.     {

  169.        #if defined(__SSE4__)

  170.         __m128d retval = _mm_dp_pd (a, vconst (kOne), 0xff);

  171.        #elif defined(__SSE3__)

  172.         __m128d retval = _mm_hadd_pd (a, a);

  173.        #else

  174.         __m128d retval = _mm_add_pd (_mm_shuffle_pd (a, a, 0x01), a);

  175.        #endif

  176.         return ((double*) &retval) [0];

  177.     }

  178. };

  179. 

  180. //==============================================================================

  181. /** Signed 8-bit integer SSE intrinsics.

  182. 

  183.     @tags{DSP}

  184. */

  185. template <>

  186. struct SIMDNativeOps<int8_t>

  187. {

  188.     //==============================================================================

  189.     typedef __m128i vSIMDType;

  190. 

  191.     //==============================================================================

  192.     DECLARE_SSE_SIMD_CONST (int8_t, kAllBitsSet);

  193. 

  194.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int8_t* a) noexcept                       { return *reinterpret_cast<const __m128i*> (a); }

  195.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int8_t s) noexcept                              { return _mm_set1_epi8 (s); }

  196.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  197.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  198.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  199.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  200.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  201.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  202.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  203.    #if defined(__SSE4__)

  204.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi8 (a, b); }

  205.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi8 (a, b); }

  206.    #else

  207.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  208.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  209.    #endif

  210.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  211.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (a, b); }

  212.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  213.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  214.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  215.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  216. 

  217.     //==============================================================================

  218.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int8_t* a) noexcept

  219.     {

  220.         const auto* b = reinterpret_cast<const char*> (a);

  221.         return _mm_set_epi8 (b[15], b[14], b[13], b[12], b[11], b[10], b[9], b[8],

  222.                              b[7],  b[6],  b[5],  b[4],  b[3],  b[2],  b[1], b[0]);

  223.     }

  224. 

  225.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int8_t* dest) noexcept

  226.     {

  227.         SIMDFallbackOps<int8_t, __m128i>::store (value, dest);

  228.     }

  229. 

  230.     static forcedinline int8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  231.     {

  232.        #ifdef __SSSE3__

  233.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  234.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  235. 

  236.         for (int i = 0; i < 3; ++i)

  237.         {

  238.             lo = _mm_hadd_epi16 (lo, lo);

  239.             hi = _mm_hadd_epi16 (hi, hi);

  240.         }

  241. 

  242.         const int8_t* lo_ptr = reinterpret_cast<const int8_t*> (&lo);

  243.         const int8_t* hi_ptr = reinterpret_cast<const int8_t*> (&hi);

  244. 

  245.         return lo_ptr[0] + hi_ptr[0];

  246.        #else

  247.         int8_t sum = 0;

  248.         const int8_t* src = reinterpret_cast<const int8_t*> (&a);

  249. 

  250.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int8_t)); ++i)

  251.             sum += src [i];

  252. 

  253.         return sum;

  254.        #endif

  255.     }

  256. 

  257.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  258.     {

  259.         // unpack and multiply

  260.         __m128i even = _mm_mullo_epi16 (a, b);

  261.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  262. 

  263.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  264.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  265.     }

  266. };

  267. 

  268. //==============================================================================

  269. /** Unsigned 8-bit integer SSE intrinsics.

  270. 

  271.     @tags{DSP}

  272. */

  273. template <>

  274. struct SIMDNativeOps<uint8_t>

  275. {

  276.     //==============================================================================

  277.     typedef __m128i vSIMDType;

  278. 

  279.     //==============================================================================

  280.     DECLARE_SSE_SIMD_CONST (uint8_t, kHighBit);

  281.     DECLARE_SSE_SIMD_CONST (uint8_t, kAllBitsSet);

  282. 

  283.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint8_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  284.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  285.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint8_t s) noexcept                             { return _mm_set1_epi8 ((int8_t) s); }

  286.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi8 (a, b); }

  287.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi8 (a, b); }

  288.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  289.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  290.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  291.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  292.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  293.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu8 (a, b); }

  294.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu8 (a, b); }

  295.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi8 (a, b); }

  296.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi8 (ssign (a), ssign (b)); }

  297.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  298.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  299.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  300.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  301. 

  302.     //==============================================================================

  303.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint8_t* a) noexcept

  304.     {

  305.         const auto* b = reinterpret_cast<const char*> (a);

  306.         return _mm_set_epi8 (b[15], b[14], b[13], b[12], b[11], b[10], b[9], b[8],

  307.                              b[7],  b[6],  b[5],  b[4],  b[3],  b[2],  b[1], b[0]);

  308.     }

  309. 

  310.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint8_t* dest) noexcept

  311.     {

  312.         SIMDFallbackOps<uint8_t, __m128i>::store (value, dest);

  313.     }

  314. 

  315.     static forcedinline uint8_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  316.     {

  317.        #ifdef __SSSE3__

  318.         __m128i lo = _mm_unpacklo_epi8 (a, _mm_setzero_si128());

  319.         __m128i hi = _mm_unpackhi_epi8 (a, _mm_setzero_si128());

  320. 

  321.         for (int i = 0; i < 3; ++i)

  322.         {

  323.             lo = _mm_hadd_epi16 (lo, lo);

  324.             hi = _mm_hadd_epi16 (hi, hi);

  325.         }

  326. 

  327.         const uint8_t* lo_ptr = reinterpret_cast<const uint8_t*> (&lo);

  328.         const uint8_t* hi_ptr = reinterpret_cast<const uint8_t*> (&hi);

  329. 

  330.         return lo_ptr[0] + hi_ptr[0];

  331.        #else

  332.         uint8_t sum = 0;

  333.         const uint8_t* src = reinterpret_cast<const uint8_t*> (&a);

  334. 

  335.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int8_t)); ++i)

  336.             sum += src [i];

  337. 

  338.         return sum;

  339.        #endif

  340.     }

  341. 

  342.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b)

  343.     {

  344.         // unpack and multiply

  345.         __m128i even = _mm_mullo_epi16 (a, b);

  346.         __m128i odd  = _mm_mullo_epi16 (_mm_srli_epi16 (a, 8), _mm_srli_epi16 (b, 8));

  347. 

  348.         return _mm_or_si128 (_mm_slli_epi16 (odd, 8),

  349.                              _mm_srli_epi16 (_mm_slli_epi16 (even, 8), 8));

  350.     }

  351. };

  352. 

  353. //==============================================================================

  354. /** Signed 16-bit integer SSE intrinsics.

  355. 

  356.     @tags{DSP}

  357. */

  358. template <>

  359. struct SIMDNativeOps<int16_t>

  360. {

  361.     //==============================================================================

  362.     typedef __m128i vSIMDType;

  363. 

  364.     //==============================================================================

  365.     DECLARE_SSE_SIMD_CONST (int16_t, kAllBitsSet);

  366. 

  367.     //==============================================================================

  368.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int16_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  369.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int16_t s) noexcept                             { return _mm_set1_epi16 (s); }

  370.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  371.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  372.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  373.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  374.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  375.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  376.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  377.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  378.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epi16 (a, b); }

  379.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epi16 (a, b); }

  380.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  381.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (a, b); }

  382.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  383.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  384.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  385.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  386. 

  387.     //==============================================================================

  388.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int16_t* a) noexcept

  389.     {

  390.         return _mm_set_epi16 (a[7], a[6], a[5], a[4], a[3], a[2], a[1], a[0]);

  391.     }

  392. 

  393.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int16_t* dest) noexcept

  394.     {

  395.         SIMDFallbackOps<int16_t, __m128i>::store (value, dest);

  396.     }

  397. 

  398.     static forcedinline int16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  399.     {

  400.        #ifdef __SSSE3__

  401.         __m128i tmp = _mm_hadd_epi16 (a, a);

  402.         tmp = _mm_hadd_epi16 (tmp, tmp);

  403.         tmp = _mm_hadd_epi16 (tmp, tmp);

  404.         return *reinterpret_cast<int16_t*> (&tmp);

  405. 

  406.        #else

  407.         int16_t sum = 0;

  408.         const int16_t* src = reinterpret_cast<const int16_t*> (&a);

  409. 

  410.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int16_t)); ++i)

  411.             sum += src [i];

  412. 

  413.         return sum;

  414.        #endif

  415.     }

  416. };

  417. 

  418. //==============================================================================

  419. /** Unsigned 16-bit integer SSE intrinsics.

  420. 

  421.     @tags{DSP}

  422. */

  423. template <>

  424. struct SIMDNativeOps<uint16_t>

  425. {

  426.     //==============================================================================

  427.     typedef __m128i vSIMDType;

  428. 

  429.     //==============================================================================

  430.     DECLARE_SSE_SIMD_CONST (uint16_t, kHighBit);

  431.     DECLARE_SSE_SIMD_CONST (uint16_t, kAllBitsSet);

  432. 

  433.     //==============================================================================

  434.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint16_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  435.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  436.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint16_t s) noexcept                            { return _mm_set1_epi16 ((int16_t) s); }

  437.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi16 (a, b); }

  438.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi16 (a, b); }

  439.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept                     { return _mm_mullo_epi16 (a, b); }

  440.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  441.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  442.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  443.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  444.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  445.    #if defined(__SSE4__)

  446.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { return _mm_min_epu16 (a, b); }

  447.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { return _mm_max_epu16 (a, b); }

  448.    #else

  449.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  450.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  451.    #endif

  452.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi16 (a, b); }

  453.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi16 (ssign (a), ssign (b)); }

  454.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  455.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  456.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  457.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  458. 

  459.     //==============================================================================

  460.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint16_t* a) noexcept

  461.     {

  462.         const auto* b = reinterpret_cast<const int16_t*> (a);

  463.         return _mm_set_epi16 (b[7], b[6], b[5], b[4], b[3], b[2], b[1], b[0]);

  464.     }

  465. 

  466.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint16_t* dest) noexcept

  467.     {

  468.         SIMDFallbackOps<uint16_t, __m128i>::store (value, dest);

  469.     }

  470. 

  471.     static forcedinline uint16_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  472.     {

  473.        #ifdef __SSSE3__

  474.         __m128i tmp = _mm_hadd_epi16 (a, a);

  475.         tmp = _mm_hadd_epi16 (tmp, tmp);

  476.         tmp = _mm_hadd_epi16 (tmp, tmp);

  477.         return *reinterpret_cast<uint16_t*> (&tmp);

  478.        #else

  479.         uint16_t sum = 0;

  480.         const uint16_t* src = reinterpret_cast<const uint16_t*> (&a);

  481. 

  482.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(uint16_t)); ++i)

  483.             sum += src [i];

  484. 

  485.         return sum;

  486.        #endif

  487.     }

  488. };

  489. 

  490. //==============================================================================

  491. /** Signed 32-bit integer SSE intrinsics.

  492. 

  493.     @tags{DSP}

  494. */

  495. template <>

  496. struct SIMDNativeOps<int32_t>

  497. {

  498.     //==============================================================================

  499.     typedef __m128i vSIMDType;

  500. 

  501.     //==============================================================================

  502.     DECLARE_SSE_SIMD_CONST (int32_t, kAllBitsSet);

  503. 

  504.     //==============================================================================

  505.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int32_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  506.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int32_t s) noexcept                             { return _mm_set1_epi32 (s); }

  507.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int32_t* a) noexcept                        { return _mm_set_epi32 (a[3], a[2], a[1], a[0]); }

  508.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  509.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  510.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  511.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  512.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  513.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  514.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  515.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  516.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (a, b); }

  517.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  518.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  519.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  520.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  521. 

  522.     //==============================================================================

  523.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int32_t* dest) noexcept

  524.     {

  525.         SIMDFallbackOps<int32_t, __m128i>::store (value, dest);

  526.     }

  527. 

  528.     static forcedinline int32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  529.     {

  530.        #ifdef __SSSE3__

  531.         __m128i tmp = _mm_hadd_epi32 (a, a);

  532.         tmp = _mm_hadd_epi32 (tmp, tmp);

  533.         return *reinterpret_cast<int32_t*> (&tmp);

  534.        #else

  535.         int32_t sum = 0;

  536.         const int32_t* src = reinterpret_cast<const int32_t*> (&a);

  537. 

  538.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(int32_t)); ++i)

  539.             sum += src [i];

  540. 

  541.         return sum;

  542.        #endif

  543.     }

  544. 

  545.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  546.     {

  547.        #if defined(__SSE4_1__)

  548.         return _mm_mullo_epi32 (a, b);

  549.        #else

  550.         __m128i even = _mm_mul_epu32 (a,b);

  551.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  552.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32(even, _MM_SHUFFLE (0,0,2,0)),

  553.                                    _mm_shuffle_epi32(odd,  _MM_SHUFFLE (0,0,2,0)));

  554.        #endif

  555.     }

  556. 

  557.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  558.     {

  559.        #if defined(__SSE4_1__)

  560.         return _mm_min_epi32 (a, b);

  561.        #else

  562.         __m128i lt = greaterThan (b, a);

  563.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  564.        #endif

  565.     }

  566. 

  567.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  568.     {

  569.        #if defined(__SSE4_1__)

  570.         return _mm_max_epi32 (a, b);

  571.        #else

  572.         __m128i gt = greaterThan (a, b);

  573.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  574.        #endif

  575.     }

  576. };

  577. 

  578. //==============================================================================

  579. /** Unsigned 32-bit integer SSE intrinsics.

  580. 

  581.     @tags{DSP}

  582. */

  583. template <>

  584. struct SIMDNativeOps<uint32_t>

  585. {

  586.     //==============================================================================

  587.     typedef __m128i vSIMDType;

  588. 

  589.     //==============================================================================

  590.     DECLARE_SSE_SIMD_CONST (uint32_t, kAllBitsSet);

  591.     DECLARE_SSE_SIMD_CONST (uint32_t, kHighBit);

  592. 

  593.     //==============================================================================

  594.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint32_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  595.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  596.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint32_t s) noexcept                            { return _mm_set1_epi32 ((int32_t) s); }

  597.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi32 (a, b); }

  598.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi32 (a, b); }

  599.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  600.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  601.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  602.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  603.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  604.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept                   { return _mm_cmpeq_epi32 (a, b); }

  605.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept             { return _mm_cmpgt_epi32 (ssign (a), ssign (b)); }

  606.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  607.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  608.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  609.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  610. 

  611.     //==============================================================================

  612.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint32_t* a) noexcept

  613.     {

  614.         const auto* b = reinterpret_cast<const int32_t*> (a);

  615.         return _mm_set_epi32 (b[3], b[2], b[1], b[0]);

  616.     }

  617. 

  618.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint32_t* dest) noexcept

  619.     {

  620.         SIMDFallbackOps<uint32_t, __m128i>::store (value, dest);

  621.     }

  622. 

  623.     static forcedinline uint32_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  624.     {

  625.        #ifdef __SSSE3__

  626.         __m128i tmp = _mm_hadd_epi32 (a, a);

  627.         tmp = _mm_hadd_epi32 (tmp, tmp);

  628.         return *reinterpret_cast<uint32_t*> (&tmp);

  629.        #else

  630.         uint32_t sum = 0;

  631.         const uint32_t* src = reinterpret_cast<const uint32_t*> (&a);

  632. 

  633.         for (std::size_t i = 0; i < (sizeof (vSIMDType) / sizeof(uint32_t)); ++i)

  634.             sum += src [i];

  635. 

  636.         return sum;

  637.        #endif

  638.     }

  639. 

  640.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  641.     {

  642.        #if defined(__SSE4_1__)

  643.         return _mm_mullo_epi32 (a, b);

  644.        #else

  645.         __m128i even = _mm_mul_epu32 (a,b);

  646.         __m128i odd = _mm_mul_epu32 (_mm_srli_si128 (a,4), _mm_srli_si128 (b,4));

  647.         return _mm_unpacklo_epi32 (_mm_shuffle_epi32(even, _MM_SHUFFLE (0,0,2,0)),

  648.                                    _mm_shuffle_epi32(odd,  _MM_SHUFFLE (0,0,2,0)));

  649.        #endif

  650.     }

  651. 

  652.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept

  653.     {

  654.        #if defined(__SSE4_1__)

  655.         return _mm_min_epi32 (a, b);

  656.        #else

  657.         __m128i lt = greaterThan (b, a);

  658.         return bit_or (bit_and (lt, a), bit_andnot (lt, b));

  659.        #endif

  660.     }

  661. 

  662.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept

  663.     {

  664.        #if defined(__SSE4_1__)

  665.         return _mm_max_epi32 (a, b);

  666.        #else

  667.         __m128i gt = greaterThan (a, b);

  668.         return bit_or (bit_and (gt, a), bit_andnot (gt, b));

  669.        #endif

  670.     }

  671. };

  672. 

  673. //==============================================================================

  674. /** Signed 64-bit integer SSE intrinsics.

  675. 

  676.     @tags{DSP}

  677. */

  678. template <>

  679. struct SIMDNativeOps<int64_t>

  680. {

  681.     //==============================================================================

  682.     typedef __m128i vSIMDType;

  683. 

  684.     //==============================================================================

  685.     DECLARE_SSE_SIMD_CONST (int64_t, kAllBitsSet);

  686. 

  687.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (int64_t s) noexcept

  688.     {

  689.         __m128i retval;

  690.         int64_t* ptr = reinterpret_cast<int64_t*> (&retval);

  691.         ptr[0] = ptr[1] = s;

  692.         return retval;

  693.     }

  694. 

  695.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const int64_t* a) noexcept                        { return _mm_set_epi64x (a[1], a[0]); }

  696.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const int64_t* a) noexcept                      { return *reinterpret_cast<const __m128i*> (a); }

  697.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  698.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  699.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  700.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  701.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  702.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  703.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  704.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  705.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  706.     static forcedinline __m128i greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  707.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  708.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  709.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  710. 

  711.     //==============================================================================

  712.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, int64_t* dest) noexcept

  713.     {

  714.         SIMDFallbackOps<int64_t, __m128i>::store (value, dest);

  715.     }

  716. 

  717.     static forcedinline int64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  718.     {

  719.         const int64_t* ptr = reinterpret_cast<const int64_t*> (&a);

  720.         return ptr[0] + ptr[1];

  721.     }

  722. 

  723.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  724.     {

  725.         __m128i retval;

  726. 

  727.         const int64_t* aptr = reinterpret_cast<const int64_t*> (&a);

  728.         const int64_t* bptr = reinterpret_cast<const int64_t*> (&b);

  729.         int64_t* dst =  reinterpret_cast<int64_t*> (&retval);

  730. 

  731.         dst[0] = aptr[0] * bptr[0];

  732.         dst[1] = aptr[1] * bptr[1];

  733. 

  734.         return retval;

  735.     }

  736. 

  737.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  738.     {

  739.        #if defined(__SSE4_1__)

  740.         return _mm_cmpeq_epi64 (a, b);

  741.        #else

  742.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  743.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  744.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  745.        #endif

  746.     }

  747. 

  748.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  749.     {

  750.        #if defined(__SSE4_1__) && !defined(__clang__)

  751.         return _mm_cmpgt_epi64 (a, b);

  752.        #else

  753.         __m128i retval;

  754. 

  755.         const int64_t* aptr = reinterpret_cast<const int64_t*> (&a);

  756.         const int64_t* bptr = reinterpret_cast<const int64_t*> (&b);

  757.         int64_t* dst =  reinterpret_cast<int64_t*> (&retval);

  758. 

  759.         dst[0] = aptr[0] > bptr[0] ? -1LL : 0;

  760.         dst[1] = aptr[1] > bptr[1] ? -1LL : 0;

  761. 

  762.         return retval;

  763.        #endif

  764.     }

  765. };

  766. 

  767. //==============================================================================

  768. /** Unsigned 64-bit integer SSE intrinsics.

  769. 

  770.     @tags{DSP}

  771. */

  772. template <>

  773. struct SIMDNativeOps<uint64_t>

  774. {

  775.     //==============================================================================

  776.     typedef __m128i vSIMDType;

  777. 

  778.     //==============================================================================

  779.     DECLARE_SSE_SIMD_CONST (uint64_t, kAllBitsSet);

  780.     DECLARE_SSE_SIMD_CONST (uint64_t, kHighBit);

  781. 

  782.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE expand (uint64_t s) noexcept

  783.     {

  784.          __m128i retval;

  785.         uint64_t* ptr = reinterpret_cast<uint64_t*> (&retval);

  786.         ptr[0] = ptr[1] = s;

  787.         return retval;

  788.     }

  789. 

  790.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE vconst (const uint64_t* a) noexcept                     { return *reinterpret_cast<const __m128i*> (a); }

  791.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE ssign (__m128i a) noexcept                              { return _mm_xor_si128 (a, vconst (kHighBit)); }

  792.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE add (__m128i a, __m128i b) noexcept                     { return _mm_add_epi64 (a, b); }

  793.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE sub (__m128i a, __m128i b) noexcept                     { return _mm_sub_epi64 (a, b); }

  794.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_and (__m128i a, __m128i b) noexcept                 { return _mm_and_si128 (a, b); }

  795.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_or  (__m128i a, __m128i b) noexcept                 { return _mm_or_si128  (a, b); }

  796.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_xor (__m128i a, __m128i b) noexcept                 { return _mm_xor_si128 (a, b); }

  797.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_andnot (__m128i a, __m128i b) noexcept              { return _mm_andnot_si128 (a, b); }

  798.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE bit_not (__m128i a) noexcept                            { return _mm_andnot_si128 (a, vconst (kAllBitsSet)); }

  799.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE min (__m128i a, __m128i b) noexcept                     { __m128i lt = greaterThan (b, a); return bit_or (bit_and (lt, a), bit_andnot (lt, b)); }

  800.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE max (__m128i a, __m128i b) noexcept                     { __m128i gt = greaterThan (a, b); return bit_or (bit_and (gt, a), bit_andnot (gt, b)); }

  801.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThanOrEqual (__m128i a, __m128i b) noexcept      { return bit_or (greaterThan (a, b), equal (a,b)); }

  802.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE multiplyAdd (__m128i a, __m128i b, __m128i c) noexcept  { return add (a, mul (b, c)); }

  803.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE notEqual (__m128i a, __m128i b) noexcept                { return bit_not (equal (a, b)); }

  804.     static forcedinline bool    JUCE_VECTOR_CALLTYPE allEqual (__m128i a, __m128i b) noexcept                { return (_mm_movemask_epi8 (equal (a, b)) == 0xffff); }

  805. 

  806.     //==============================================================================

  807.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE load (const uint64_t* a) noexcept

  808.     {

  809.         const auto* b = reinterpret_cast<const int64_t*> (a);

  810.         return _mm_set_epi64x (b[1], b[0]);

  811.     }

  812. 

  813.     static forcedinline void JUCE_VECTOR_CALLTYPE store (__m128i value, uint64_t* dest) noexcept

  814.     {

  815.         SIMDFallbackOps<uint64_t, __m128i>::store (value, dest);

  816.     }

  817. 

  818.     static forcedinline uint64_t JUCE_VECTOR_CALLTYPE sum (__m128i a) noexcept

  819.     {

  820.         const uint64_t* ptr = reinterpret_cast<const uint64_t*> (&a);

  821.         return ptr[0] + ptr[1];

  822.     }

  823. 

  824.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE mul (__m128i a, __m128i b) noexcept

  825.     {

  826.         __m128i retval;

  827. 

  828.         const uint64_t* aptr = reinterpret_cast<const uint64_t*> (&a);

  829.         const uint64_t* bptr = reinterpret_cast<const uint64_t*> (&b);

  830.         uint64_t* dst =  reinterpret_cast<uint64_t*> (&retval);

  831. 

  832.         dst[0] = aptr[0] * bptr[0];

  833.         dst[1] = aptr[1] * bptr[1];

  834. 

  835.         return retval;

  836.     }

  837. 

  838.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE equal (__m128i a, __m128i b) noexcept

  839.     {

  840.        #if defined(__SSE4_1__)

  841.         return _mm_cmpeq_epi64 (a, b);

  842.        #else

  843.         __m128i bitmask = _mm_cmpeq_epi32 (a, b);

  844.         bitmask = _mm_and_si128 (bitmask, _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 3, 0, 1)));

  845.         return _mm_shuffle_epi32 (bitmask, _MM_SHUFFLE (2, 2, 0, 0));

  846.        #endif

  847.     }

  848. 

  849.     static forcedinline __m128i JUCE_VECTOR_CALLTYPE greaterThan (__m128i a, __m128i b) noexcept

  850.     {

  851.        #if defined(__SSE4_1__) && !defined(__clang__)

  852.         return _mm_cmpgt_epi64 (a, b);

  853.        #else

  854.         __m128i retval;

  855. 

  856.         const uint64_t* aptr = reinterpret_cast<const uint64_t*> (&a);

  857.         const uint64_t* bptr = reinterpret_cast<const uint64_t*> (&b);

  858.         uint64_t* dst =  reinterpret_cast<uint64_t*> (&retval);

  859. 

  860.         dst[0] = aptr[0] > bptr[0] ? (uint64_t) -1LL : 0;

  861.         dst[1] = aptr[1] > bptr[1] ? (uint64_t) -1LL : 0;

  862. 

  863.         return retval;

  864.        #endif

  865.     }

  866. };

  867. 

  868. #endif

  869. 

  870. } // namespace dsp

  871. } // namespace juce