jackaudio
/
jack2
mirror of https://github.com/jackaudio/jack2
1
0
Fork 0
Code Releases 45 Activity
jack2 codebase
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3975 Commits
8 Branches
67MB
Tree: 8d51e9a005
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '8d51e9a005'
${ noResults }
jack2/common/memops.c

1470 lines
41KB
Raw Blame History 

  1. /*

  2.     Copyright (C) 2000 Paul Davis 

  3. 

  4.     This program is free software; you can redistribute it and/or modify

  5.     it under the terms of the GNU General Public License as published by

  6.     the Free Software Foundation; either version 2 of the License, or

  7.     (at your option) any later version.

  8. 

  9.     This program is distributed in the hope that it will be useful,

  10.     but WITHOUT ANY WARRANTY; without even the implied warranty of

  11.     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  12.     GNU General Public License for more details.

  13. 

  14.     You should have received a copy of the GNU General Public License

  15.     along with this program; if not, write to the Free Software

  16.     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

  17. 

  18. */

  19. 

  20. #define _ISOC9X_SOURCE  1

  21. #define _ISOC99_SOURCE  1

  22. 

  23. #define __USE_ISOC9X    1

  24. #define __USE_ISOC99    1

  25. 

  26. #include <stdio.h>

  27. #include <string.h>

  28. #include <math.h>

  29. #include <memory.h>

  30. #include <stdlib.h>

  31. #include <stdint.h>

  32. #include <limits.h>

  33. #ifdef __linux__

  34. #include <endian.h>

  35. #endif

  36. #include "memops.h"

  37. 

  38. #if defined (__SSE2__) && !defined (__sun__)

  39. #include <emmintrin.h>

  40. #ifdef __SSE4_1__

  41. #include <smmintrin.h>

  42. #endif

  43. #endif

  44. 

  45. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  46. #include <arm_neon.h>

  47. #endif

  48. 

  49. /* Notes about these *_SCALING values.

  50. 

  51.    the MAX_<N>BIT values are floating point. when multiplied by

  52.    a full-scale normalized floating point sample value (-1.0..+1.0)

  53.    they should give the maximum value representable with an integer

  54.    sample type of N bits. Note that this is asymmetric. Sample ranges 

  55.    for signed integer, 2's complement values are -(2^(N-1) to +(2^(N-1)-1)

  56. 

  57.    Complications

  58.    -------------

  59.    If we use +2^(N-1) for the scaling factors, we run into a problem:

  60. 

  61.    if we start with a normalized float value of -1.0, scaling

  62.    to 24 bits would give -8388608 (-2^23), which is ideal.

  63.    But with +1.0, we get +8388608, which is technically out of range.

  64. 

  65.    We never multiply a full range normalized value by this constant,

  66.    but we could multiply it by a positive value that is close enough to +1.0

  67.    to produce a value > +(2^(N-1)-1.

  68. 

  69.    There is no way around this paradox without wasting CPU cycles to determine

  70.    which scaling factor to use (i.e. determine if its negative or not,

  71.    use the right factor).

  72. 

  73.    So, for now (October 2008) we use 2^(N-1)-1 as the scaling factor.

  74. */

  75. 

  76. #define SAMPLE_24BIT_SCALING  8388607.0f

  77. #define SAMPLE_16BIT_SCALING  32767.0f

  78. 

  79. /* these are just values to use if the floating point value was out of range

  80.    

  81.    advice from Fons Adriaensen: make the limits symmetrical

  82.  */

  83. 

  84. #define SAMPLE_24BIT_MAX  8388607  

  85. #define SAMPLE_24BIT_MIN  -8388607 

  86. #define SAMPLE_24BIT_MAX_F  8388607.0f  

  87. #define SAMPLE_24BIT_MIN_F  -8388607.0f 

  88. 

  89. #define SAMPLE_16BIT_MAX  32767

  90. #define SAMPLE_16BIT_MIN  -32767

  91. #define SAMPLE_16BIT_MAX_F  32767.0f

  92. #define SAMPLE_16BIT_MIN_F  -32767.0f

  93. 

  94. /* these mark the outer edges of the range considered "within" range

  95.    for a floating point sample value. values outside (and on the boundaries) 

  96.    of this range will be clipped before conversion; values within this 

  97.    range will be scaled to appropriate values for the target sample

  98.    type.

  99. */

  100. 

  101. #define NORMALIZED_FLOAT_MIN -1.0f

  102. #define NORMALIZED_FLOAT_MAX  1.0f

  103. 

  104. /* define this in case we end up on a platform that is missing

  105.    the real lrintf functions

  106. */

  107. 

  108. #define f_round(f) lrintf(f)

  109. 

  110. #define float_16(s, d)\

  111. 	if ((s) <= NORMALIZED_FLOAT_MIN) {\

  112. 		(d) = SAMPLE_16BIT_MIN;\

  113. 	} else if ((s) >= NORMALIZED_FLOAT_MAX) {\

  114. 		(d) = SAMPLE_16BIT_MAX;\

  115. 	} else {\

  116. 		(d) = f_round ((s) * SAMPLE_16BIT_SCALING);\

  117. 	}

  118. 

  119. /* call this when "s" has already been scaled (e.g. when dithering)

  120.  */

  121. 

  122. #define float_16_scaled(s, d)\

  123.         if ((s) <= SAMPLE_16BIT_MIN_F) {\

  124. 		(d) = SAMPLE_16BIT_MIN_F;\

  125. 	} else if ((s) >= SAMPLE_16BIT_MAX_F) {	\

  126. 		(d) = SAMPLE_16BIT_MAX;\

  127. 	} else {\

  128. 	        (d) = f_round ((s));\

  129. 	}

  130. 

  131. #define float_24u32(s, d) \

  132. 	if ((s) <= NORMALIZED_FLOAT_MIN) {\

  133. 		(d) = SAMPLE_24BIT_MIN << 8;\

  134. 	} else if ((s) >= NORMALIZED_FLOAT_MAX) {\

  135. 		(d) = SAMPLE_24BIT_MAX << 8;\

  136. 	} else {\

  137. 		(d) = f_round ((s) * SAMPLE_24BIT_SCALING) << 8;\

  138. 	}

  139. 

  140. #define float_24l32(s, d)                                              \

  141. 	if ((s) <= NORMALIZED_FLOAT_MIN) {\

  142. 		(d) = SAMPLE_24BIT_MIN;                                        \

  143. 	} else if ((s) >= NORMALIZED_FLOAT_MAX) {\

  144. 		(d) = SAMPLE_24BIT_MAX;                                        \

  145. 	} else {\

  146. 		(d) = f_round ((s) * SAMPLE_24BIT_SCALING);                    \

  147. 	}

  148. 

  149. /* call this when "s" has already been scaled (e.g. when dithering)

  150.  */

  151. 

  152. #define float_24u32_scaled(s, d)\

  153.         if ((s) <= SAMPLE_24BIT_MIN_F) {\

  154. 		(d) = SAMPLE_24BIT_MIN << 8;\

  155. 	} else if ((s) >= SAMPLE_24BIT_MAX_F) {	\

  156. 		(d) = SAMPLE_24BIT_MAX << 8;		\

  157. 	} else {\

  158. 		(d) = f_round ((s)) << 8; \

  159. 	}

  160. 

  161. #define float_24(s, d) \

  162. 	if ((s) <= NORMALIZED_FLOAT_MIN) {\

  163. 		(d) = SAMPLE_24BIT_MIN;\

  164. 	} else if ((s) >= NORMALIZED_FLOAT_MAX) {\

  165. 		(d) = SAMPLE_24BIT_MAX;\

  166. 	} else {\

  167. 		(d) = f_round ((s) * SAMPLE_24BIT_SCALING);\

  168. 	}

  169. 

  170. /* call this when "s" has already been scaled (e.g. when dithering)

  171.  */

  172. 

  173. #define float_24_scaled(s, d)\

  174.         if ((s) <= SAMPLE_24BIT_MIN_F) {\

  175. 		(d) = SAMPLE_24BIT_MIN;\

  176. 	} else if ((s) >= SAMPLE_24BIT_MAX_F) {	\

  177. 		(d) = SAMPLE_24BIT_MAX;		\

  178. 	} else {\

  179. 		(d) = f_round ((s)); \

  180. 	}

  181. 

  182. 

  183. #if defined (__SSE2__) && !defined (__sun__)

  184. 

  185. /* generates same as _mm_set_ps(1.f, 1.f, 1f., 1f) but faster  */

  186. static inline __m128 gen_one(void)

  187. {

  188.     volatile __m128i x = { 0 }; /* shut up, GCC */

  189.     __m128i ones = _mm_cmpeq_epi32(x, x);

  190.     return (__m128)_mm_slli_epi32 (_mm_srli_epi32(ones, 25), 23);

  191. }

  192. 

  193. static inline __m128 clip(__m128 s, __m128 min, __m128 max)

  194. {

  195.     return _mm_min_ps(max, _mm_max_ps(s, min));

  196. }

  197. 

  198. static inline __m128i float_24_sse(__m128 s)

  199. {

  200.     const __m128 upper_bound = gen_one(); /* NORMALIZED_FLOAT_MAX */

  201.     const __m128 lower_bound = _mm_sub_ps(_mm_setzero_ps(), upper_bound);

  202. 

  203.     __m128 clipped = clip(s, lower_bound, upper_bound);

  204.     __m128 scaled = _mm_mul_ps(clipped, _mm_set1_ps(SAMPLE_24BIT_SCALING));

  205.     return _mm_cvtps_epi32(scaled);

  206. }

  207. #endif

  208. 

  209. 

  210. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  211. 

  212. static inline float32x4_t clip(float32x4_t s, float32x4_t min, float32x4_t max)

  213. {

  214. 	return vminq_f32(max, vmaxq_f32(s, min));

  215. }

  216. 

  217. static inline int32x4_t float_24_neon(float32x4_t s)

  218. {

  219. 	const float32x4_t upper_bound = vdupq_n_f32(NORMALIZED_FLOAT_MAX);

  220. 	const float32x4_t lower_bound = vdupq_n_f32(NORMALIZED_FLOAT_MIN);

  221. 

  222. 	float32x4_t clipped = clip(s, lower_bound, upper_bound);

  223. 	float32x4_t scaled = vmulq_f32(clipped, vdupq_n_f32(SAMPLE_24BIT_SCALING));

  224. 	return vcvtq_s32_f32(scaled);

  225. }

  226. 

  227. static inline int16x4_t float_16_neon(float32x4_t s)

  228. {

  229. 	const float32x4_t upper_bound = vdupq_n_f32(NORMALIZED_FLOAT_MAX);

  230. 	const float32x4_t lower_bound = vdupq_n_f32(NORMALIZED_FLOAT_MIN);

  231. 

  232. 	float32x4_t clipped = clip(s, lower_bound, upper_bound);

  233. 	float32x4_t scaled = vmulq_f32(clipped, vdupq_n_f32(SAMPLE_16BIT_SCALING));

  234. 	return vmovn_s32(vcvtq_s32_f32(scaled));

  235. }

  236. #endif

  237. 

  238. /* Linear Congruential noise generator. From the music-dsp list

  239.  * less random than rand(), but good enough and 10x faster 

  240.  */

  241. static unsigned int seed = 22222;

  242. 

  243. static inline unsigned int fast_rand() {

  244. 	seed = (seed * 196314165) + 907633515;

  245. 	return seed;

  246. }

  247. 

  248. /* functions for native float sample data */

  249. 

  250. void sample_move_floatLE_sSs (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip) {

  251. 	while (nsamples--) {

  252. 		*dst = *((float *) src);

  253. 		dst++;

  254. 		src += src_skip;

  255. 	}

  256. }

  257. 

  258. void sample_move_dS_floatLE (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state) {

  259. 	while (nsamples--) {

  260. 		*((float *) dst) = *src;

  261. 		dst += dst_skip;

  262. 		src++;

  263. 	}

  264. }

  265. 

  266. /* NOTES on function naming:

  267. 

  268.    foo_bar_d<TYPE>_s<TYPE>

  269. 

  270.    the "d<TYPE>" component defines the destination type for the operation

  271.    the "s<TYPE>" component defines the source type for the operation

  272. 

  273.    TYPE can be one of:

  274.    

  275.    S      - sample is a jack_default_audio_sample_t, currently (October 2008) a 32 bit floating point value

  276.    Ss     - like S but reverse endian from the host CPU

  277.    32u24  - sample is an signed 32 bit integer value, but data is in upper 24 bits only

  278.    32u24s - like 32u24 but reverse endian from the host CPU

  279.    32l24  - sample is an signed 32 bit integer value, but data is in lower 24 bits only

  280.    32l24s - like 32l24 but reverse endian from the host CPU

  281.    24     - sample is an signed 24 bit integer value

  282.    24s    - like 24 but reverse endian from the host CPU

  283.    16     - sample is an signed 16 bit integer value

  284.    16s    - like 16 but reverse endian from the host CPU

  285. 

  286.    For obvious reasons, the reverse endian versions only show as source types.

  287. 

  288.    This covers all known sample formats at 16 bits or larger.

  289. */   

  290. 

  291. /* functions for native integer sample data */

  292. 

  293. void sample_move_d32u24_sSs (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  294. {

  295. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  296. 	unsigned long unrolled = nsamples / 4;

  297. 	nsamples = nsamples & 3;

  298. 

  299. 	while (unrolled--) {

  300. 		float32x4_t samples = vld1q_f32(src);

  301. 		int32x4_t converted = float_24_neon(samples);

  302. 		int32x4_t shifted = vshlq_n_s32(converted, 8);

  303. 		shifted = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(shifted)));

  304. 

  305. 		switch(dst_skip) {

  306. 			case 4:

  307. 				vst1q_s32((int32_t*)dst, shifted);

  308. 				break;

  309. 			default:

  310. 				vst1q_lane_s32((int32_t*)(dst),            shifted, 0);

  311. 				vst1q_lane_s32((int32_t*)(dst+dst_skip),   shifted, 1);

  312. 				vst1q_lane_s32((int32_t*)(dst+2*dst_skip), shifted, 2);

  313.                 vst1q_lane_s32((int32_t*)(dst+3*dst_skip), shifted, 3);

  314. 				break;

  315. 		}

  316. 		dst += 4*dst_skip;

  317. 		src+= 4;

  318. 	}

  319. #endif

  320. 

  321. 	int32_t z;

  322. 

  323. 	while (nsamples--) {

  324. 

  325. 		float_24u32 (*src, z);

  326. 

  327. #if __BYTE_ORDER == __LITTLE_ENDIAN

  328. 		dst[0]=(char)(z>>24);

  329. 		dst[1]=(char)(z>>16);

  330. 		dst[2]=(char)(z>>8);

  331. 		dst[3]=(char)(z);

  332. #elif __BYTE_ORDER == __BIG_ENDIAN

  333. 		dst[0]=(char)(z);

  334. 		dst[1]=(char)(z>>8);

  335. 		dst[2]=(char)(z>>16);

  336. 		dst[3]=(char)(z>>24);

  337. #endif

  338. 		dst += dst_skip;

  339. 		src++;

  340. 	}

  341. }	

  342. 

  343. void sample_move_d32u24_sS (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  344. {

  345. #if defined (__SSE2__) && !defined (__sun__)

  346. 	__m128 int_max = _mm_set1_ps(SAMPLE_24BIT_MAX_F);

  347. 	__m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max);

  348. 	__m128 factor = int_max;

  349. 

  350. 	unsigned long unrolled = nsamples / 4;

  351. 	nsamples = nsamples & 3;

  352. 

  353. 	while (unrolled--) {

  354. 		__m128 in = _mm_load_ps(src);

  355. 		__m128 scaled = _mm_mul_ps(in, factor);

  356. 		__m128 clipped = clip(scaled, int_min, int_max);

  357. 

  358. 		__m128i y = _mm_cvttps_epi32(clipped);

  359. 		__m128i shifted = _mm_slli_epi32(y, 8);

  360. 

  361. #ifdef __SSE4_1__

  362. 		*(int32_t*)dst              = _mm_extract_epi32(shifted, 0);

  363. 		*(int32_t*)(dst+dst_skip)   = _mm_extract_epi32(shifted, 1);

  364. 		*(int32_t*)(dst+2*dst_skip) = _mm_extract_epi32(shifted, 2);

  365. 		*(int32_t*)(dst+3*dst_skip) = _mm_extract_epi32(shifted, 3);

  366. #else

  367. 		__m128i shuffled1 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(0, 3, 2, 1));

  368. 		__m128i shuffled2 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(1, 0, 3, 2));

  369. 		__m128i shuffled3 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(2, 1, 0, 3));

  370. 

  371. 		_mm_store_ss((float*)dst, (__m128)shifted);

  372. 

  373. 		_mm_store_ss((float*)(dst+dst_skip), (__m128)shuffled1);

  374. 		_mm_store_ss((float*)(dst+2*dst_skip), (__m128)shuffled2);

  375. 		_mm_store_ss((float*)(dst+3*dst_skip), (__m128)shuffled3);

  376. #endif

  377. 		dst += 4*dst_skip;

  378. 

  379. 		src+= 4;

  380. 	}

  381. 

  382. 	while (nsamples--) {

  383. 		__m128 in = _mm_load_ss(src);

  384. 		__m128 scaled = _mm_mul_ss(in, factor);

  385. 		__m128 clipped = _mm_min_ss(int_max, _mm_max_ss(scaled, int_min));

  386. 

  387. 		int y = _mm_cvttss_si32(clipped);

  388. 		*((int *) dst) = y<<8;

  389. 

  390. 		dst += dst_skip;

  391. 		src++;

  392. 	}

  393. 

  394. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  395. 	unsigned long unrolled = nsamples / 4;

  396. 	nsamples = nsamples & 3;

  397. 

  398. 	while (unrolled--) {

  399. 		float32x4_t samples = vld1q_f32(src);

  400. 		int32x4_t converted = float_24_neon(samples);

  401. 		int32x4_t shifted = vshlq_n_s32(converted, 8);

  402. 

  403. 		switch(dst_skip) {

  404. 			case 4:

  405. 				vst1q_s32((int32_t*)dst, shifted);

  406. 				break;

  407. 			default:

  408. 				vst1q_lane_s32((int32_t*)(dst),            shifted, 0);

  409. 				vst1q_lane_s32((int32_t*)(dst+dst_skip),   shifted, 1);

  410. 				vst1q_lane_s32((int32_t*)(dst+2*dst_skip), shifted, 2);

  411.                 vst1q_lane_s32((int32_t*)(dst+3*dst_skip), shifted, 3);

  412. 				break;

  413. 		}

  414. 		dst += 4*dst_skip;

  415. 

  416. 		src+= 4;

  417. 	}

  418. #endif

  419. 

  420. #if !defined (__SSE2__)

  421. 	while (nsamples--) {

  422. 		float_24u32 (*src, *((int32_t*) dst));

  423. 		dst += dst_skip;

  424. 		src++;

  425. 	}

  426. #endif

  427. }	

  428. 

  429. void sample_move_dS_s32u24s (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  430. {

  431. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  432. 	float32x4_t factor = vdupq_n_f32(1.0 / SAMPLE_24BIT_SCALING);

  433. 	unsigned long unrolled = nsamples / 4;

  434. 	while (unrolled--) {

  435. 		int32x4_t src128;

  436. 		switch(src_skip)

  437. 		{

  438. 			case 4:

  439. 				src128 = vld1q_s32((int32_t*)src);

  440. 				break;

  441. 			case 8:

  442. 				src128 = vld2q_s32((int32_t*)src).val[0];

  443. 				break;

  444. 			default:

  445. 				src128 = vld1q_lane_s32((int32_t*)src,              src128, 0);

  446. 				src128 = vld1q_lane_s32((int32_t*)(src+src_skip),   src128, 1);

  447. 				src128 = vld1q_lane_s32((int32_t*)(src+2*src_skip), src128, 2);

  448. 				src128 = vld1q_lane_s32((int32_t*)(src+3*src_skip), src128, 3);

  449. 				break;

  450. 		}

  451. 		src128 = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(src128)));

  452. 		int32x4_t shifted = vshrq_n_s32(src128, 8);

  453. 		float32x4_t as_float = vcvtq_f32_s32(shifted);

  454. 		float32x4_t divided = vmulq_f32(as_float, factor);

  455. 		vst1q_f32(dst, divided);

  456. 

  457. 		src += 4*src_skip;

  458. 		dst += 4;

  459. 	}

  460. 	nsamples = nsamples & 3;

  461. #endif

  462. 

  463. 	/* ALERT: signed sign-extension portability !!! */

  464. 

  465. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_24BIT_SCALING;

  466. 

  467. 	while (nsamples--) {

  468. 		int x;

  469. #if __BYTE_ORDER == __LITTLE_ENDIAN

  470. 		x = (unsigned char)(src[0]);

  471. 		x <<= 8;

  472. 		x |= (unsigned char)(src[1]);

  473. 		x <<= 8;

  474. 		x |= (unsigned char)(src[2]);

  475. 		x <<= 8;

  476. 		x |= (unsigned char)(src[3]);

  477. #elif __BYTE_ORDER == __BIG_ENDIAN

  478. 		x = (unsigned char)(src[3]);

  479. 		x <<= 8;

  480. 		x |= (unsigned char)(src[2]);

  481. 		x <<= 8;

  482. 		x |= (unsigned char)(src[1]);

  483. 		x <<= 8;

  484. 		x |= (unsigned char)(src[0]);

  485. #endif

  486. 		*dst = (x >> 8) * scaling;

  487. 		dst++;

  488. 		src += src_skip;

  489. 	}

  490. }	

  491. 

  492. void sample_move_dS_s32u24 (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  493. {

  494. #if defined (__SSE2__) && !defined (__sun__)

  495. 	unsigned long unrolled = nsamples / 4;

  496. 	static float inv_sample_max_24bit = 1.0 / SAMPLE_24BIT_SCALING;

  497. 	__m128 factor = _mm_set1_ps(inv_sample_max_24bit);

  498. 	while (unrolled--)

  499. 	{

  500. 		int i1 = *((int *) src);

  501. 		src+= src_skip;

  502. 		int i2 = *((int *) src);

  503. 		src+= src_skip;

  504. 		int i3 = *((int *) src);

  505. 		src+= src_skip;

  506. 		int i4 = *((int *) src);

  507. 		src+= src_skip;

  508. 

  509. 		__m128i src = _mm_set_epi32(i4, i3, i2, i1);

  510. 		__m128i shifted = _mm_srai_epi32(src, 8);

  511. 

  512. 		__m128 as_float = _mm_cvtepi32_ps(shifted);

  513. 		__m128 divided = _mm_mul_ps(as_float, factor);

  514. 

  515. 		_mm_storeu_ps(dst, divided);

  516. 

  517. 		dst += 4;

  518. 	}

  519. 	nsamples = nsamples & 3;

  520. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  521. 	unsigned long unrolled = nsamples / 4;

  522. 	float32x4_t factor = vdupq_n_f32(1.0 / SAMPLE_24BIT_SCALING);

  523. 	while (unrolled--) {

  524. 		int32x4_t src128;

  525. 		switch(src_skip) {

  526. 			case 4:

  527. 				src128 = vld1q_s32((int32_t*)src);

  528. 				break;

  529. 			case 8:

  530. 				src128 = vld2q_s32((int32_t*)src).val[0];

  531. 				break;

  532. 			default:

  533. 				src128 = vld1q_lane_s32((int32_t*)src,              src128, 0);

  534. 				src128 = vld1q_lane_s32((int32_t*)(src+src_skip),   src128, 1);

  535. 				src128 = vld1q_lane_s32((int32_t*)(src+2*src_skip), src128, 2);

  536. 				src128 = vld1q_lane_s32((int32_t*)(src+3*src_skip), src128, 3);

  537. 				break;

  538. 		}

  539. 		int32x4_t shifted = vshrq_n_s32(src128, 8);

  540. 		float32x4_t as_float = vcvtq_f32_s32(shifted);

  541. 		float32x4_t divided = vmulq_f32(as_float, factor);

  542. 		vst1q_f32(dst, divided);

  543. 

  544. 		src += 4*src_skip;

  545. 		dst += 4;

  546. 	}

  547. 	nsamples = nsamples & 3;

  548. #endif

  549. 

  550. 	/* ALERT: signed sign-extension portability !!! */

  551. 

  552. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_24BIT_SCALING;

  553. 	while (nsamples--) {

  554. 		*dst = (*((int *) src) >> 8) * scaling;

  555. 		dst++;

  556. 		src += src_skip;

  557. 	}

  558. }	

  559. 

  560. void sample_move_d32l24_sSs (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  561. {

  562. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  563. 	unsigned long unrolled = nsamples / 4;

  564. 	nsamples = nsamples & 3;

  565. 

  566. 	while (unrolled--) {

  567. 		float32x4_t samples = vld1q_f32(src);

  568. 		int32x4_t converted = float_24_neon(samples);

  569. 		converted = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(converted)));

  570. 

  571. 		switch(dst_skip) {

  572. 			case 4:

  573. 				vst1q_s32((int32_t*)dst, converted);

  574. 				break;

  575. 			default:

  576. 				vst1q_lane_s32((int32_t*)(dst),            converted, 0);

  577. 				vst1q_lane_s32((int32_t*)(dst+dst_skip),   converted, 1);

  578. 				vst1q_lane_s32((int32_t*)(dst+2*dst_skip), converted, 2);

  579.                 vst1q_lane_s32((int32_t*)(dst+3*dst_skip), converted, 3);

  580. 				break;

  581. 		}

  582. 		dst += 4*dst_skip;

  583. 		src+= 4;

  584. 	}

  585. #endif

  586. 

  587. 	int32_t z;

  588. 

  589. 	while (nsamples--) {

  590. 

  591. 		float_24l32 (*src, z);

  592. 

  593. #if __BYTE_ORDER == __LITTLE_ENDIAN

  594. 		dst[0]=(char)(z>>24);

  595. 		dst[1]=(char)(z>>16);

  596. 		dst[2]=(char)(z>>8);

  597. 		dst[3]=(char)(z);

  598. #elif __BYTE_ORDER == __BIG_ENDIAN

  599. 		dst[0]=(char)(z);

  600. 		dst[1]=(char)(z>>8);

  601. 		dst[2]=(char)(z>>16);

  602. 		dst[3]=(char)(z>>24);

  603. #endif

  604. 		dst += dst_skip;

  605. 		src++;

  606. 	}

  607. }	

  608. 

  609. void sample_move_d32l24_sS (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  610. {

  611. #if defined (__SSE2__) && !defined (__sun__)

  612. 	__m128 int_max = _mm_set1_ps(SAMPLE_24BIT_MAX_F);

  613. 	__m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max);

  614. 	__m128 factor = int_max;

  615. 

  616. 	unsigned long unrolled = nsamples / 4;

  617. 	nsamples = nsamples & 3;

  618. 

  619. 	while (unrolled--) {

  620. 		__m128 in = _mm_load_ps(src);

  621. 		__m128 scaled = _mm_mul_ps(in, factor);

  622. 		__m128 clipped = clip(scaled, int_min, int_max);

  623. 

  624. 		__m128i shifted = _mm_cvttps_epi32(clipped);

  625. 

  626. #ifdef __SSE4_1__

  627. 		*(int32_t*)dst              = _mm_extract_epi32(shifted, 0);

  628. 		*(int32_t*)(dst+dst_skip)   = _mm_extract_epi32(shifted, 1);

  629. 		*(int32_t*)(dst+2*dst_skip) = _mm_extract_epi32(shifted, 2);

  630. 		*(int32_t*)(dst+3*dst_skip) = _mm_extract_epi32(shifted, 3);

  631. #else

  632. 		__m128i shuffled1 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(0, 3, 2, 1));

  633. 		__m128i shuffled2 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(1, 0, 3, 2));

  634. 		__m128i shuffled3 = _mm_shuffle_epi32(shifted, _MM_SHUFFLE(2, 1, 0, 3));

  635. 

  636. 		_mm_store_ss((float*)dst, (__m128)shifted);

  637. 

  638. 		_mm_store_ss((float*)(dst+dst_skip), (__m128)shuffled1);

  639. 		_mm_store_ss((float*)(dst+2*dst_skip), (__m128)shuffled2);

  640. 		_mm_store_ss((float*)(dst+3*dst_skip), (__m128)shuffled3);

  641. #endif

  642. 		dst += 4*dst_skip;

  643. 

  644. 		src+= 4;

  645. 	}

  646. 

  647. 	while (nsamples--) {

  648. 		__m128 in = _mm_load_ss(src);

  649. 		__m128 scaled = _mm_mul_ss(in, factor);

  650. 		__m128 clipped = _mm_min_ss(int_max, _mm_max_ss(scaled, int_min));

  651. 

  652. 		int y = _mm_cvttss_si32(clipped);

  653. 		*((int *) dst) = y<<8;

  654. 

  655. 		dst += dst_skip;

  656. 		src++;

  657. 	}

  658. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  659. 	unsigned long unrolled = nsamples / 4;

  660. 	nsamples = nsamples & 3;

  661. 

  662. 	while (unrolled--) {

  663. 		float32x4_t samples = vld1q_f32(src);

  664. 		int32x4_t converted = float_24_neon(samples);

  665. 

  666. 		switch(dst_skip) {

  667. 			case 4:

  668. 				vst1q_s32((int32_t*)dst, converted);

  669. 				break;

  670. 			default:

  671. 				vst1q_lane_s32((int32_t*)(dst),            converted, 0);

  672. 				vst1q_lane_s32((int32_t*)(dst+dst_skip),   converted, 1);

  673. 				vst1q_lane_s32((int32_t*)(dst+2*dst_skip), converted, 2);

  674.                 vst1q_lane_s32((int32_t*)(dst+3*dst_skip), converted, 3);

  675. 				break;

  676. 		}

  677. 		dst += 4*dst_skip;

  678. 

  679. 		src+= 4;

  680. 	}

  681. #endif

  682. 

  683. #if !defined (__SSE2__)

  684. 	while (nsamples--) {

  685. 		float_24l32 (*src, *((int32_t*) dst));

  686. 		dst += dst_skip;

  687. 		src++;

  688. 	}

  689. #endif

  690. }	

  691. 

  692. void sample_move_dS_s32l24s (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  693. {

  694. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  695. 	float32x4_t factor = vdupq_n_f32(1.0 / SAMPLE_24BIT_SCALING);

  696. 	unsigned long unrolled = nsamples / 4;

  697. 	while (unrolled--) {

  698. 		uint32x4_t src128;

  699. 		switch(src_skip)

  700. 		{

  701. 			case 4:

  702. 				src128 = vld1q_u32((uint32_t*)src);

  703. 				break;

  704. 			case 8:

  705. 				src128 = vld2q_u32((uint32_t*)src).val[0];

  706. 				break;

  707. 			default:

  708. 				src128 = vld1q_lane_u32((uint32_t*)src,              src128, 0);

  709. 				src128 = vld1q_lane_u32((uint32_t*)(src+src_skip),   src128, 1);

  710. 				src128 = vld1q_lane_u32((uint32_t*)(src+2*src_skip), src128, 2);

  711. 				src128 = vld1q_lane_u32((uint32_t*)(src+3*src_skip), src128, 3);

  712. 				break;

  713. 		}

  714. 		src128 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(src128)));

  715. 		uint32x4_t toupper = vshlq_n_u32(src128, 8);

  716. 		int32x4_t shifted = vshrq_n_s32((int32x4_t)toupper, 8);

  717. 		float32x4_t as_float = vcvtq_f32_s32(shifted);

  718. 		float32x4_t divided = vmulq_f32(as_float, factor);

  719. 		vst1q_f32(dst, divided);

  720. 

  721. 		src += 4*src_skip;

  722. 		dst += 4;

  723. 	}

  724. 	nsamples = nsamples & 3;

  725. #endif

  726. 

  727. 	/* ALERT: signed sign-extension portability !!! */

  728. 

  729. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_24BIT_SCALING;

  730. 

  731. 	while (nsamples--) {

  732. 		int32_t x;

  733. #if __BYTE_ORDER == __LITTLE_ENDIAN

  734. 		x = (unsigned char)(src[0]);

  735. 		x <<= 8;

  736. 		x |= (unsigned char)(src[1]);

  737. 		x <<= 8;

  738. 		x |= (unsigned char)(src[2]);

  739. 		x <<= 8;

  740. 		x |= (unsigned char)(src[3]);

  741. #elif __BYTE_ORDER == __BIG_ENDIAN

  742. 		x = (unsigned char)(src[3]);

  743. 		x <<= 8;

  744. 		x |= (unsigned char)(src[2]);

  745. 		x <<= 8;

  746. 		x |= (unsigned char)(src[1]);

  747. 		x <<= 8;

  748. 		x |= (unsigned char)(src[0]);

  749. #endif

  750. 		*dst = (x >> 0) * scaling;

  751. 		dst++;

  752. 		src += src_skip;

  753. 	}

  754. }	

  755. 

  756. void sample_move_dS_s32l24 (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  757. {

  758. #if defined (__SSE2__) && !defined (__sun__)

  759. 	unsigned long unrolled = nsamples / 4;

  760. 	static float inv_sample_max_24bit = 1.0 / SAMPLE_24BIT_SCALING;

  761. 	__m128 factor = _mm_set1_ps(inv_sample_max_24bit);

  762. 	while (unrolled--)

  763. 	{

  764. 		int i1 = *((int *) src);

  765. 		src+= src_skip;

  766. 		int i2 = *((int *) src);

  767. 		src+= src_skip;

  768. 		int i3 = *((int *) src);

  769. 		src+= src_skip;

  770. 		int i4 = *((int *) src);

  771. 		src+= src_skip;

  772. 

  773. 		__m128i shifted = _mm_set_epi32(i4, i3, i2, i1);

  774. 

  775. 		__m128 as_float = _mm_cvtepi32_ps(shifted);

  776. 		__m128 divided = _mm_mul_ps(as_float, factor);

  777. 

  778. 		_mm_storeu_ps(dst, divided);

  779. 

  780. 		dst += 4;

  781. 	}

  782. 	nsamples = nsamples & 3;

  783. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  784. 	unsigned long unrolled = nsamples / 4;

  785. 	float32x4_t factor = vdupq_n_f32(1.0 / SAMPLE_24BIT_SCALING);

  786. 	while (unrolled--) {

  787. 		uint32x4_t src128;

  788. 		switch(src_skip) {

  789. 			case 4:

  790. 				src128 = vld1q_u32((uint32_t*)src);

  791. 				break;

  792. 			case 8:

  793. 				src128 = vld2q_u32((uint32_t*)src).val[0];

  794. 				break;

  795. 			default:

  796. 				src128 = vld1q_lane_u32((uint32_t*)src,              src128, 0);

  797. 				src128 = vld1q_lane_u32((uint32_t*)(src+src_skip),   src128, 1);

  798. 				src128 = vld1q_lane_u32((uint32_t*)(src+2*src_skip), src128, 2);

  799. 				src128 = vld1q_lane_u32((uint32_t*)(src+3*src_skip), src128, 3);

  800. 				break;

  801. 		}

  802. 		// Sign extension by moving to upper as unsigned, then down

  803. 		uint32x4_t toupper = vshlq_n_u32(src128, 8);

  804. 		int32x4_t shifted = vshrq_n_s32((int32x4_t)toupper, 8);

  805. 		float32x4_t as_float = vcvtq_f32_s32(shifted);

  806. 		float32x4_t divided = vmulq_f32(as_float, factor);

  807. 		vst1q_f32(dst, divided);

  808. 

  809. 		src += 4*src_skip;

  810. 		dst += 4;

  811. 	}

  812. 	nsamples = nsamples & 3;

  813. #endif

  814. 

  815. 	/* ALERT: signed sign-extension portability !!! */

  816. 

  817. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_24BIT_SCALING;

  818. 	while (nsamples--) {

  819. 		uint32_t val=(*((uint32_t*)src));

  820. 		if (val & 0x800000u) val|=0xFF000000u;

  821. 		*dst = (*((int32_t *) &val)) * scaling;

  822. 		dst++;

  823. 		src += src_skip;

  824. 	}

  825. }	

  826. 

  827. void sample_move_d24_sSs (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  828. {

  829. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  830. 	unsigned long unrolled = nsamples / 4;

  831. 	while (unrolled--) {

  832. 		int i;

  833. 		int32_t z[4];

  834. 		float32x4_t samples = vld1q_f32(src);

  835. 		int32x4_t converted = float_24_neon(samples);

  836. 		converted = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(converted)));

  837. 		vst1q_s32(z, converted);

  838. 

  839. 		for (i = 0; i != 4; ++i) {

  840. 			memcpy (dst, ((char*)(z+i))+1, 3);

  841. 			dst += dst_skip;

  842. 		}

  843. 		src += 4;

  844. 	}

  845. 	nsamples = nsamples & 3;

  846. #endif

  847. 

  848. 	int32_t z;

  849. 

  850. 	while (nsamples--) {

  851. 		float_24 (*src, z);

  852. #if __BYTE_ORDER == __LITTLE_ENDIAN

  853. 		dst[0]=(char)(z>>16);

  854. 		dst[1]=(char)(z>>8);

  855. 		dst[2]=(char)(z);

  856. #elif __BYTE_ORDER == __BIG_ENDIAN

  857. 		dst[0]=(char)(z);

  858. 		dst[1]=(char)(z>>8);

  859. 		dst[2]=(char)(z>>16);

  860. #endif

  861. 		dst += dst_skip;

  862. 		src++;

  863. 	}

  864. }	

  865. 

  866. void sample_move_d24_sS (char *dst, jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)

  867. {

  868. #if defined (__SSE2__) && !defined (__sun__)

  869. 	_MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST);

  870. 	while (nsamples >= 4) {

  871. 		int i;

  872. 		int32_t z[4];

  873. 		__m128 samples = _mm_loadu_ps(src);

  874. 		__m128i converted = float_24_sse(samples);

  875. 

  876. #ifdef __SSE4_1__

  877. 		z[0] = _mm_extract_epi32(converted, 0);

  878. 		z[1] = _mm_extract_epi32(converted, 1);

  879. 		z[2] = _mm_extract_epi32(converted, 2);

  880. 		z[3] = _mm_extract_epi32(converted, 3);

  881. #else

  882. 		__m128i shuffled1 = _mm_shuffle_epi32(converted, _MM_SHUFFLE(0, 3, 2, 1));

  883. 		__m128i shuffled2 = _mm_shuffle_epi32(converted, _MM_SHUFFLE(1, 0, 3, 2));

  884. 		__m128i shuffled3 = _mm_shuffle_epi32(converted, _MM_SHUFFLE(2, 1, 0, 3));

  885. 

  886. 		_mm_store_ss((float*)z, (__m128)converted);

  887. 		_mm_store_ss((float*)z+1, (__m128)shuffled1);

  888. 		_mm_store_ss((float*)z+2, (__m128)shuffled2);

  889. 		_mm_store_ss((float*)z+3, (__m128)shuffled3);

  890. #endif

  891. 

  892. 		for (i = 0; i != 4; ++i) {

  893. 			memcpy (dst, z+i, 3);

  894. 			dst += dst_skip;

  895. 		}

  896. 

  897. 		nsamples -= 4;

  898. 		src += 4;

  899. 	}

  900. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  901. 	unsigned long unrolled = nsamples / 4;

  902. 	while (unrolled--) {

  903. 		int i;

  904. 		int32_t z[4];

  905. 		float32x4_t samples = vld1q_f32(src);

  906. 		int32x4_t converted = float_24_neon(samples);

  907. 		vst1q_s32(z, converted);

  908. 

  909. 		for (i = 0; i != 4; ++i) {

  910. 			memcpy (dst, z+i, 3);

  911. 			dst += dst_skip;

  912. 		}

  913. 		src += 4;

  914. 	}

  915. 	nsamples = nsamples & 3;

  916. #endif

  917. 

  918.     int32_t z;

  919. 

  920. 	while (nsamples--) {

  921. 		float_24 (*src, z);

  922. #if __BYTE_ORDER == __LITTLE_ENDIAN

  923. 		memcpy (dst, &z, 3);

  924. #elif __BYTE_ORDER == __BIG_ENDIAN

  925. 		memcpy (dst, (char *)&z + 1, 3);

  926. #endif

  927. 		dst += dst_skip;

  928. 		src++;

  929. 	}

  930. }

  931. 

  932. void sample_move_dS_s24s (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  933. {

  934. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_24BIT_SCALING;

  935. 

  936. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  937. 	// we shift 8 to the right by dividing by 256.0 -> no sign extra handling

  938. 	const float32x4_t vscaling = vdupq_n_f32(scaling/256.0);

  939. 	int32_t x[4];

  940. 	memset(x, 0, sizeof(x));

  941. 	unsigned long unrolled = nsamples / 4;

  942. 	while (unrolled--) {

  943. #if __BYTE_ORDER == __BIG_ENDIAN	 /* ARM big endian?? */

  944. 		// right aligned / inverse sequence below -> *256

  945. 		memcpy(((char*)&x[0])+1, src, 3);

  946. 		memcpy(((char*)&x[1])+1, src+src_skip, 3);

  947. 		memcpy(((char*)&x[2])+1, src+2*src_skip, 3);

  948. 		memcpy(((char*)&x[3])+1, src+3*src_skip, 3);

  949. #else

  950. 		memcpy(&x[0], src, 3);

  951. 		memcpy(&x[1], src+src_skip, 3);

  952. 		memcpy(&x[2], src+2*src_skip, 3);

  953. 		memcpy(&x[3], src+3*src_skip, 3);

  954. #endif

  955. 		src += 4 * src_skip;

  956. 

  957. 		int32x4_t source = vld1q_s32(x);

  958. 		source = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(source)));

  959. 		float32x4_t converted = vcvtq_f32_s32(source);

  960. 		float32x4_t scaled = vmulq_f32(converted, vscaling);

  961. 		vst1q_f32(dst, scaled);

  962. 		dst += 4;

  963. 	}

  964. 	nsamples = nsamples & 3;

  965. #endif

  966. 

  967. 	/* ALERT: signed sign-extension portability !!! */

  968. 

  969. 	while (nsamples--) {

  970. 		int x;

  971. #if __BYTE_ORDER == __LITTLE_ENDIAN

  972. 		x = (unsigned char)(src[0]);

  973. 		x <<= 8;

  974. 		x |= (unsigned char)(src[1]);

  975. 		x <<= 8;

  976. 		x |= (unsigned char)(src[2]);

  977. 		/* correct sign bit and the rest of the top byte */

  978. 		if (src[0] & 0x80) {

  979. 			x |= 0xff << 24;

  980. 		}

  981. #elif __BYTE_ORDER == __BIG_ENDIAN

  982. 		x = (unsigned char)(src[2]);

  983. 		x <<= 8;

  984. 		x |= (unsigned char)(src[1]);

  985. 		x <<= 8;

  986. 		x |= (unsigned char)(src[0]);

  987. 		/* correct sign bit and the rest of the top byte */

  988. 		if (src[2] & 0x80) {

  989. 			x |= 0xff << 24;

  990. 		}

  991. #endif

  992. 		*dst = x * scaling;

  993. 		dst++;

  994. 		src += src_skip;

  995. 	}

  996. }

  997. 

  998. void sample_move_dS_s24 (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip)

  999. {

  1000. 	const jack_default_audio_sample_t scaling = 1.f/SAMPLE_24BIT_SCALING;

  1001. 

  1002. #if defined (__SSE2__) && !defined (__sun__)

  1003. 	const __m128 scaling_block = _mm_set_ps1(scaling);

  1004. 	while (nsamples >= 4) {

  1005. 		int x0, x1, x2, x3;

  1006. 

  1007. 		memcpy((char*)&x0 + 1, src, 3);

  1008. 		memcpy((char*)&x1 + 1, src+src_skip, 3);

  1009. 		memcpy((char*)&x2 + 1, src+2*src_skip, 3);

  1010. 		memcpy((char*)&x3 + 1, src+3*src_skip, 3);

  1011. 		src += 4 * src_skip;

  1012. 

  1013. 		const __m128i block_i = _mm_set_epi32(x3, x2, x1, x0);

  1014. 		const __m128i shifted = _mm_srai_epi32(block_i, 8);

  1015. 		const __m128 converted = _mm_cvtepi32_ps (shifted);

  1016. 		const __m128 scaled = _mm_mul_ps(converted, scaling_block);

  1017. 		_mm_storeu_ps(dst, scaled);

  1018. 		dst += 4;

  1019. 		nsamples -= 4;

  1020. 	}

  1021. #elif defined (__ARM_NEON__) || defined (__ARM_NEON)

  1022. 	// we shift 8 to the right by dividing by 256.0 -> no sign extra handling

  1023. 	const float32x4_t vscaling = vdupq_n_f32(scaling/256.0);

  1024. 	int32_t x[4];

  1025. 	memset(x, 0, sizeof(x));

  1026. 	unsigned long unrolled = nsamples / 4;

  1027. 	while (unrolled--) {

  1028. #if __BYTE_ORDER == __BIG_ENDIAN	/* ARM big endian?? */

  1029. 		// left aligned -> *256

  1030. 		memcpy(&x[0], src, 3);

  1031. 		memcpy(&x[1], src+src_skip, 3);

  1032. 		memcpy(&x[2], src+2*src_skip, 3);

  1033. 		memcpy(&x[3], src+3*src_skip, 3);

  1034. #else

  1035. 		memcpy(((char*)&x[0])+1, src, 3);

  1036. 		memcpy(((char*)&x[1])+1, src+src_skip, 3);

  1037. 		memcpy(((char*)&x[2])+1, src+2*src_skip, 3);

  1038. 		memcpy(((char*)&x[3])+1, src+3*src_skip, 3);

  1039. #endif

  1040. 		src += 4 * src_skip;

  1041. 

  1042. 		int32x4_t source = vld1q_s32(x);

  1043. 		float32x4_t converted = vcvtq_f32_s32(source);

  1044. 		float32x4_t scaled = vmulq_f32(converted, vscaling);

  1045. 		vst1q_f32(dst, scaled);

  1046. 		dst += 4;

  1047. 	}

  1048. 	nsamples = nsamples & 3;

  1049. #endif

  1050. 

  1051. 	while (nsamples--) {

  1052. 		int x;

  1053. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1054. 		memcpy((char*)&x + 1, src, 3);

  1055. #elif __BYTE_ORDER == __BIG_ENDIAN

  1056. 		memcpy(&x, src, 3);

  1057. #endif

  1058. 		x >>= 8;

  1059. 		*dst = x * scaling;

  1060. 		dst++;

  1061. 		src += src_skip;

  1062. 	}

  1063. }

  1064. 

  1065. 

  1066. void sample_move_d16_sSs (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1067. {

  1068. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  1069. 	unsigned long unrolled = nsamples / 4;

  1070. 	nsamples = nsamples & 3;

  1071. 

  1072. 	while (unrolled--) {

  1073. 		float32x4_t samples = vld1q_f32(src);

  1074. 		int16x4_t converted = float_16_neon(samples);

  1075. 		converted = vreinterpret_s16_u8(vrev16_u8(vreinterpret_u8_s16(converted)));

  1076. 

  1077. 		switch(dst_skip) {

  1078. 			case 2:

  1079. 				vst1_s16((int16_t*)dst, converted);

  1080. 				break;

  1081. 			default:

  1082. 				vst1_lane_s16((int16_t*)(dst),            converted, 0);

  1083. 				vst1_lane_s16((int16_t*)(dst+dst_skip),   converted, 1);

  1084. 				vst1_lane_s16((int16_t*)(dst+2*dst_skip), converted, 2);

  1085. 				vst1_lane_s16((int16_t*)(dst+3*dst_skip), converted, 3);

  1086. 				break;

  1087. 		}

  1088. 		dst += 4*dst_skip;

  1089. 		src+= 4;

  1090. 	}

  1091. #endif

  1092. 	int16_t tmp;

  1093. 

  1094. 	while (nsamples--) {

  1095. 		// float_16 (*src, tmp);

  1096. 

  1097. 		if (*src <= NORMALIZED_FLOAT_MIN) {

  1098. 			tmp = SAMPLE_16BIT_MIN;

  1099. 		} else if (*src >= NORMALIZED_FLOAT_MAX) {

  1100. 			tmp = SAMPLE_16BIT_MAX;

  1101. 		} else {

  1102. 			tmp = (int16_t) f_round (*src * SAMPLE_16BIT_SCALING);

  1103. 		}

  1104. 

  1105. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1106. 		dst[0]=(char)(tmp>>8);

  1107. 		dst[1]=(char)(tmp);

  1108. #elif __BYTE_ORDER == __BIG_ENDIAN

  1109. 		dst[0]=(char)(tmp);

  1110. 		dst[1]=(char)(tmp>>8);

  1111. #endif

  1112. 		dst += dst_skip;

  1113. 		src++;

  1114. 	}

  1115. }

  1116. 

  1117. void sample_move_d16_sS (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1118. {

  1119. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  1120. 	unsigned long unrolled = nsamples / 4;

  1121. 	nsamples = nsamples & 3;

  1122. 

  1123. 	while (unrolled--) {

  1124. 		float32x4_t samples = vld1q_f32(src);

  1125. 		int16x4_t converted = float_16_neon(samples);

  1126. 

  1127. 		switch(dst_skip) {

  1128. 			case 2:

  1129. 				vst1_s16((int16_t*)dst, converted);

  1130. 				break;

  1131. 			default:

  1132. 				vst1_lane_s16((int16_t*)(dst),            converted, 0);

  1133. 				vst1_lane_s16((int16_t*)(dst+dst_skip),   converted, 1);

  1134. 				vst1_lane_s16((int16_t*)(dst+2*dst_skip), converted, 2);

  1135. 				vst1_lane_s16((int16_t*)(dst+3*dst_skip), converted, 3);

  1136. 				break;

  1137. 		}

  1138. 		dst += 4*dst_skip;

  1139. 		src+= 4;

  1140. 	}

  1141. #endif

  1142. 	while (nsamples--) {

  1143. 		float_16 (*src, *((int16_t*) dst));

  1144. 		dst += dst_skip;

  1145. 		src++;

  1146. 	}

  1147. }

  1148. 

  1149. void sample_move_dither_rect_d16_sSs (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1150. {

  1151. 	jack_default_audio_sample_t val;

  1152. 	int16_t      tmp;

  1153. 

  1154. 	while (nsamples--) {

  1155. 		val = (*src * SAMPLE_16BIT_SCALING) + fast_rand() / (float) UINT_MAX - 0.5f;

  1156. 		float_16_scaled (val, tmp);

  1157. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1158. 		dst[0]=(char)(tmp>>8);

  1159. 		dst[1]=(char)(tmp);

  1160. #elif __BYTE_ORDER == __BIG_ENDIAN

  1161. 		dst[0]=(char)(tmp);

  1162. 		dst[1]=(char)(tmp>>8);

  1163. #endif

  1164. 		dst += dst_skip;

  1165. 		src++;

  1166. 	}

  1167. }

  1168. 

  1169. void sample_move_dither_rect_d16_sS (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1170. {

  1171. 	jack_default_audio_sample_t val;

  1172. 

  1173. 	while (nsamples--) {

  1174. 		val = (*src * SAMPLE_16BIT_SCALING) + fast_rand() / (float)UINT_MAX - 0.5f;

  1175. 		float_16_scaled (val, *((int16_t*) dst));

  1176. 		dst += dst_skip;

  1177. 		src++;

  1178. 	}

  1179. }

  1180. 

  1181. void sample_move_dither_tri_d16_sSs (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1182. {

  1183. 	jack_default_audio_sample_t val;

  1184. 	int16_t      tmp;

  1185. 

  1186. 	while (nsamples--) {

  1187. 		val = (*src * SAMPLE_16BIT_SCALING) + ((float)fast_rand() + (float)fast_rand()) / (float)UINT_MAX - 1.0f;

  1188. 		float_16_scaled (val, tmp);

  1189. 

  1190. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1191. 		dst[0]=(char)(tmp>>8);

  1192. 		dst[1]=(char)(tmp);

  1193. #elif __BYTE_ORDER == __BIG_ENDIAN

  1194. 		dst[0]=(char)(tmp);

  1195. 		dst[1]=(char)(tmp>>8);

  1196. #endif

  1197. 		dst += dst_skip;

  1198. 		src++;

  1199. 	}

  1200. }

  1201. 

  1202. void sample_move_dither_tri_d16_sS (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1203. {

  1204. 	jack_default_audio_sample_t val;

  1205. 

  1206. 	while (nsamples--) {

  1207. 		val = (*src * SAMPLE_16BIT_SCALING) + ((float)fast_rand() + (float)fast_rand()) / (float)UINT_MAX - 1.0f;

  1208. 		float_16_scaled (val, *((int16_t*) dst));

  1209. 		dst += dst_skip;

  1210. 		src++;

  1211. 	}

  1212. }

  1213. 

  1214. void sample_move_dither_shaped_d16_sSs (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1215. {

  1216. 	jack_default_audio_sample_t     x;

  1217. 	jack_default_audio_sample_t     xe; /* the innput sample - filtered error */

  1218. 	jack_default_audio_sample_t     xp; /* x' */

  1219. 	float        r;

  1220. 	float        rm1 = state->rm1;

  1221. 	unsigned int idx = state->idx;

  1222. 	int16_t      tmp;

  1223. 

  1224. 	while (nsamples--) {

  1225. 		x = *src * SAMPLE_16BIT_SCALING;

  1226. 		r = ((float)fast_rand() + (float)fast_rand())  / (float)UINT_MAX - 1.0f;

  1227. 		/* Filter the error with Lipshitz's minimally audible FIR:

  1228. 		   [2.033 -2.165 1.959 -1.590 0.6149] */

  1229. 		xe = x

  1230. 		     - state->e[idx] * 2.033f

  1231. 		     + state->e[(idx - 1) & DITHER_BUF_MASK] * 2.165f

  1232. 		     - state->e[(idx - 2) & DITHER_BUF_MASK] * 1.959f

  1233. 		     + state->e[(idx - 3) & DITHER_BUF_MASK] * 1.590f

  1234. 		     - state->e[(idx - 4) & DITHER_BUF_MASK] * 0.6149f;

  1235. 		xp = xe + r - rm1;

  1236. 		rm1 = r;

  1237. 

  1238. 		float_16_scaled (xp, tmp);

  1239. 

  1240. 		/* Intrinsic z^-1 delay */

  1241. 		idx = (idx + 1) & DITHER_BUF_MASK;

  1242. 		state->e[idx] = xp - xe;

  1243. 

  1244. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1245. 		dst[0]=(char)(tmp>>8);

  1246. 		dst[1]=(char)(tmp);

  1247. #elif __BYTE_ORDER == __BIG_ENDIAN

  1248. 		dst[0]=(char)(tmp);

  1249. 		dst[1]=(char)(tmp>>8);

  1250. #endif

  1251. 		dst += dst_skip;

  1252. 		src++;

  1253. 	}

  1254. 	state->rm1 = rm1;

  1255. 	state->idx = idx;

  1256. }

  1257. 

  1258. void sample_move_dither_shaped_d16_sS (char *dst,  jack_default_audio_sample_t *src, unsigned long nsamples, unsigned long dst_skip, dither_state_t *state)	

  1259. {

  1260. 	jack_default_audio_sample_t     x;

  1261. 	jack_default_audio_sample_t     xe; /* the innput sample - filtered error */

  1262. 	jack_default_audio_sample_t     xp; /* x' */

  1263. 	float        r;

  1264. 	float        rm1 = state->rm1;

  1265. 	unsigned int idx = state->idx;

  1266. 

  1267. 	while (nsamples--) {

  1268. 		x = *src * SAMPLE_16BIT_SCALING;

  1269. 		r = ((float)fast_rand() + (float)fast_rand()) / (float)UINT_MAX - 1.0f;

  1270. 		/* Filter the error with Lipshitz's minimally audible FIR:

  1271. 		   [2.033 -2.165 1.959 -1.590 0.6149] */

  1272. 		xe = x

  1273. 		     - state->e[idx] * 2.033f

  1274. 		     + state->e[(idx - 1) & DITHER_BUF_MASK] * 2.165f

  1275. 		     - state->e[(idx - 2) & DITHER_BUF_MASK] * 1.959f

  1276. 		     + state->e[(idx - 3) & DITHER_BUF_MASK] * 1.590f

  1277. 		     - state->e[(idx - 4) & DITHER_BUF_MASK] * 0.6149f;

  1278. 		xp = xe + r - rm1;

  1279. 		rm1 = r;

  1280. 

  1281. 		float_16_scaled (xp, *((int16_t*) dst));

  1282. 

  1283. 		/* Intrinsic z^-1 delay */

  1284. 		idx = (idx + 1) & DITHER_BUF_MASK;

  1285. 		state->e[idx] = *((int16_t*) dst) - xe;

  1286. 

  1287. 		dst += dst_skip;

  1288. 		src++;

  1289. 	}

  1290. 	state->rm1 = rm1;

  1291. 	state->idx = idx;

  1292. }

  1293. 

  1294. void sample_move_dS_s16s (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip) 	

  1295. {

  1296. 	short z;

  1297. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_16BIT_SCALING;

  1298. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  1299. 	const float32x4_t vscaling = vdupq_n_f32(scaling);

  1300. 	unsigned long unrolled = nsamples / 4;

  1301. 	while (unrolled--) {

  1302. 		int16x4_t source16x4;

  1303. 		switch(src_skip) {

  1304. 			case 2:

  1305. 				source16x4 = vld1_s16((int16_t*)src);

  1306. 				break;

  1307. 			case 4:

  1308. 				source16x4 = vld2_s16((int16_t*)src).val[0];

  1309. 				break;

  1310. 			default:

  1311. 				source16x4 = vld1_lane_s16((int16_t*)src,              source16x4, 0);

  1312. 				source16x4 = vld1_lane_s16((int16_t*)(src+src_skip),   source16x4, 1);

  1313. 				source16x4 = vld1_lane_s16((int16_t*)(src+2*src_skip), source16x4, 2);

  1314. 				source16x4 = vld1_lane_s16((int16_t*)(src+3*src_skip), source16x4, 3);

  1315. 				break;

  1316. 		}

  1317. 		source16x4 = vreinterpret_s16_u8(vrev16_u8(vreinterpret_u8_s16(source16x4)));

  1318. 		int32x4_t source32x4 = vmovl_s16(source16x4);

  1319. 		src += 4 * src_skip;

  1320. 

  1321. 		float32x4_t converted = vcvtq_f32_s32(source32x4);

  1322. 		float32x4_t scaled = vmulq_f32(converted, vscaling);

  1323. 		vst1q_f32(dst, scaled);

  1324. 		dst += 4;

  1325. 	}

  1326. 	nsamples = nsamples & 3;

  1327. #endif

  1328. 

  1329. 	/* ALERT: signed sign-extension portability !!! */

  1330. 	while (nsamples--) {

  1331. #if __BYTE_ORDER == __LITTLE_ENDIAN

  1332. 		z = (unsigned char)(src[0]);

  1333. 		z <<= 8;

  1334. 		z |= (unsigned char)(src[1]);

  1335. #elif __BYTE_ORDER == __BIG_ENDIAN

  1336. 		z = (unsigned char)(src[1]);

  1337. 		z <<= 8;

  1338. 		z |= (unsigned char)(src[0]);

  1339. #endif

  1340. 		*dst = z * scaling;

  1341. 		dst++;

  1342. 		src += src_skip;

  1343. 	}

  1344. }	

  1345. 

  1346. void sample_move_dS_s16 (jack_default_audio_sample_t *dst, char *src, unsigned long nsamples, unsigned long src_skip) 

  1347. {

  1348. 	/* ALERT: signed sign-extension portability !!! */

  1349. 	const jack_default_audio_sample_t scaling = 1.0/SAMPLE_16BIT_SCALING;

  1350. #if defined (__ARM_NEON__) || defined (__ARM_NEON)

  1351. 	const float32x4_t vscaling = vdupq_n_f32(scaling);

  1352. 	unsigned long unrolled = nsamples / 4;

  1353. 	while (unrolled--) {

  1354. 		int16x4_t source16x4;

  1355. 		switch(src_skip) {

  1356. 			case 2:

  1357. 				source16x4 = vld1_s16((int16_t*)src);

  1358. 				break;

  1359. 			case 4:

  1360. 				source16x4 = vld2_s16((int16_t*)src).val[0];

  1361. 				break;

  1362. 			default:

  1363. 				source16x4 = vld1_lane_s16((int16_t*)src,              source16x4, 0);

  1364. 				source16x4 = vld1_lane_s16((int16_t*)(src+src_skip),   source16x4, 1);

  1365. 				source16x4 = vld1_lane_s16((int16_t*)(src+2*src_skip), source16x4, 2);

  1366. 				source16x4 = vld1_lane_s16((int16_t*)(src+3*src_skip), source16x4, 3);

  1367. 				break;

  1368. 		}

  1369. 		int32x4_t source32x4 = vmovl_s16(source16x4);

  1370. 		src += 4 * src_skip;

  1371. 

  1372. 		float32x4_t converted = vcvtq_f32_s32(source32x4);

  1373. 		float32x4_t scaled = vmulq_f32(converted, vscaling);

  1374. 		vst1q_f32(dst, scaled);

  1375. 		dst += 4;

  1376. 	}

  1377. 	nsamples = nsamples & 3;

  1378. #endif

  1379. 

  1380. 	while (nsamples--) {

  1381. 		*dst = (*((short *) src)) * scaling;

  1382. 		dst++;

  1383. 		src += src_skip;

  1384. 	}

  1385. }	

  1386. 

  1387. void memset_interleave (char *dst, char val, unsigned long bytes, 

  1388. 			unsigned long unit_bytes, 

  1389. 			unsigned long skip_bytes) 

  1390. {

  1391. 	switch (unit_bytes) {

  1392. 	case 1:

  1393. 		while (bytes--) {

  1394. 			*dst = val;

  1395. 			dst += skip_bytes;

  1396. 		}

  1397. 		break;

  1398. 	case 2:

  1399. 		while (bytes) {

  1400. 			*((short *) dst) = (short) val;

  1401. 			dst += skip_bytes;

  1402. 			bytes -= 2;

  1403. 		}

  1404. 		break;

  1405. 	case 4:		    

  1406. 		while (bytes) {

  1407. 			*((int *) dst) = (int) val;

  1408. 			dst += skip_bytes;

  1409. 			bytes -= 4;

  1410. 		}

  1411. 		break;

  1412. 	default:

  1413. 		while (bytes) {

  1414. 			memset(dst, val, unit_bytes);

  1415. 			dst += skip_bytes;

  1416. 			bytes -= unit_bytes;

  1417. 		}

  1418. 		break;

  1419. 	}

  1420. }

  1421. 

  1422. /* COPY FUNCTIONS: used to move data from an input channel to an

  1423.    output channel. Note that we assume that the skip distance

  1424.    is the same for both channels. This is completely fine

  1425.    unless the input and output were on different audio interfaces that

  1426.    were interleaved differently. We don't try to handle that.

  1427. */

  1428. 

  1429. void 

  1430. memcpy_fake (char *dst, char *src, unsigned long src_bytes, unsigned long foo, unsigned long bar)

  1431. {

  1432. 	memcpy (dst, src, src_bytes);

  1433. }

  1434. 

  1435. void 

  1436. memcpy_interleave_d16_s16 (char *dst, char *src, unsigned long src_bytes,

  1437. 			   unsigned long dst_skip_bytes, unsigned long src_skip_bytes)

  1438. {

  1439. 	while (src_bytes) {

  1440. 		*((short *) dst) = *((short *) src);

  1441. 		dst += dst_skip_bytes;

  1442. 		src += src_skip_bytes;

  1443. 		src_bytes -= 2;

  1444. 	}

  1445. }

  1446. 

  1447. void 

  1448. memcpy_interleave_d24_s24 (char *dst, char *src, unsigned long src_bytes,

  1449. 			   unsigned long dst_skip_bytes, unsigned long src_skip_bytes)

  1450. {

  1451. 	while (src_bytes) {

  1452. 		memcpy(dst, src, 3);

  1453. 		dst += dst_skip_bytes;

  1454. 		src += src_skip_bytes;

  1455. 		src_bytes -= 3;

  1456. 	}

  1457. }

  1458. 

  1459. void 

  1460. memcpy_interleave_d32_s32 (char *dst, char *src, unsigned long src_bytes,

  1461. 			   unsigned long dst_skip_bytes, unsigned long src_skip_bytes)

  1462. {

  1463. 	while (src_bytes) {

  1464. 		*((int *) dst) = *((int *) src);

  1465. 		dst += dst_skip_bytes;

  1466. 		src += src_skip_bytes;

  1467. 		src_bytes -= 4;

  1468. 	}

  1469. }