ESPResSo
Extensible Simulation Package for Research on Soft Matter Systems
Loading...
Searching...
No Matches
lattice_boltzmann/generated_kernels/philox_rand.h
Go to the documentation of this file.
1/*
2Copyright 2010-2011, D. E. Shaw Research. All rights reserved.
3Copyright 2019-2021, Michael Kuron.
4
5Redistribution and use in source and binary forms, with or without
6modification, are permitted provided that the following conditions are
7met:
8
9* Redistributions of source code must retain the above copyright
10 notice, this list of conditions, and the following disclaimer.
11
12* Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions, and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16* Neither the name of the copyright holder nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31*/
32
33/**
34 * @file
35 * Philox counter-based RNG from @cite salmon11a.
36 * Adapted from the pystencils source file
37 * https://i10git.cs.fau.de/pycodegen/pystencils/-/blob/896b4192/pystencils/include/philox_rand.h
38 */
39
40#include <cstdint>
41
42#if defined(__SSE2__) || defined(_MSC_VER)
43#include <emmintrin.h> // SSE2
44#endif
45#ifdef __AVX2__
46#include <immintrin.h> // AVX*
47#elif defined(__SSE4_1__) || defined(_MSC_VER)
48#include <smmintrin.h> // SSE4
49#ifdef __FMA__
50#include <immintrin.h> // FMA
51#endif
52#endif
53
54#ifdef __ARM_NEON
55#include <arm_neon.h>
56#endif
57#ifdef __ARM_FEATURE_SVE
58#include <arm_sve.h>
59#endif
60
61#if defined(__powerpc__) && defined(__GNUC__) && !defined(__clang__) && \
62 !defined(__xlC__)
63#include <ppu_intrinsics.h>
64#endif
65#ifdef __ALTIVEC__
66#include <altivec.h>
67#undef bool
68#ifndef _ARCH_PWR8
69#include <pveclib/vec_int64_ppc.h>
70#endif
71#endif
72
73#if defined(__CUDA_ARCH__) || defined(__clang__) && defined(__CUDA__)
74#if defined(__clang__) && defined(QUALIFIERS)
75#undef QUALIFIERS
76#endif
77#define QUALIFIERS static __forceinline__ __device__
78#else
79#if defined(__clang__) && defined(QUALIFIERS)
80#undef QUALIFIERS
81#endif
82#define QUALIFIERS inline
83#include "myintrin.h"
84#endif
85
86#define PHILOX_W32_0 (0x9E3779B9)
87#define PHILOX_W32_1 (0xBB67AE85)
88#define PHILOX_M4x32_0 (0xD2511F53)
89#define PHILOX_M4x32_1 (0xCD9E8D57)
90#define TWOPOW53_INV_DOUBLE (1.1102230246251565e-16)
91#define TWOPOW32_INV_FLOAT (2.3283064e-10f)
92
93typedef std::uint32_t uint32;
94typedef std::uint64_t uint64;
95
96#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_SVE_BITS) && \
97 __ARM_FEATURE_SVE_BITS > 0
98typedef svfloat32_t svfloat32_st
99 __attribute__((arm_sve_vector_bits(__ARM_FEATURE_SVE_BITS)));
100typedef svfloat64_t svfloat64_st
101 __attribute__((arm_sve_vector_bits(__ARM_FEATURE_SVE_BITS)));
102#elif defined(__ARM_FEATURE_SVE)
103typedef svfloat32_t svfloat32_st;
104typedef svfloat64_t svfloat64_st;
105#endif
106
107QUALIFIERS uint32 mulhilo32(uint32 a, uint32 b, uint32 *hip) {
108#ifndef __CUDA_ARCH__
109 // host code
110#if defined(__powerpc__) && (!defined(__clang__) || defined(__xlC__))
111 *hip = __mulhwu(a, b);
112 return a * b;
113#else
114 uint64 product = ((uint64)a) * ((uint64)b);
115 *hip = product >> 32;
116 return (uint32)product;
117#endif
118#else
119 // device code
120 *hip = __umulhi(a, b);
121 return a * b;
122#endif
123}
124
125QUALIFIERS void _philox4x32round(uint32 *ctr, uint32 *key) {
126 uint32 hi0;
127 uint32 hi1;
128 uint32 lo0 = mulhilo32(PHILOX_M4x32_0, ctr[0], &hi0);
129 uint32 lo1 = mulhilo32(PHILOX_M4x32_1, ctr[2], &hi1);
130
131 ctr[0] = hi1 ^ ctr[1] ^ key[0];
132 ctr[1] = lo1;
133 ctr[2] = hi0 ^ ctr[3] ^ key[1];
134 ctr[3] = lo0;
135}
136
141
142QUALIFIERS double _uniform_double_hq(uint32 x, uint32 y) {
143 double z = (double)((uint64)x ^ ((uint64)y << (53 - 32)));
144 return z * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE / 2.0);
145}
146
175
207
208#ifndef __CUDA_ARCH__
209#if defined(__SSE4_1__) || defined(_MSC_VER)
210QUALIFIERS void _philox4x32round(__m128i *ctr, __m128i *key) {
211 __m128i lohi0a = _mm_mul_epu32(ctr[0], _mm_set1_epi32(PHILOX_M4x32_0));
212 __m128i lohi0b =
213 _mm_mul_epu32(_mm_srli_epi64(ctr[0], 32), _mm_set1_epi32(PHILOX_M4x32_0));
214 __m128i lohi1a = _mm_mul_epu32(ctr[2], _mm_set1_epi32(PHILOX_M4x32_1));
215 __m128i lohi1b =
216 _mm_mul_epu32(_mm_srli_epi64(ctr[2], 32), _mm_set1_epi32(PHILOX_M4x32_1));
217
218 lohi0a = _mm_shuffle_epi32(lohi0a, 0xD8);
219 lohi0b = _mm_shuffle_epi32(lohi0b, 0xD8);
220 lohi1a = _mm_shuffle_epi32(lohi1a, 0xD8);
221 lohi1b = _mm_shuffle_epi32(lohi1b, 0xD8);
222
223 __m128i lo0 = _mm_unpacklo_epi32(lohi0a, lohi0b);
224 __m128i hi0 = _mm_unpackhi_epi32(lohi0a, lohi0b);
225 __m128i lo1 = _mm_unpacklo_epi32(lohi1a, lohi1b);
226 __m128i hi1 = _mm_unpackhi_epi32(lohi1a, lohi1b);
227
228 ctr[0] = _mm_xor_si128(_mm_xor_si128(hi1, ctr[1]), key[0]);
229 ctr[1] = lo1;
230 ctr[2] = _mm_xor_si128(_mm_xor_si128(hi0, ctr[3]), key[1]);
231 ctr[3] = lo0;
232}
233
234QUALIFIERS void _philox4x32bumpkey(__m128i *key) {
235 key[0] = _mm_add_epi32(key[0], _mm_set1_epi32(PHILOX_W32_0));
236 key[1] = _mm_add_epi32(key[1], _mm_set1_epi32(PHILOX_W32_1));
237}
238
239template <bool high>
240QUALIFIERS __m128d _uniform_double_hq(__m128i x, __m128i y) {
241 // convert 32 to 64 bit
242 if (high) {
243 x = _mm_unpackhi_epi32(x, _mm_setzero_si128());
244 y = _mm_unpackhi_epi32(y, _mm_setzero_si128());
245 } else {
246 x = _mm_unpacklo_epi32(x, _mm_setzero_si128());
247 y = _mm_unpacklo_epi32(y, _mm_setzero_si128());
248 }
249
250 // calculate z = x ^ y << (53 - 32))
251 __m128i z = _mm_sll_epi64(y, _mm_set1_epi64x(53 - 32));
252 z = _mm_xor_si128(x, z);
253
254 // convert uint64 to double
255 __m128d rs = _my_cvtepu64_pd(z);
256 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
257#ifdef __FMA__
258 rs = _mm_fmadd_pd(rs, _mm_set1_pd(TWOPOW53_INV_DOUBLE),
259 _mm_set1_pd(TWOPOW53_INV_DOUBLE / 2.0));
260#else
261 rs = _mm_mul_pd(rs, _mm_set1_pd(TWOPOW53_INV_DOUBLE));
262 rs = _mm_add_pd(rs, _mm_set1_pd(TWOPOW53_INV_DOUBLE / 2.0));
263#endif
264
265 return rs;
266}
267
268QUALIFIERS void philox_float4(__m128i ctr0, __m128i ctr1, __m128i ctr2,
269 __m128i ctr3, uint32 key0, uint32 key1,
270 __m128 &rnd1, __m128 &rnd2, __m128 &rnd3,
271 __m128 &rnd4) {
272 __m128i key[2] = {_mm_set1_epi32(key0), _mm_set1_epi32(key1)};
273 __m128i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
274 _philox4x32round(ctr, key); // 1
275 _philox4x32bumpkey(key);
276 _philox4x32round(ctr, key); // 2
277 _philox4x32bumpkey(key);
278 _philox4x32round(ctr, key); // 3
279 _philox4x32bumpkey(key);
280 _philox4x32round(ctr, key); // 4
281 _philox4x32bumpkey(key);
282 _philox4x32round(ctr, key); // 5
283 _philox4x32bumpkey(key);
284 _philox4x32round(ctr, key); // 6
285 _philox4x32bumpkey(key);
286 _philox4x32round(ctr, key); // 7
287 _philox4x32bumpkey(key);
288 _philox4x32round(ctr, key); // 8
289 _philox4x32bumpkey(key);
290 _philox4x32round(ctr, key); // 9
291 _philox4x32bumpkey(key);
292 _philox4x32round(ctr, key); // 10
293
294 // convert uint32 to float
295 rnd1 = _my_cvtepu32_ps(ctr[0]);
296 rnd2 = _my_cvtepu32_ps(ctr[1]);
297 rnd3 = _my_cvtepu32_ps(ctr[2]);
298 rnd4 = _my_cvtepu32_ps(ctr[3]);
299 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
300#ifdef __FMA__
301 rnd1 = _mm_fmadd_ps(rnd1, _mm_set1_ps(TWOPOW32_INV_FLOAT),
302 _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
303 rnd2 = _mm_fmadd_ps(rnd2, _mm_set1_ps(TWOPOW32_INV_FLOAT),
304 _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
305 rnd3 = _mm_fmadd_ps(rnd3, _mm_set1_ps(TWOPOW32_INV_FLOAT),
306 _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
307 rnd4 = _mm_fmadd_ps(rnd4, _mm_set1_ps(TWOPOW32_INV_FLOAT),
308 _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
309#else
310 rnd1 = _mm_mul_ps(rnd1, _mm_set1_ps(TWOPOW32_INV_FLOAT));
311 rnd1 = _mm_add_ps(rnd1, _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
312 rnd2 = _mm_mul_ps(rnd2, _mm_set1_ps(TWOPOW32_INV_FLOAT));
313 rnd2 = _mm_add_ps(rnd2, _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
314 rnd3 = _mm_mul_ps(rnd3, _mm_set1_ps(TWOPOW32_INV_FLOAT));
315 rnd3 = _mm_add_ps(rnd3, _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
316 rnd4 = _mm_mul_ps(rnd4, _mm_set1_ps(TWOPOW32_INV_FLOAT));
317 rnd4 = _mm_add_ps(rnd4, _mm_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
318#endif
319}
320
321QUALIFIERS void philox_double2(__m128i ctr0, __m128i ctr1, __m128i ctr2,
322 __m128i ctr3, uint32 key0, uint32 key1,
323 __m128d &rnd1lo, __m128d &rnd1hi,
324 __m128d &rnd2lo, __m128d &rnd2hi) {
325 __m128i key[2] = {_mm_set1_epi32(key0), _mm_set1_epi32(key1)};
326 __m128i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
327 _philox4x32round(ctr, key); // 1
328 _philox4x32bumpkey(key);
329 _philox4x32round(ctr, key); // 2
330 _philox4x32bumpkey(key);
331 _philox4x32round(ctr, key); // 3
332 _philox4x32bumpkey(key);
333 _philox4x32round(ctr, key); // 4
334 _philox4x32bumpkey(key);
335 _philox4x32round(ctr, key); // 5
336 _philox4x32bumpkey(key);
337 _philox4x32round(ctr, key); // 6
338 _philox4x32bumpkey(key);
339 _philox4x32round(ctr, key); // 7
340 _philox4x32bumpkey(key);
341 _philox4x32round(ctr, key); // 8
342 _philox4x32bumpkey(key);
343 _philox4x32round(ctr, key); // 9
344 _philox4x32bumpkey(key);
345 _philox4x32round(ctr, key); // 10
346
347 rnd1lo = _uniform_double_hq<false>(ctr[0], ctr[1]);
348 rnd1hi = _uniform_double_hq<true>(ctr[0], ctr[1]);
349 rnd2lo = _uniform_double_hq<false>(ctr[2], ctr[3]);
350 rnd2hi = _uniform_double_hq<true>(ctr[2], ctr[3]);
351}
352
353QUALIFIERS void philox_float4(uint32 ctr0, __m128i ctr1, uint32 ctr2,
354 uint32 ctr3, uint32 key0, uint32 key1,
355 __m128 &rnd1, __m128 &rnd2, __m128 &rnd3,
356 __m128 &rnd4) {
357 __m128i ctr0v = _mm_set1_epi32(ctr0);
358 __m128i ctr2v = _mm_set1_epi32(ctr2);
359 __m128i ctr3v = _mm_set1_epi32(ctr3);
360
361 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
362}
363
364QUALIFIERS void philox_double2(uint32 ctr0, __m128i ctr1, uint32 ctr2,
365 uint32 ctr3, uint32 key0, uint32 key1,
366 __m128d &rnd1lo, __m128d &rnd1hi,
367 __m128d &rnd2lo, __m128d &rnd2hi) {
368 __m128i ctr0v = _mm_set1_epi32(ctr0);
369 __m128i ctr2v = _mm_set1_epi32(ctr2);
370 __m128i ctr3v = _mm_set1_epi32(ctr3);
371
372 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
373 rnd2hi);
374}
375
376QUALIFIERS void philox_double2(uint32 ctr0, __m128i ctr1, uint32 ctr2,
377 uint32 ctr3, uint32 key0, uint32 key1,
378 __m128d &rnd1, __m128d &rnd2) {
379 __m128i ctr0v = _mm_set1_epi32(ctr0);
380 __m128i ctr2v = _mm_set1_epi32(ctr2);
381 __m128i ctr3v = _mm_set1_epi32(ctr3);
382
383 __m128d ignore;
384 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2,
385 ignore);
386}
387#endif
388
389#ifdef __ALTIVEC__
390QUALIFIERS void _philox4x32round(__vector unsigned int *ctr,
391 __vector unsigned int *key) {
392#ifndef _ARCH_PWR8
393 __vector unsigned int lo0 = vec_mul(ctr[0], vec_splats(PHILOX_M4x32_0));
394 __vector unsigned int lo1 = vec_mul(ctr[2], vec_splats(PHILOX_M4x32_1));
395 __vector unsigned int hi0 = vec_mulhuw(ctr[0], vec_splats(PHILOX_M4x32_0));
396 __vector unsigned int hi1 = vec_mulhuw(ctr[2], vec_splats(PHILOX_M4x32_1));
397#elif defined(_ARCH_PWR10)
398 __vector unsigned int lo0 = vec_mul(ctr[0], vec_splats(PHILOX_M4x32_0));
399 __vector unsigned int lo1 = vec_mul(ctr[2], vec_splats(PHILOX_M4x32_1));
400 __vector unsigned int hi0 = vec_mulh(ctr[0], vec_splats(PHILOX_M4x32_0));
401 __vector unsigned int hi1 = vec_mulh(ctr[2], vec_splats(PHILOX_M4x32_1));
402#else
403 __vector unsigned int lohi0a =
404 (__vector unsigned int)vec_mule(ctr[0], vec_splats(PHILOX_M4x32_0));
405 __vector unsigned int lohi0b =
406 (__vector unsigned int)vec_mulo(ctr[0], vec_splats(PHILOX_M4x32_0));
407 __vector unsigned int lohi1a =
408 (__vector unsigned int)vec_mule(ctr[2], vec_splats(PHILOX_M4x32_1));
409 __vector unsigned int lohi1b =
410 (__vector unsigned int)vec_mulo(ctr[2], vec_splats(PHILOX_M4x32_1));
411
412#ifdef __LITTLE_ENDIAN__
413 __vector unsigned int lo0 = vec_mergee(lohi0a, lohi0b);
414 __vector unsigned int lo1 = vec_mergee(lohi1a, lohi1b);
415 __vector unsigned int hi0 = vec_mergeo(lohi0a, lohi0b);
416 __vector unsigned int hi1 = vec_mergeo(lohi1a, lohi1b);
417#else
418 __vector unsigned int lo0 = vec_mergeo(lohi0a, lohi0b);
419 __vector unsigned int lo1 = vec_mergeo(lohi1a, lohi1b);
420 __vector unsigned int hi0 = vec_mergee(lohi0a, lohi0b);
421 __vector unsigned int hi1 = vec_mergee(lohi1a, lohi1b);
422#endif
423#endif
424
425 ctr[0] = vec_xor(vec_xor(hi1, ctr[1]), key[0]);
426 ctr[1] = lo1;
427 ctr[2] = vec_xor(vec_xor(hi0, ctr[3]), key[1]);
428 ctr[3] = lo0;
429}
430
431QUALIFIERS void _philox4x32bumpkey(__vector unsigned int *key) {
432 key[0] = vec_add(key[0], vec_splats(PHILOX_W32_0));
433 key[1] = vec_add(key[1], vec_splats(PHILOX_W32_1));
434}
435
436#ifdef __VSX__
437template <bool high>
438QUALIFIERS __vector double _uniform_double_hq(__vector unsigned int x,
439 __vector unsigned int y) {
440 // convert 32 to 64 bit
441#ifdef __LITTLE_ENDIAN__
442 if (high) {
443 x = vec_mergel(x, vec_splats(0U));
444 y = vec_mergel(y, vec_splats(0U));
445 } else {
446 x = vec_mergeh(x, vec_splats(0U));
447 y = vec_mergeh(y, vec_splats(0U));
448 }
449#else
450 if (high) {
451 x = vec_mergel(vec_splats(0U), x);
452 y = vec_mergel(vec_splats(0U), y);
453 } else {
454 x = vec_mergeh(vec_splats(0U), x);
455 y = vec_mergeh(vec_splats(0U), y);
456 }
457#endif
458
459 // calculate z = x ^ y << (53 - 32))
460#ifdef _ARCH_PWR8
461 __vector unsigned long long z =
462 vec_sl((__vector unsigned long long)y, vec_splats(53ULL - 32ULL));
463#else
464 __vector unsigned long long z =
465 vec_vsld((__vector unsigned long long)y, vec_splats(53ULL - 32ULL));
466#endif
467 z = vec_xor((__vector unsigned long long)x, z);
468
469 // convert uint64 to double
470#ifdef __xlC__
471 __vector double rs = vec_ctd(z, 0);
472#else
473 __vector double rs = vec_ctf(z, 0);
474#endif
475 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
476 rs = vec_madd(rs, vec_splats(TWOPOW53_INV_DOUBLE),
477 vec_splats(TWOPOW53_INV_DOUBLE / 2.0));
478
479 return rs;
480}
481#endif
482
483QUALIFIERS void philox_float4(__vector unsigned int ctr0,
484 __vector unsigned int ctr1,
485 __vector unsigned int ctr2,
486 __vector unsigned int ctr3, uint32 key0,
487 uint32 key1, __vector float &rnd1,
488 __vector float &rnd2, __vector float &rnd3,
489 __vector float &rnd4) {
490 __vector unsigned int key[2] = {vec_splats(key0), vec_splats(key1)};
491 __vector unsigned int ctr[4] = {ctr0, ctr1, ctr2, ctr3};
492 _philox4x32round(ctr, key); // 1
493 _philox4x32bumpkey(key);
494 _philox4x32round(ctr, key); // 2
495 _philox4x32bumpkey(key);
496 _philox4x32round(ctr, key); // 3
497 _philox4x32bumpkey(key);
498 _philox4x32round(ctr, key); // 4
499 _philox4x32bumpkey(key);
500 _philox4x32round(ctr, key); // 5
501 _philox4x32bumpkey(key);
502 _philox4x32round(ctr, key); // 6
503 _philox4x32bumpkey(key);
504 _philox4x32round(ctr, key); // 7
505 _philox4x32bumpkey(key);
506 _philox4x32round(ctr, key); // 8
507 _philox4x32bumpkey(key);
508 _philox4x32round(ctr, key); // 9
509 _philox4x32bumpkey(key);
510 _philox4x32round(ctr, key); // 10
511
512 // convert uint32 to float
513 rnd1 = vec_ctf(ctr[0], 0);
514 rnd2 = vec_ctf(ctr[1], 0);
515 rnd3 = vec_ctf(ctr[2], 0);
516 rnd4 = vec_ctf(ctr[3], 0);
517 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
518 rnd1 = vec_madd(rnd1, vec_splats(TWOPOW32_INV_FLOAT),
519 vec_splats(TWOPOW32_INV_FLOAT / 2.0f));
520 rnd2 = vec_madd(rnd2, vec_splats(TWOPOW32_INV_FLOAT),
521 vec_splats(TWOPOW32_INV_FLOAT / 2.0f));
522 rnd3 = vec_madd(rnd3, vec_splats(TWOPOW32_INV_FLOAT),
523 vec_splats(TWOPOW32_INV_FLOAT / 2.0f));
524 rnd4 = vec_madd(rnd4, vec_splats(TWOPOW32_INV_FLOAT),
525 vec_splats(TWOPOW32_INV_FLOAT / 2.0f));
526}
527
528#ifdef __VSX__
529QUALIFIERS void philox_double2(__vector unsigned int ctr0,
530 __vector unsigned int ctr1,
531 __vector unsigned int ctr2,
532 __vector unsigned int ctr3, uint32 key0,
533 uint32 key1, __vector double &rnd1lo,
534 __vector double &rnd1hi, __vector double &rnd2lo,
535 __vector double &rnd2hi) {
536 __vector unsigned int key[2] = {vec_splats(key0), vec_splats(key1)};
537 __vector unsigned int ctr[4] = {ctr0, ctr1, ctr2, ctr3};
538 _philox4x32round(ctr, key); // 1
539 _philox4x32bumpkey(key);
540 _philox4x32round(ctr, key); // 2
541 _philox4x32bumpkey(key);
542 _philox4x32round(ctr, key); // 3
543 _philox4x32bumpkey(key);
544 _philox4x32round(ctr, key); // 4
545 _philox4x32bumpkey(key);
546 _philox4x32round(ctr, key); // 5
547 _philox4x32bumpkey(key);
548 _philox4x32round(ctr, key); // 6
549 _philox4x32bumpkey(key);
550 _philox4x32round(ctr, key); // 7
551 _philox4x32bumpkey(key);
552 _philox4x32round(ctr, key); // 8
553 _philox4x32bumpkey(key);
554 _philox4x32round(ctr, key); // 9
555 _philox4x32bumpkey(key);
556 _philox4x32round(ctr, key); // 10
557
558 rnd1lo = _uniform_double_hq<false>(ctr[0], ctr[1]);
559 rnd1hi = _uniform_double_hq<true>(ctr[0], ctr[1]);
560 rnd2lo = _uniform_double_hq<false>(ctr[2], ctr[3]);
561 rnd2hi = _uniform_double_hq<true>(ctr[2], ctr[3]);
562}
563#endif
564
565QUALIFIERS void philox_float4(uint32 ctr0, __vector unsigned int ctr1,
566 uint32 ctr2, uint32 ctr3, uint32 key0,
567 uint32 key1, __vector float &rnd1,
568 __vector float &rnd2, __vector float &rnd3,
569 __vector float &rnd4) {
570 __vector unsigned int ctr0v = vec_splats(ctr0);
571 __vector unsigned int ctr2v = vec_splats(ctr2);
572 __vector unsigned int ctr3v = vec_splats(ctr3);
573
574 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
575}
576
577QUALIFIERS void philox_float4(uint32 ctr0, __vector int ctr1, uint32 ctr2,
578 uint32 ctr3, uint32 key0, uint32 key1,
579 __vector float &rnd1, __vector float &rnd2,
580 __vector float &rnd3, __vector float &rnd4) {
581 philox_float4(ctr0, (__vector unsigned int)ctr1, ctr2, ctr3, key0, key1, rnd1,
582 rnd2, rnd3, rnd4);
583}
584
585#ifdef __VSX__
586QUALIFIERS void philox_double2(uint32 ctr0, __vector unsigned int ctr1,
587 uint32 ctr2, uint32 ctr3, uint32 key0,
588 uint32 key1, __vector double &rnd1lo,
589 __vector double &rnd1hi, __vector double &rnd2lo,
590 __vector double &rnd2hi) {
591 __vector unsigned int ctr0v = vec_splats(ctr0);
592 __vector unsigned int ctr2v = vec_splats(ctr2);
593 __vector unsigned int ctr3v = vec_splats(ctr3);
594
595 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
596 rnd2hi);
597}
598
599QUALIFIERS void philox_double2(uint32 ctr0, __vector unsigned int ctr1,
600 uint32 ctr2, uint32 ctr3, uint32 key0,
601 uint32 key1, __vector double &rnd1,
602 __vector double &rnd2) {
603 __vector unsigned int ctr0v = vec_splats(ctr0);
604 __vector unsigned int ctr2v = vec_splats(ctr2);
605 __vector unsigned int ctr3v = vec_splats(ctr3);
606
607 __vector double ignore;
608 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2,
609 ignore);
610}
611
612QUALIFIERS void philox_double2(uint32 ctr0, __vector int ctr1, uint32 ctr2,
613 uint32 ctr3, uint32 key0, uint32 key1,
614 __vector double &rnd1, __vector double &rnd2) {
615 philox_double2(ctr0, (__vector unsigned int)ctr1, ctr2, ctr3, key0, key1,
616 rnd1, rnd2);
617}
618#endif
619#endif
620
621#if defined(__ARM_NEON)
622QUALIFIERS void _philox4x32round(uint32x4_t *ctr, uint32x4_t *key) {
623 uint32x4_t lohi0a = vreinterpretq_u32_u64(
624 vmull_u32(vget_low_u32(ctr[0]), vdup_n_u32(PHILOX_M4x32_0)));
625 uint32x4_t lohi0b = vreinterpretq_u32_u64(
626 vmull_high_u32(ctr[0], vdupq_n_u32(PHILOX_M4x32_0)));
627 uint32x4_t lohi1a = vreinterpretq_u32_u64(
628 vmull_u32(vget_low_u32(ctr[2]), vdup_n_u32(PHILOX_M4x32_1)));
629 uint32x4_t lohi1b = vreinterpretq_u32_u64(
630 vmull_high_u32(ctr[2], vdupq_n_u32(PHILOX_M4x32_1)));
631
632 uint32x4_t lo0 = vuzp1q_u32(lohi0a, lohi0b);
633 uint32x4_t lo1 = vuzp1q_u32(lohi1a, lohi1b);
634 uint32x4_t hi0 = vuzp2q_u32(lohi0a, lohi0b);
635 uint32x4_t hi1 = vuzp2q_u32(lohi1a, lohi1b);
636
637 ctr[0] = veorq_u32(veorq_u32(hi1, ctr[1]), key[0]);
638 ctr[1] = lo1;
639 ctr[2] = veorq_u32(veorq_u32(hi0, ctr[3]), key[1]);
640 ctr[3] = lo0;
641}
642
643QUALIFIERS void _philox4x32bumpkey(uint32x4_t *key) {
644 key[0] = vaddq_u32(key[0], vdupq_n_u32(PHILOX_W32_0));
645 key[1] = vaddq_u32(key[1], vdupq_n_u32(PHILOX_W32_1));
646}
647
648template <bool high>
649QUALIFIERS float64x2_t _uniform_double_hq(uint32x4_t x, uint32x4_t y) {
650 // convert 32 to 64 bit
651 if (high) {
652 x = vzip2q_u32(x, vdupq_n_u32(0));
653 y = vzip2q_u32(y, vdupq_n_u32(0));
654 } else {
655 x = vzip1q_u32(x, vdupq_n_u32(0));
656 y = vzip1q_u32(y, vdupq_n_u32(0));
657 }
658
659 // calculate z = x ^ y << (53 - 32))
660 uint64x2_t z = vshlq_n_u64(vreinterpretq_u64_u32(y), 53 - 32);
661 z = veorq_u64(vreinterpretq_u64_u32(x), z);
662
663 // convert uint64 to double
664 float64x2_t rs = vcvtq_f64_u64(z);
665 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
666 rs = vfmaq_f64(vdupq_n_f64(TWOPOW53_INV_DOUBLE / 2.0),
667 vdupq_n_f64(TWOPOW53_INV_DOUBLE), rs);
668
669 return rs;
670}
671
672QUALIFIERS void philox_float4(uint32x4_t ctr0, uint32x4_t ctr1, uint32x4_t ctr2,
673 uint32x4_t ctr3, uint32 key0, uint32 key1,
674 float32x4_t &rnd1, float32x4_t &rnd2,
675 float32x4_t &rnd3, float32x4_t &rnd4) {
676 uint32x4_t key[2] = {vdupq_n_u32(key0), vdupq_n_u32(key1)};
677 uint32x4_t ctr[4] = {ctr0, ctr1, ctr2, ctr3};
678 _philox4x32round(ctr, key); // 1
679 _philox4x32bumpkey(key);
680 _philox4x32round(ctr, key); // 2
681 _philox4x32bumpkey(key);
682 _philox4x32round(ctr, key); // 3
683 _philox4x32bumpkey(key);
684 _philox4x32round(ctr, key); // 4
685 _philox4x32bumpkey(key);
686 _philox4x32round(ctr, key); // 5
687 _philox4x32bumpkey(key);
688 _philox4x32round(ctr, key); // 6
689 _philox4x32bumpkey(key);
690 _philox4x32round(ctr, key); // 7
691 _philox4x32bumpkey(key);
692 _philox4x32round(ctr, key); // 8
693 _philox4x32bumpkey(key);
694 _philox4x32round(ctr, key); // 9
695 _philox4x32bumpkey(key);
696 _philox4x32round(ctr, key); // 10
697
698 // convert uint32 to float
699 rnd1 = vcvtq_f32_u32(ctr[0]);
700 rnd2 = vcvtq_f32_u32(ctr[1]);
701 rnd3 = vcvtq_f32_u32(ctr[2]);
702 rnd4 = vcvtq_f32_u32(ctr[3]);
703 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
704 rnd1 = vfmaq_f32(vdupq_n_f32(TWOPOW32_INV_FLOAT / 2.0),
705 vdupq_n_f32(TWOPOW32_INV_FLOAT), rnd1);
706 rnd2 = vfmaq_f32(vdupq_n_f32(TWOPOW32_INV_FLOAT / 2.0),
707 vdupq_n_f32(TWOPOW32_INV_FLOAT), rnd2);
708 rnd3 = vfmaq_f32(vdupq_n_f32(TWOPOW32_INV_FLOAT / 2.0),
709 vdupq_n_f32(TWOPOW32_INV_FLOAT), rnd3);
710 rnd4 = vfmaq_f32(vdupq_n_f32(TWOPOW32_INV_FLOAT / 2.0),
711 vdupq_n_f32(TWOPOW32_INV_FLOAT), rnd4);
712}
713
714QUALIFIERS void philox_double2(uint32x4_t ctr0, uint32x4_t ctr1,
715 uint32x4_t ctr2, uint32x4_t ctr3, uint32 key0,
716 uint32 key1, float64x2_t &rnd1lo,
717 float64x2_t &rnd1hi, float64x2_t &rnd2lo,
718 float64x2_t &rnd2hi) {
719 uint32x4_t key[2] = {vdupq_n_u32(key0), vdupq_n_u32(key1)};
720 uint32x4_t ctr[4] = {ctr0, ctr1, ctr2, ctr3};
721 _philox4x32round(ctr, key); // 1
722 _philox4x32bumpkey(key);
723 _philox4x32round(ctr, key); // 2
724 _philox4x32bumpkey(key);
725 _philox4x32round(ctr, key); // 3
726 _philox4x32bumpkey(key);
727 _philox4x32round(ctr, key); // 4
728 _philox4x32bumpkey(key);
729 _philox4x32round(ctr, key); // 5
730 _philox4x32bumpkey(key);
731 _philox4x32round(ctr, key); // 6
732 _philox4x32bumpkey(key);
733 _philox4x32round(ctr, key); // 7
734 _philox4x32bumpkey(key);
735 _philox4x32round(ctr, key); // 8
736 _philox4x32bumpkey(key);
737 _philox4x32round(ctr, key); // 9
738 _philox4x32bumpkey(key);
739 _philox4x32round(ctr, key); // 10
740
741 rnd1lo = _uniform_double_hq<false>(ctr[0], ctr[1]);
742 rnd1hi = _uniform_double_hq<true>(ctr[0], ctr[1]);
743 rnd2lo = _uniform_double_hq<false>(ctr[2], ctr[3]);
744 rnd2hi = _uniform_double_hq<true>(ctr[2], ctr[3]);
745}
746
747QUALIFIERS void philox_float4(uint32 ctr0, uint32x4_t ctr1, uint32 ctr2,
748 uint32 ctr3, uint32 key0, uint32 key1,
749 float32x4_t &rnd1, float32x4_t &rnd2,
750 float32x4_t &rnd3, float32x4_t &rnd4) {
751 uint32x4_t ctr0v = vdupq_n_u32(ctr0);
752 uint32x4_t ctr2v = vdupq_n_u32(ctr2);
753 uint32x4_t ctr3v = vdupq_n_u32(ctr3);
754
755 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
756}
757
758QUALIFIERS void philox_float4(uint32 ctr0, int32x4_t ctr1, uint32 ctr2,
759 uint32 ctr3, uint32 key0, uint32 key1,
760 float32x4_t &rnd1, float32x4_t &rnd2,
761 float32x4_t &rnd3, float32x4_t &rnd4) {
762 philox_float4(ctr0, vreinterpretq_u32_s32(ctr1), ctr2, ctr3, key0, key1, rnd1,
763 rnd2, rnd3, rnd4);
764}
765
766QUALIFIERS void philox_double2(uint32 ctr0, uint32x4_t ctr1, uint32 ctr2,
767 uint32 ctr3, uint32 key0, uint32 key1,
768 float64x2_t &rnd1lo, float64x2_t &rnd1hi,
769 float64x2_t &rnd2lo, float64x2_t &rnd2hi) {
770 uint32x4_t ctr0v = vdupq_n_u32(ctr0);
771 uint32x4_t ctr2v = vdupq_n_u32(ctr2);
772 uint32x4_t ctr3v = vdupq_n_u32(ctr3);
773
774 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
775 rnd2hi);
776}
777
778QUALIFIERS void philox_double2(uint32 ctr0, uint32x4_t ctr1, uint32 ctr2,
779 uint32 ctr3, uint32 key0, uint32 key1,
780 float64x2_t &rnd1, float64x2_t &rnd2) {
781 uint32x4_t ctr0v = vdupq_n_u32(ctr0);
782 uint32x4_t ctr2v = vdupq_n_u32(ctr2);
783 uint32x4_t ctr3v = vdupq_n_u32(ctr3);
784
785 float64x2_t ignore;
786 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2,
787 ignore);
788}
789
790QUALIFIERS void philox_double2(uint32 ctr0, int32x4_t ctr1, uint32 ctr2,
791 uint32 ctr3, uint32 key0, uint32 key1,
792 float64x2_t &rnd1, float64x2_t &rnd2) {
793 philox_double2(ctr0, vreinterpretq_u32_s32(ctr1), ctr2, ctr3, key0, key1,
794 rnd1, rnd2);
795}
796#endif
797
798#if defined(__ARM_FEATURE_SVE)
799QUALIFIERS void _philox4x32round(svuint32x4_t &ctr, svuint32x2_t &key) {
800 svuint32_t lo0 =
801 svmul_u32_x(svptrue_b32(), svget4_u32(ctr, 0), svdup_u32(PHILOX_M4x32_0));
802 svuint32_t lo1 =
803 svmul_u32_x(svptrue_b32(), svget4_u32(ctr, 2), svdup_u32(PHILOX_M4x32_1));
804 svuint32_t hi0 = svmulh_u32_x(svptrue_b32(), svget4_u32(ctr, 0),
805 svdup_u32(PHILOX_M4x32_0));
806 svuint32_t hi1 = svmulh_u32_x(svptrue_b32(), svget4_u32(ctr, 2),
807 svdup_u32(PHILOX_M4x32_1));
808
809 ctr = svset4_u32(
810 ctr, 0,
811 sveor_u32_x(svptrue_b32(),
812 sveor_u32_x(svptrue_b32(), hi1, svget4_u32(ctr, 1)),
813 svget2_u32(key, 0)));
814 ctr = svset4_u32(ctr, 1, lo1);
815 ctr = svset4_u32(
816 ctr, 2,
817 sveor_u32_x(svptrue_b32(),
818 sveor_u32_x(svptrue_b32(), hi0, svget4_u32(ctr, 3)),
819 svget2_u32(key, 1)));
820 ctr = svset4_u32(ctr, 3, lo0);
821}
822
823QUALIFIERS void _philox4x32bumpkey(svuint32x2_t &key) {
824 key = svset2_u32(
825 key, 0,
826 svadd_u32_x(svptrue_b32(), svget2_u32(key, 0), svdup_u32(PHILOX_W32_0)));
827 key = svset2_u32(
828 key, 1,
829 svadd_u32_x(svptrue_b32(), svget2_u32(key, 1), svdup_u32(PHILOX_W32_1)));
830}
831
832template <bool high>
833QUALIFIERS svfloat64_t _uniform_double_hq(svuint32_t x, svuint32_t y) {
834 // convert 32 to 64 bit
835 if (high) {
836 x = svzip2_u32(x, svdup_u32(0));
837 y = svzip2_u32(y, svdup_u32(0));
838 } else {
839 x = svzip1_u32(x, svdup_u32(0));
840 y = svzip1_u32(y, svdup_u32(0));
841 }
842
843 // calculate z = x ^ y << (53 - 32))
844 svuint64_t z =
845 svlsl_n_u64_x(svptrue_b64(), svreinterpret_u64_u32(y), 53 - 32);
846 z = sveor_u64_x(svptrue_b64(), svreinterpret_u64_u32(x), z);
847
848 // convert uint64 to double
849 svfloat64_t rs = svcvt_f64_u64_x(svptrue_b64(), z);
850 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
851 rs = svmad_f64_x(svptrue_b64(), rs, svdup_f64(TWOPOW53_INV_DOUBLE),
852 svdup_f64(TWOPOW53_INV_DOUBLE / 2.0));
853
854 return rs;
855}
856
857QUALIFIERS void philox_float4(svuint32_t ctr0, svuint32_t ctr1, svuint32_t ctr2,
858 svuint32_t ctr3, uint32 key0, uint32 key1,
859 svfloat32_st &rnd1, svfloat32_st &rnd2,
860 svfloat32_st &rnd3, svfloat32_st &rnd4) {
861 svuint32x2_t key = svcreate2_u32(svdup_u32(key0), svdup_u32(key1));
862 svuint32x4_t ctr = svcreate4_u32(ctr0, ctr1, ctr2, ctr3);
863 _philox4x32round(ctr, key); // 1
864 _philox4x32bumpkey(key);
865 _philox4x32round(ctr, key); // 2
866 _philox4x32bumpkey(key);
867 _philox4x32round(ctr, key); // 3
868 _philox4x32bumpkey(key);
869 _philox4x32round(ctr, key); // 4
870 _philox4x32bumpkey(key);
871 _philox4x32round(ctr, key); // 5
872 _philox4x32bumpkey(key);
873 _philox4x32round(ctr, key); // 6
874 _philox4x32bumpkey(key);
875 _philox4x32round(ctr, key); // 7
876 _philox4x32bumpkey(key);
877 _philox4x32round(ctr, key); // 8
878 _philox4x32bumpkey(key);
879 _philox4x32round(ctr, key); // 9
880 _philox4x32bumpkey(key);
881 _philox4x32round(ctr, key); // 10
882
883 // convert uint32 to float
884 rnd1 = svcvt_f32_u32_x(svptrue_b32(), svget4_u32(ctr, 0));
885 rnd2 = svcvt_f32_u32_x(svptrue_b32(), svget4_u32(ctr, 1));
886 rnd3 = svcvt_f32_u32_x(svptrue_b32(), svget4_u32(ctr, 2));
887 rnd4 = svcvt_f32_u32_x(svptrue_b32(), svget4_u32(ctr, 3));
888 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
889 rnd1 = svmad_f32_x(svptrue_b32(), rnd1, svdup_f32(TWOPOW32_INV_FLOAT),
890 svdup_f32(TWOPOW32_INV_FLOAT / 2.0));
891 rnd2 = svmad_f32_x(svptrue_b32(), rnd2, svdup_f32(TWOPOW32_INV_FLOAT),
892 svdup_f32(TWOPOW32_INV_FLOAT / 2.0));
893 rnd3 = svmad_f32_x(svptrue_b32(), rnd3, svdup_f32(TWOPOW32_INV_FLOAT),
894 svdup_f32(TWOPOW32_INV_FLOAT / 2.0));
895 rnd4 = svmad_f32_x(svptrue_b32(), rnd4, svdup_f32(TWOPOW32_INV_FLOAT),
896 svdup_f32(TWOPOW32_INV_FLOAT / 2.0));
897}
898
899QUALIFIERS void philox_double2(svuint32_t ctr0, svuint32_t ctr1,
900 svuint32_t ctr2, svuint32_t ctr3, uint32 key0,
901 uint32 key1, svfloat64_st &rnd1lo,
902 svfloat64_st &rnd1hi, svfloat64_st &rnd2lo,
903 svfloat64_st &rnd2hi) {
904 svuint32x2_t key = svcreate2_u32(svdup_u32(key0), svdup_u32(key1));
905 svuint32x4_t ctr = svcreate4_u32(ctr0, ctr1, ctr2, ctr3);
906 _philox4x32round(ctr, key); // 1
907 _philox4x32bumpkey(key);
908 _philox4x32round(ctr, key); // 2
909 _philox4x32bumpkey(key);
910 _philox4x32round(ctr, key); // 3
911 _philox4x32bumpkey(key);
912 _philox4x32round(ctr, key); // 4
913 _philox4x32bumpkey(key);
914 _philox4x32round(ctr, key); // 5
915 _philox4x32bumpkey(key);
916 _philox4x32round(ctr, key); // 6
917 _philox4x32bumpkey(key);
918 _philox4x32round(ctr, key); // 7
919 _philox4x32bumpkey(key);
920 _philox4x32round(ctr, key); // 8
921 _philox4x32bumpkey(key);
922 _philox4x32round(ctr, key); // 9
923 _philox4x32bumpkey(key);
924 _philox4x32round(ctr, key); // 10
925
926 rnd1lo = _uniform_double_hq<false>(svget4_u32(ctr, 0), svget4_u32(ctr, 1));
927 rnd1hi = _uniform_double_hq<true>(svget4_u32(ctr, 0), svget4_u32(ctr, 1));
928 rnd2lo = _uniform_double_hq<false>(svget4_u32(ctr, 2), svget4_u32(ctr, 3));
929 rnd2hi = _uniform_double_hq<true>(svget4_u32(ctr, 2), svget4_u32(ctr, 3));
930}
931
932QUALIFIERS void philox_float4(uint32 ctr0, svuint32_t ctr1, uint32 ctr2,
933 uint32 ctr3, uint32 key0, uint32 key1,
934 svfloat32_st &rnd1, svfloat32_st &rnd2,
935 svfloat32_st &rnd3, svfloat32_st &rnd4) {
936 svuint32_t ctr0v = svdup_u32(ctr0);
937 svuint32_t ctr2v = svdup_u32(ctr2);
938 svuint32_t ctr3v = svdup_u32(ctr3);
939
940 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
941}
942
943QUALIFIERS void philox_float4(uint32 ctr0, svint32_t ctr1, uint32 ctr2,
944 uint32 ctr3, uint32 key0, uint32 key1,
945 svfloat32_st &rnd1, svfloat32_st &rnd2,
946 svfloat32_st &rnd3, svfloat32_st &rnd4) {
947 philox_float4(ctr0, svreinterpret_u32_s32(ctr1), ctr2, ctr3, key0, key1, rnd1,
948 rnd2, rnd3, rnd4);
949}
950
951QUALIFIERS void philox_double2(uint32 ctr0, svuint32_t ctr1, uint32 ctr2,
952 uint32 ctr3, uint32 key0, uint32 key1,
953 svfloat64_st &rnd1lo, svfloat64_st &rnd1hi,
954 svfloat64_st &rnd2lo, svfloat64_st &rnd2hi) {
955 svuint32_t ctr0v = svdup_u32(ctr0);
956 svuint32_t ctr2v = svdup_u32(ctr2);
957 svuint32_t ctr3v = svdup_u32(ctr3);
958
959 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
960 rnd2hi);
961}
962
963QUALIFIERS void philox_double2(uint32 ctr0, svuint32_t ctr1, uint32 ctr2,
964 uint32 ctr3, uint32 key0, uint32 key1,
965 svfloat64_st &rnd1, svfloat64_st &rnd2) {
966 svuint32_t ctr0v = svdup_u32(ctr0);
967 svuint32_t ctr2v = svdup_u32(ctr2);
968 svuint32_t ctr3v = svdup_u32(ctr3);
969
970 svfloat64_st ignore;
971 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2,
972 ignore);
973}
974
975QUALIFIERS void philox_double2(uint32 ctr0, svint32_t ctr1, uint32 ctr2,
976 uint32 ctr3, uint32 key0, uint32 key1,
977 svfloat64_st &rnd1, svfloat64_st &rnd2) {
978 philox_double2(ctr0, svreinterpret_u32_s32(ctr1), ctr2, ctr3, key0, key1,
979 rnd1, rnd2);
980}
981#endif
982
983#ifdef __AVX2__
984QUALIFIERS void _philox4x32round(__m256i *ctr, __m256i *key) {
985 __m256i lohi0a = _mm256_mul_epu32(ctr[0], _mm256_set1_epi32(PHILOX_M4x32_0));
986 __m256i lohi0b = _mm256_mul_epu32(_mm256_srli_epi64(ctr[0], 32),
987 _mm256_set1_epi32(PHILOX_M4x32_0));
988 __m256i lohi1a = _mm256_mul_epu32(ctr[2], _mm256_set1_epi32(PHILOX_M4x32_1));
989 __m256i lohi1b = _mm256_mul_epu32(_mm256_srli_epi64(ctr[2], 32),
990 _mm256_set1_epi32(PHILOX_M4x32_1));
991
992 lohi0a = _mm256_shuffle_epi32(lohi0a, 0xD8);
993 lohi0b = _mm256_shuffle_epi32(lohi0b, 0xD8);
994 lohi1a = _mm256_shuffle_epi32(lohi1a, 0xD8);
995 lohi1b = _mm256_shuffle_epi32(lohi1b, 0xD8);
996
997 __m256i lo0 = _mm256_unpacklo_epi32(lohi0a, lohi0b);
998 __m256i hi0 = _mm256_unpackhi_epi32(lohi0a, lohi0b);
999 __m256i lo1 = _mm256_unpacklo_epi32(lohi1a, lohi1b);
1000 __m256i hi1 = _mm256_unpackhi_epi32(lohi1a, lohi1b);
1001
1002 ctr[0] = _mm256_xor_si256(_mm256_xor_si256(hi1, ctr[1]), key[0]);
1003 ctr[1] = lo1;
1004 ctr[2] = _mm256_xor_si256(_mm256_xor_si256(hi0, ctr[3]), key[1]);
1005 ctr[3] = lo0;
1006}
1007
1008QUALIFIERS void _philox4x32bumpkey(__m256i *key) {
1009 key[0] = _mm256_add_epi32(key[0], _mm256_set1_epi32(PHILOX_W32_0));
1010 key[1] = _mm256_add_epi32(key[1], _mm256_set1_epi32(PHILOX_W32_1));
1011}
1012
1013template <bool high>
1014QUALIFIERS __m256d _uniform_double_hq(__m256i x, __m256i y) {
1015 // convert 32 to 64 bit
1016 if (high) {
1017 x = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(x, 1));
1018 y = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(y, 1));
1019 } else {
1020 x = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(x, 0));
1021 y = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(y, 0));
1022 }
1023
1024 // calculate z = x ^ y << (53 - 32))
1025 __m256i z = _mm256_sll_epi64(y, _mm_set1_epi64x(53 - 32));
1026 z = _mm256_xor_si256(x, z);
1027
1028 // convert uint64 to double
1029 __m256d rs = _my256_cvtepu64_pd(z);
1030 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
1031#ifdef __FMA__
1032 rs = _mm256_fmadd_pd(rs, _mm256_set1_pd(TWOPOW53_INV_DOUBLE),
1033 _mm256_set1_pd(TWOPOW53_INV_DOUBLE / 2.0));
1034#else
1035 rs = _mm256_mul_pd(rs, _mm256_set1_pd(TWOPOW53_INV_DOUBLE));
1036 rs = _mm256_add_pd(rs, _mm256_set1_pd(TWOPOW53_INV_DOUBLE / 2.0));
1037#endif
1038
1039 return rs;
1040}
1041
1042QUALIFIERS void philox_float4(__m256i ctr0, __m256i ctr1, __m256i ctr2,
1043 __m256i ctr3, uint32 key0, uint32 key1,
1044 __m256 &rnd1, __m256 &rnd2, __m256 &rnd3,
1045 __m256 &rnd4) {
1046 __m256i key[2] = {_mm256_set1_epi32(key0), _mm256_set1_epi32(key1)};
1047 __m256i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
1048 _philox4x32round(ctr, key); // 1
1049 _philox4x32bumpkey(key);
1050 _philox4x32round(ctr, key); // 2
1051 _philox4x32bumpkey(key);
1052 _philox4x32round(ctr, key); // 3
1053 _philox4x32bumpkey(key);
1054 _philox4x32round(ctr, key); // 4
1055 _philox4x32bumpkey(key);
1056 _philox4x32round(ctr, key); // 5
1057 _philox4x32bumpkey(key);
1058 _philox4x32round(ctr, key); // 6
1059 _philox4x32bumpkey(key);
1060 _philox4x32round(ctr, key); // 7
1061 _philox4x32bumpkey(key);
1062 _philox4x32round(ctr, key); // 8
1063 _philox4x32bumpkey(key);
1064 _philox4x32round(ctr, key); // 9
1065 _philox4x32bumpkey(key);
1066 _philox4x32round(ctr, key); // 10
1067
1068 // convert uint32 to float
1069 rnd1 = _my256_cvtepu32_ps(ctr[0]);
1070 rnd2 = _my256_cvtepu32_ps(ctr[1]);
1071 rnd3 = _my256_cvtepu32_ps(ctr[2]);
1072 rnd4 = _my256_cvtepu32_ps(ctr[3]);
1073 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
1074#ifdef __FMA__
1075 rnd1 = _mm256_fmadd_ps(rnd1, _mm256_set1_ps(TWOPOW32_INV_FLOAT),
1076 _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1077 rnd2 = _mm256_fmadd_ps(rnd2, _mm256_set1_ps(TWOPOW32_INV_FLOAT),
1078 _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1079 rnd3 = _mm256_fmadd_ps(rnd3, _mm256_set1_ps(TWOPOW32_INV_FLOAT),
1080 _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1081 rnd4 = _mm256_fmadd_ps(rnd4, _mm256_set1_ps(TWOPOW32_INV_FLOAT),
1082 _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1083#else
1084 rnd1 = _mm256_mul_ps(rnd1, _mm256_set1_ps(TWOPOW32_INV_FLOAT));
1085 rnd1 = _mm256_add_ps(rnd1, _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
1086 rnd2 = _mm256_mul_ps(rnd2, _mm256_set1_ps(TWOPOW32_INV_FLOAT));
1087 rnd2 = _mm256_add_ps(rnd2, _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
1088 rnd3 = _mm256_mul_ps(rnd3, _mm256_set1_ps(TWOPOW32_INV_FLOAT));
1089 rnd3 = _mm256_add_ps(rnd3, _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
1090 rnd4 = _mm256_mul_ps(rnd4, _mm256_set1_ps(TWOPOW32_INV_FLOAT));
1091 rnd4 = _mm256_add_ps(rnd4, _mm256_set1_ps(TWOPOW32_INV_FLOAT / 2.0f));
1092#endif
1093}
1094
1095QUALIFIERS void philox_double2(__m256i ctr0, __m256i ctr1, __m256i ctr2,
1096 __m256i ctr3, uint32 key0, uint32 key1,
1097 __m256d &rnd1lo, __m256d &rnd1hi,
1098 __m256d &rnd2lo, __m256d &rnd2hi) {
1099 __m256i key[2] = {_mm256_set1_epi32(key0), _mm256_set1_epi32(key1)};
1100 __m256i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
1101 _philox4x32round(ctr, key); // 1
1102 _philox4x32bumpkey(key);
1103 _philox4x32round(ctr, key); // 2
1104 _philox4x32bumpkey(key);
1105 _philox4x32round(ctr, key); // 3
1106 _philox4x32bumpkey(key);
1107 _philox4x32round(ctr, key); // 4
1108 _philox4x32bumpkey(key);
1109 _philox4x32round(ctr, key); // 5
1110 _philox4x32bumpkey(key);
1111 _philox4x32round(ctr, key); // 6
1112 _philox4x32bumpkey(key);
1113 _philox4x32round(ctr, key); // 7
1114 _philox4x32bumpkey(key);
1115 _philox4x32round(ctr, key); // 8
1116 _philox4x32bumpkey(key);
1117 _philox4x32round(ctr, key); // 9
1118 _philox4x32bumpkey(key);
1119 _philox4x32round(ctr, key); // 10
1120
1121 rnd1lo = _uniform_double_hq<false>(ctr[0], ctr[1]);
1122 rnd1hi = _uniform_double_hq<true>(ctr[0], ctr[1]);
1123 rnd2lo = _uniform_double_hq<false>(ctr[2], ctr[3]);
1124 rnd2hi = _uniform_double_hq<true>(ctr[2], ctr[3]);
1125}
1126
1127QUALIFIERS void philox_float4(uint32 ctr0, __m256i ctr1, uint32 ctr2,
1128 uint32 ctr3, uint32 key0, uint32 key1,
1129 __m256 &rnd1, __m256 &rnd2, __m256 &rnd3,
1130 __m256 &rnd4) {
1131 __m256i ctr0v = _mm256_set1_epi32(ctr0);
1132 __m256i ctr2v = _mm256_set1_epi32(ctr2);
1133 __m256i ctr3v = _mm256_set1_epi32(ctr3);
1134
1135 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
1136}
1137
1138QUALIFIERS void philox_double2(uint32 ctr0, __m256i ctr1, uint32 ctr2,
1139 uint32 ctr3, uint32 key0, uint32 key1,
1140 __m256d &rnd1lo, __m256d &rnd1hi,
1141 __m256d &rnd2lo, __m256d &rnd2hi) {
1142 __m256i ctr0v = _mm256_set1_epi32(ctr0);
1143 __m256i ctr2v = _mm256_set1_epi32(ctr2);
1144 __m256i ctr3v = _mm256_set1_epi32(ctr3);
1145
1146 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
1147 rnd2hi);
1148}
1149
1150QUALIFIERS void philox_double2(uint32 ctr0, __m256i ctr1, uint32 ctr2,
1151 uint32 ctr3, uint32 key0, uint32 key1,
1152 __m256d &rnd1, __m256d &rnd2) {
1153#if 0
1154 __m256i ctr0v = _mm256_set1_epi32(ctr0);
1155 __m256i ctr2v = _mm256_set1_epi32(ctr2);
1156 __m256i ctr3v = _mm256_set1_epi32(ctr3);
1157
1158 __m256d ignore;
1159 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2, ignore);
1160#else
1161 __m128d rnd1lo, rnd1hi, rnd2lo, rnd2hi;
1162 philox_double2(ctr0, _mm256_extractf128_si256(ctr1, 0), ctr2, ctr3, key0,
1163 key1, rnd1lo, rnd1hi, rnd2lo, rnd2hi);
1164 rnd1 = _my256_set_m128d(rnd1hi, rnd1lo);
1165 rnd2 = _my256_set_m128d(rnd2hi, rnd2lo);
1166#endif
1167}
1168#endif
1169
1170#ifdef __AVX512F__
1171QUALIFIERS void _philox4x32round(__m512i *ctr, __m512i *key) {
1172 __m512i lohi0a = _mm512_mul_epu32(ctr[0], _mm512_set1_epi32(PHILOX_M4x32_0));
1173 __m512i lohi0b = _mm512_mul_epu32(_mm512_srli_epi64(ctr[0], 32),
1174 _mm512_set1_epi32(PHILOX_M4x32_0));
1175 __m512i lohi1a = _mm512_mul_epu32(ctr[2], _mm512_set1_epi32(PHILOX_M4x32_1));
1176 __m512i lohi1b = _mm512_mul_epu32(_mm512_srli_epi64(ctr[2], 32),
1177 _mm512_set1_epi32(PHILOX_M4x32_1));
1178
1179 lohi0a = _mm512_shuffle_epi32(lohi0a, _MM_PERM_DBCA);
1180 lohi0b = _mm512_shuffle_epi32(lohi0b, _MM_PERM_DBCA);
1181 lohi1a = _mm512_shuffle_epi32(lohi1a, _MM_PERM_DBCA);
1182 lohi1b = _mm512_shuffle_epi32(lohi1b, _MM_PERM_DBCA);
1183
1184 __m512i lo0 = _mm512_unpacklo_epi32(lohi0a, lohi0b);
1185 __m512i hi0 = _mm512_unpackhi_epi32(lohi0a, lohi0b);
1186 __m512i lo1 = _mm512_unpacklo_epi32(lohi1a, lohi1b);
1187 __m512i hi1 = _mm512_unpackhi_epi32(lohi1a, lohi1b);
1188
1189 ctr[0] = _mm512_xor_si512(_mm512_xor_si512(hi1, ctr[1]), key[0]);
1190 ctr[1] = lo1;
1191 ctr[2] = _mm512_xor_si512(_mm512_xor_si512(hi0, ctr[3]), key[1]);
1192 ctr[3] = lo0;
1193}
1194
1195QUALIFIERS void _philox4x32bumpkey(__m512i *key) {
1196 key[0] = _mm512_add_epi32(key[0], _mm512_set1_epi32(PHILOX_W32_0));
1197 key[1] = _mm512_add_epi32(key[1], _mm512_set1_epi32(PHILOX_W32_1));
1198}
1199
1200template <bool high>
1201QUALIFIERS __m512d _uniform_double_hq(__m512i x, __m512i y) {
1202 // convert 32 to 64 bit
1203 if (high) {
1204 x = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(x, 1));
1205 y = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(y, 1));
1206 } else {
1207 x = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(x, 0));
1208 y = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(y, 0));
1209 }
1210
1211 // calculate z = x ^ y << (53 - 32))
1212 __m512i z = _mm512_sll_epi64(y, _mm_set1_epi64x(53 - 32));
1213 z = _mm512_xor_si512(x, z);
1214
1215 // convert uint64 to double
1216 __m512d rs = _mm512_cvtepu64_pd(z);
1217 // calculate rs * TWOPOW53_INV_DOUBLE + (TWOPOW53_INV_DOUBLE/2.0)
1218 rs = _mm512_fmadd_pd(rs, _mm512_set1_pd(TWOPOW53_INV_DOUBLE),
1219 _mm512_set1_pd(TWOPOW53_INV_DOUBLE / 2.0));
1220
1221 return rs;
1222}
1223
1224QUALIFIERS void philox_float4(__m512i ctr0, __m512i ctr1, __m512i ctr2,
1225 __m512i ctr3, uint32 key0, uint32 key1,
1226 __m512 &rnd1, __m512 &rnd2, __m512 &rnd3,
1227 __m512 &rnd4) {
1228 __m512i key[2] = {_mm512_set1_epi32(key0), _mm512_set1_epi32(key1)};
1229 __m512i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
1230 _philox4x32round(ctr, key); // 1
1231 _philox4x32bumpkey(key);
1232 _philox4x32round(ctr, key); // 2
1233 _philox4x32bumpkey(key);
1234 _philox4x32round(ctr, key); // 3
1235 _philox4x32bumpkey(key);
1236 _philox4x32round(ctr, key); // 4
1237 _philox4x32bumpkey(key);
1238 _philox4x32round(ctr, key); // 5
1239 _philox4x32bumpkey(key);
1240 _philox4x32round(ctr, key); // 6
1241 _philox4x32bumpkey(key);
1242 _philox4x32round(ctr, key); // 7
1243 _philox4x32bumpkey(key);
1244 _philox4x32round(ctr, key); // 8
1245 _philox4x32bumpkey(key);
1246 _philox4x32round(ctr, key); // 9
1247 _philox4x32bumpkey(key);
1248 _philox4x32round(ctr, key); // 10
1249
1250 // convert uint32 to float
1251 rnd1 = _mm512_cvtepu32_ps(ctr[0]);
1252 rnd2 = _mm512_cvtepu32_ps(ctr[1]);
1253 rnd3 = _mm512_cvtepu32_ps(ctr[2]);
1254 rnd4 = _mm512_cvtepu32_ps(ctr[3]);
1255 // calculate rnd * TWOPOW32_INV_FLOAT + (TWOPOW32_INV_FLOAT/2.0f)
1256 rnd1 = _mm512_fmadd_ps(rnd1, _mm512_set1_ps(TWOPOW32_INV_FLOAT),
1257 _mm512_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1258 rnd2 = _mm512_fmadd_ps(rnd2, _mm512_set1_ps(TWOPOW32_INV_FLOAT),
1259 _mm512_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1260 rnd3 = _mm512_fmadd_ps(rnd3, _mm512_set1_ps(TWOPOW32_INV_FLOAT),
1261 _mm512_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1262 rnd4 = _mm512_fmadd_ps(rnd4, _mm512_set1_ps(TWOPOW32_INV_FLOAT),
1263 _mm512_set1_ps(TWOPOW32_INV_FLOAT / 2.0));
1264}
1265
1266QUALIFIERS void philox_double2(__m512i ctr0, __m512i ctr1, __m512i ctr2,
1267 __m512i ctr3, uint32 key0, uint32 key1,
1268 __m512d &rnd1lo, __m512d &rnd1hi,
1269 __m512d &rnd2lo, __m512d &rnd2hi) {
1270 __m512i key[2] = {_mm512_set1_epi32(key0), _mm512_set1_epi32(key1)};
1271 __m512i ctr[4] = {ctr0, ctr1, ctr2, ctr3};
1272 _philox4x32round(ctr, key); // 1
1273 _philox4x32bumpkey(key);
1274 _philox4x32round(ctr, key); // 2
1275 _philox4x32bumpkey(key);
1276 _philox4x32round(ctr, key); // 3
1277 _philox4x32bumpkey(key);
1278 _philox4x32round(ctr, key); // 4
1279 _philox4x32bumpkey(key);
1280 _philox4x32round(ctr, key); // 5
1281 _philox4x32bumpkey(key);
1282 _philox4x32round(ctr, key); // 6
1283 _philox4x32bumpkey(key);
1284 _philox4x32round(ctr, key); // 7
1285 _philox4x32bumpkey(key);
1286 _philox4x32round(ctr, key); // 8
1287 _philox4x32bumpkey(key);
1288 _philox4x32round(ctr, key); // 9
1289 _philox4x32bumpkey(key);
1290 _philox4x32round(ctr, key); // 10
1291
1292 rnd1lo = _uniform_double_hq<false>(ctr[0], ctr[1]);
1293 rnd1hi = _uniform_double_hq<true>(ctr[0], ctr[1]);
1294 rnd2lo = _uniform_double_hq<false>(ctr[2], ctr[3]);
1295 rnd2hi = _uniform_double_hq<true>(ctr[2], ctr[3]);
1296}
1297
1298QUALIFIERS void philox_float4(uint32 ctr0, __m512i ctr1, uint32 ctr2,
1299 uint32 ctr3, uint32 key0, uint32 key1,
1300 __m512 &rnd1, __m512 &rnd2, __m512 &rnd3,
1301 __m512 &rnd4) {
1302 __m512i ctr0v = _mm512_set1_epi32(ctr0);
1303 __m512i ctr2v = _mm512_set1_epi32(ctr2);
1304 __m512i ctr3v = _mm512_set1_epi32(ctr3);
1305
1306 philox_float4(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, rnd2, rnd3, rnd4);
1307}
1308
1309QUALIFIERS void philox_double2(uint32 ctr0, __m512i ctr1, uint32 ctr2,
1310 uint32 ctr3, uint32 key0, uint32 key1,
1311 __m512d &rnd1lo, __m512d &rnd1hi,
1312 __m512d &rnd2lo, __m512d &rnd2hi) {
1313 __m512i ctr0v = _mm512_set1_epi32(ctr0);
1314 __m512i ctr2v = _mm512_set1_epi32(ctr2);
1315 __m512i ctr3v = _mm512_set1_epi32(ctr3);
1316
1317 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1lo, rnd1hi, rnd2lo,
1318 rnd2hi);
1319}
1320
1321QUALIFIERS void philox_double2(uint32 ctr0, __m512i ctr1, uint32 ctr2,
1322 uint32 ctr3, uint32 key0, uint32 key1,
1323 __m512d &rnd1, __m512d &rnd2) {
1324#if 0
1325 __m512i ctr0v = _mm512_set1_epi32(ctr0);
1326 __m512i ctr2v = _mm512_set1_epi32(ctr2);
1327 __m512i ctr3v = _mm512_set1_epi32(ctr3);
1328
1329 __m512d ignore;
1330 philox_double2(ctr0v, ctr1, ctr2v, ctr3v, key0, key1, rnd1, ignore, rnd2, ignore);
1331#else
1332 __m256d rnd1lo, rnd1hi, rnd2lo, rnd2hi;
1333 philox_double2(ctr0, _mm512_extracti64x4_epi64(ctr1, 0), ctr2, ctr3, key0,
1334 key1, rnd1lo, rnd1hi, rnd2lo, rnd2hi);
1335 rnd1 = _my512_set_m256d(rnd1hi, rnd1lo);
1336 rnd2 = _my512_set_m256d(rnd2hi, rnd2lo);
1337#endif
1338}
1339#endif
1340#endif
stream
cudaStream_t stream[1]
CUDA streams for parallel computing on CPU and GPU.
Definition common_cuda.cu:34
uint32
std::uint32_t uint32
Definition electrokinetics/generated_kernels/philox_rand.h:93
uint64
std::uint64_t uint64
Definition electrokinetics/generated_kernels/philox_rand.h:94
myintrin.h
Philox counter-based RNG utility functions.
_philox4x32bumpkey
QUALIFIERS void _philox4x32bumpkey(uint32 *key)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:137
uint32
std::uint32_t uint32
Definition lattice_boltzmann/generated_kernels/philox_rand.h:93
TWOPOW53_INV_DOUBLE
#define TWOPOW53_INV_DOUBLE
Definition lattice_boltzmann/generated_kernels/philox_rand.h:90
QUALIFIERS
#define QUALIFIERS
Definition lattice_boltzmann/generated_kernels/philox_rand.h:82
_philox4x32round
QUALIFIERS void _philox4x32round(uint32 *ctr, uint32 *key)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:125
mulhilo32
QUALIFIERS uint32 mulhilo32(uint32 a, uint32 b, uint32 *hip)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:107
PHILOX_W32_1
#define PHILOX_W32_1
Definition lattice_boltzmann/generated_kernels/philox_rand.h:87
PHILOX_M4x32_0
#define PHILOX_M4x32_0
Definition lattice_boltzmann/generated_kernels/philox_rand.h:88
PHILOX_M4x32_1
#define PHILOX_M4x32_1
Definition lattice_boltzmann/generated_kernels/philox_rand.h:89
TWOPOW32_INV_FLOAT
#define TWOPOW32_INV_FLOAT
Definition lattice_boltzmann/generated_kernels/philox_rand.h:91
uint64
std::uint64_t uint64
Definition lattice_boltzmann/generated_kernels/philox_rand.h:94
PHILOX_W32_0
#define PHILOX_W32_0
Definition lattice_boltzmann/generated_kernels/philox_rand.h:86
philox_double2
QUALIFIERS void philox_double2(uint32 ctr0, uint32 ctr1, uint32 ctr2, uint32 ctr3, uint32 key0, uint32 key1, double &rnd1, double &rnd2)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:147
philox_float4
QUALIFIERS void philox_float4(uint32 ctr0, uint32 ctr1, uint32 ctr2, uint32 ctr3, uint32 key0, uint32 key1, float &rnd1, float &rnd2, float &rnd3, float &rnd4)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:176
_uniform_double_hq
QUALIFIERS double _uniform_double_hq(uint32 x, uint32 y)
Definition lattice_boltzmann/generated_kernels/philox_rand.h:142