Botan 3.0.0
Crypto and TLS for C&
simd_32.h
Go to the documentation of this file.
1/*
2* Lightweight wrappers for SIMD operations
3* (C) 2009,2011,2016,2017,2019 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#ifndef BOTAN_SIMD_32_H_
9#define BOTAN_SIMD_32_H_
10
11#include <botan/types.h>
12
13#if defined(BOTAN_TARGET_SUPPORTS_SSE2)
14 #include <emmintrin.h>
15 #define BOTAN_SIMD_USE_SSE2
16
17#elif defined(BOTAN_TARGET_SUPPORTS_ALTIVEC)
18 #include <botan/internal/bswap.h>
19 #include <botan/internal/loadstor.h>
20 #include <altivec.h>
21 #undef vector
22 #undef bool
23 #define BOTAN_SIMD_USE_ALTIVEC
24
25#elif defined(BOTAN_TARGET_SUPPORTS_NEON)
26 #include <botan/internal/cpuid.h>
27 #include <arm_neon.h>
28 #define BOTAN_SIMD_USE_NEON
29
30#else
31 #error "No SIMD instruction set enabled"
32#endif
33
34#if defined(BOTAN_SIMD_USE_SSE2)
35 #define BOTAN_SIMD_ISA "sse2"
36 #define BOTAN_VPERM_ISA "ssse3"
37 #define BOTAN_CLMUL_ISA "pclmul"
38#elif defined(BOTAN_SIMD_USE_NEON)
39 #if defined(BOTAN_TARGET_ARCH_IS_ARM64)
40 #define BOTAN_SIMD_ISA "+simd"
41 #define BOTAN_CLMUL_ISA "+crypto"
42 #else
43 #define BOTAN_SIMD_ISA "fpu=neon"
44 #endif
45 #define BOTAN_VPERM_ISA BOTAN_SIMD_ISA
46#elif defined(BOTAN_SIMD_USE_ALTIVEC)
47 #define BOTAN_SIMD_ISA "altivec"
48 #define BOTAN_VPERM_ISA "altivec"
49 #define BOTAN_CLMUL_ISA "crypto"
50#endif
51
52namespace Botan {
53
54#if defined(BOTAN_SIMD_USE_SSE2)
55 using native_simd_type = __m128i;
56#elif defined(BOTAN_SIMD_USE_ALTIVEC)
57 using native_simd_type = __vector unsigned int;
58#elif defined(BOTAN_SIMD_USE_NEON)
59 using native_simd_type = uint32x4_t;
60#endif
61
62/**
63* 4x32 bit SIMD register
64*
65* This class is not a general purpose SIMD type, and only offers
66* instructions needed for evaluation of specific crypto primitives.
67* For example it does not currently have equality operators of any
68* kind.
69*
70* Implemented for SSE2, VMX (Altivec), and NEON.
71*/
73 {
74 public:
75
76 SIMD_4x32& operator=(const SIMD_4x32& other) = default;
77 SIMD_4x32(const SIMD_4x32& other) = default;
78
79 SIMD_4x32& operator=(SIMD_4x32&& other) = default;
80 SIMD_4x32(SIMD_4x32&& other) = default;
81
82 ~SIMD_4x32() = default;
83
84 /**
85 * Zero initialize SIMD register with 4 32-bit elements
86 */
87 SIMD_4x32() noexcept // zero initialized
88 {
89#if defined(BOTAN_SIMD_USE_SSE2)
90 m_simd = _mm_setzero_si128();
91#elif defined(BOTAN_SIMD_USE_ALTIVEC)
92 m_simd = vec_splat_u32(0);
93#elif defined(BOTAN_SIMD_USE_NEON)
94 m_simd = vdupq_n_u32(0);
95#endif
96 }
97
98 /**
99 * Load SIMD register with 4 32-bit elements
100 */
101 explicit SIMD_4x32(const uint32_t B[4]) noexcept
102 {
103#if defined(BOTAN_SIMD_USE_SSE2)
104 m_simd = _mm_loadu_si128(reinterpret_cast<const __m128i*>(B));
105#elif defined(BOTAN_SIMD_USE_ALTIVEC)
106 __vector unsigned int val = { B[0], B[1], B[2], B[3]};
107 m_simd = val;
108#elif defined(BOTAN_SIMD_USE_NEON)
109 m_simd = vld1q_u32(B);
110#endif
111 }
112
113 /**
114 * Load SIMD register with 4 32-bit elements
115 */
116 SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept
117 {
118#if defined(BOTAN_SIMD_USE_SSE2)
119 m_simd = _mm_set_epi32(B3, B2, B1, B0);
120#elif defined(BOTAN_SIMD_USE_ALTIVEC)
121 __vector unsigned int val = {B0, B1, B2, B3};
122 m_simd = val;
123#elif defined(BOTAN_SIMD_USE_NEON)
124 // Better way to do this?
125 const uint32_t B[4] = { B0, B1, B2, B3 };
126 m_simd = vld1q_u32(B);
127#endif
128 }
129
130 /**
131 * Load SIMD register with one 32-bit element repeated
132 */
133 static SIMD_4x32 splat(uint32_t B) noexcept
134 {
135#if defined(BOTAN_SIMD_USE_SSE2)
136 return SIMD_4x32(_mm_set1_epi32(B));
137#elif defined(BOTAN_SIMD_USE_NEON)
138 return SIMD_4x32(vdupq_n_u32(B));
139#else
140 return SIMD_4x32(B, B, B, B);
141#endif
142 }
143
144 /**
145 * Load SIMD register with one 8-bit element repeated
146 */
147 static SIMD_4x32 splat_u8(uint8_t B) noexcept
148 {
149#if defined(BOTAN_SIMD_USE_SSE2)
150 return SIMD_4x32(_mm_set1_epi8(B));
151#elif defined(BOTAN_SIMD_USE_NEON)
152 return SIMD_4x32(vreinterpretq_u32_u8(vdupq_n_u8(B)));
153#else
154 const uint32_t B4 = make_uint32(B, B, B, B);
155 return SIMD_4x32(B4, B4, B4, B4);
156#endif
157 }
158
159 /**
160 * Load a SIMD register with little-endian convention
161 */
162 static SIMD_4x32 load_le(const void* in) noexcept
163 {
164#if defined(BOTAN_SIMD_USE_SSE2)
165 return SIMD_4x32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(in)));
166#elif defined(BOTAN_SIMD_USE_ALTIVEC)
167 uint32_t R[4];
168 Botan::load_le(R, static_cast<const uint8_t*>(in), 4);
169 return SIMD_4x32(R);
170#elif defined(BOTAN_SIMD_USE_NEON)
171 SIMD_4x32 l(vld1q_u32(static_cast<const uint32_t*>(in)));
172 return CPUID::is_big_endian() ? l.bswap() : l;
173#endif
174 }
175
176 /**
177 * Load a SIMD register with big-endian convention
178 */
179 static SIMD_4x32 load_be(const void* in) noexcept
180 {
181#if defined(BOTAN_SIMD_USE_SSE2)
182 return load_le(in).bswap();
183
184#elif defined(BOTAN_SIMD_USE_ALTIVEC)
185 uint32_t R[4];
186 Botan::load_be(R, static_cast<const uint8_t*>(in), 4);
187 return SIMD_4x32(R);
188
189#elif defined(BOTAN_SIMD_USE_NEON)
190 SIMD_4x32 l(vld1q_u32(static_cast<const uint32_t*>(in)));
191 return CPUID::is_little_endian() ? l.bswap() : l;
192#endif
193 }
194
195 void store_le(uint32_t out[4]) const noexcept
196 {
197 this->store_le(reinterpret_cast<uint8_t*>(out));
198 }
199
200 void store_be(uint32_t out[4]) const noexcept
201 {
202 this->store_be(reinterpret_cast<uint8_t*>(out));
203 }
204
205 void store_le(uint64_t out[2]) const noexcept
206 {
207 this->store_le(reinterpret_cast<uint8_t*>(out));
208 }
209
210 /**
211 * Load a SIMD register with little-endian convention
212 */
213 void store_le(uint8_t out[]) const noexcept
214 {
215#if defined(BOTAN_SIMD_USE_SSE2)
216
217 _mm_storeu_si128(reinterpret_cast<__m128i*>(out), raw());
218
219#elif defined(BOTAN_SIMD_USE_ALTIVEC)
220
221 union {
222 __vector unsigned int V;
223 uint32_t R[4];
224 } vec;
225 vec.V = raw();
226 Botan::store_le(out, vec.R[0], vec.R[1], vec.R[2], vec.R[3]);
227
228#elif defined(BOTAN_SIMD_USE_NEON)
230 {
231 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
232 }
233 else
234 {
235 vst1q_u8(out, vreinterpretq_u8_u32(bswap().m_simd));
236 }
237#endif
238 }
239
240 /**
241 * Load a SIMD register with big-endian convention
242 */
243 void store_be(uint8_t out[]) const noexcept
244 {
245#if defined(BOTAN_SIMD_USE_SSE2)
246
247 bswap().store_le(out);
248
249#elif defined(BOTAN_SIMD_USE_ALTIVEC)
250
251 union {
252 __vector unsigned int V;
253 uint32_t R[4];
254 } vec;
255 vec.V = m_simd;
256 Botan::store_be(out, vec.R[0], vec.R[1], vec.R[2], vec.R[3]);
257
258#elif defined(BOTAN_SIMD_USE_NEON)
260 {
261 vst1q_u8(out, vreinterpretq_u8_u32(bswap().m_simd));
262 }
263 else
264 {
265 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
266 }
267#endif
268 }
269
270 /*
271 * This is used for SHA-2/SHACAL2
272 */
273 SIMD_4x32 sigma0() const noexcept
274 {
275#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
276 return SIMD_4x32(__builtin_crypto_vshasigmaw(raw(), 1, 0));
277#else
278 const SIMD_4x32 rot1 = this->rotr<2>();
279 const SIMD_4x32 rot2 = this->rotr<13>();
280 const SIMD_4x32 rot3 = this->rotr<22>();
281 return (rot1 ^ rot2 ^ rot3);
282#endif
283 }
284
285 /*
286 * This is used for SHA-2/SHACAL2
287 */
288 SIMD_4x32 sigma1() const noexcept
289 {
290#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
291 return SIMD_4x32(__builtin_crypto_vshasigmaw(raw(), 1, 0xF));
292#else
293 const SIMD_4x32 rot1 = this->rotr<6>();
294 const SIMD_4x32 rot2 = this->rotr<11>();
295 const SIMD_4x32 rot3 = this->rotr<25>();
296 return (rot1 ^ rot2 ^ rot3);
297#endif
298 }
299
300 /**
301 * Left rotation by a compile time constant
302 */
303 template<size_t ROT>
304 SIMD_4x32 rotl() const noexcept
305 requires (ROT > 0 && ROT < 32)
306 {
307
308#if defined(BOTAN_SIMD_USE_SSE2)
309
310 return SIMD_4x32(_mm_or_si128(_mm_slli_epi32(m_simd, static_cast<int>(ROT)),
311 _mm_srli_epi32(m_simd, static_cast<int>(32-ROT))));
312
313#elif defined(BOTAN_SIMD_USE_ALTIVEC)
314
315 const unsigned int r = static_cast<unsigned int>(ROT);
316 __vector unsigned int rot = {r, r, r, r};
317 return SIMD_4x32(vec_rl(m_simd, rot));
318
319#elif defined(BOTAN_SIMD_USE_NEON)
320
321#if defined(BOTAN_TARGET_ARCH_IS_ARM64)
322
323 if constexpr(ROT == 8)
324 {
325 const uint8_t maskb[16] = { 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 };
326 const uint8x16_t mask = vld1q_u8(maskb);
327 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(m_simd), mask)));
328 }
329 else if constexpr(ROT == 16)
330 {
331 return SIMD_4x32(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(m_simd))));
332 }
333#endif
334 return SIMD_4x32(vorrq_u32(vshlq_n_u32(m_simd, static_cast<int>(ROT)),
335 vshrq_n_u32(m_simd, static_cast<int>(32-ROT))));
336#endif
337 }
338
339 /**
340 * Right rotation by a compile time constant
341 */
342 template<size_t ROT>
343 SIMD_4x32 rotr() const noexcept
344 {
345 return this->rotl<32-ROT>();
346 }
347
348 /**
349 * Add elements of a SIMD vector
350 */
351 SIMD_4x32 operator+(const SIMD_4x32& other) const noexcept
352 {
353 SIMD_4x32 retval(*this);
354 retval += other;
355 return retval;
356 }
357
358 /**
359 * Subtract elements of a SIMD vector
360 */
361 SIMD_4x32 operator-(const SIMD_4x32& other) const noexcept
362 {
363 SIMD_4x32 retval(*this);
364 retval -= other;
365 return retval;
366 }
367
368 /**
369 * XOR elements of a SIMD vector
370 */
371 SIMD_4x32 operator^(const SIMD_4x32& other) const noexcept
372 {
373 SIMD_4x32 retval(*this);
374 retval ^= other;
375 return retval;
376 }
377
378 /**
379 * Binary OR elements of a SIMD vector
380 */
381 SIMD_4x32 operator|(const SIMD_4x32& other) const noexcept
382 {
383 SIMD_4x32 retval(*this);
384 retval |= other;
385 return retval;
386 }
387
388 /**
389 * Binary AND elements of a SIMD vector
390 */
391 SIMD_4x32 operator&(const SIMD_4x32& other) const noexcept
392 {
393 SIMD_4x32 retval(*this);
394 retval &= other;
395 return retval;
396 }
397
398 void operator+=(const SIMD_4x32& other) noexcept
399 {
400#if defined(BOTAN_SIMD_USE_SSE2)
401 m_simd = _mm_add_epi32(m_simd, other.m_simd);
402#elif defined(BOTAN_SIMD_USE_ALTIVEC)
403 m_simd = vec_add(m_simd, other.m_simd);
404#elif defined(BOTAN_SIMD_USE_NEON)
405 m_simd = vaddq_u32(m_simd, other.m_simd);
406#endif
407 }
408
409 void operator-=(const SIMD_4x32& other) noexcept
410 {
411#if defined(BOTAN_SIMD_USE_SSE2)
412 m_simd = _mm_sub_epi32(m_simd, other.m_simd);
413#elif defined(BOTAN_SIMD_USE_ALTIVEC)
414 m_simd = vec_sub(m_simd, other.m_simd);
415#elif defined(BOTAN_SIMD_USE_NEON)
416 m_simd = vsubq_u32(m_simd, other.m_simd);
417#endif
418 }
419
420 void operator^=(const SIMD_4x32& other) noexcept
421 {
422#if defined(BOTAN_SIMD_USE_SSE2)
423 m_simd = _mm_xor_si128(m_simd, other.m_simd);
424#elif defined(BOTAN_SIMD_USE_ALTIVEC)
425 m_simd = vec_xor(m_simd, other.m_simd);
426#elif defined(BOTAN_SIMD_USE_NEON)
427 m_simd = veorq_u32(m_simd, other.m_simd);
428#endif
429 }
430
431 void operator^=(uint32_t other) noexcept
432 {
433 *this ^= SIMD_4x32::splat(other);
434 }
435
436 void operator|=(const SIMD_4x32& other) noexcept
437 {
438#if defined(BOTAN_SIMD_USE_SSE2)
439 m_simd = _mm_or_si128(m_simd, other.m_simd);
440#elif defined(BOTAN_SIMD_USE_ALTIVEC)
441 m_simd = vec_or(m_simd, other.m_simd);
442#elif defined(BOTAN_SIMD_USE_NEON)
443 m_simd = vorrq_u32(m_simd, other.m_simd);
444#endif
445 }
446
447 void operator&=(const SIMD_4x32& other) noexcept
448 {
449#if defined(BOTAN_SIMD_USE_SSE2)
450 m_simd = _mm_and_si128(m_simd, other.m_simd);
451#elif defined(BOTAN_SIMD_USE_ALTIVEC)
452 m_simd = vec_and(m_simd, other.m_simd);
453#elif defined(BOTAN_SIMD_USE_NEON)
454 m_simd = vandq_u32(m_simd, other.m_simd);
455#endif
456 }
457
458
459 template<int SHIFT> SIMD_4x32 shl() const noexcept
460 requires (SHIFT > 0 && SHIFT < 32)
461 {
462#if defined(BOTAN_SIMD_USE_SSE2)
463 return SIMD_4x32(_mm_slli_epi32(m_simd, SHIFT));
464
465#elif defined(BOTAN_SIMD_USE_ALTIVEC)
466 const unsigned int s = static_cast<unsigned int>(SHIFT);
467 const __vector unsigned int shifts = {s, s, s, s};
468 return SIMD_4x32(vec_sl(m_simd, shifts));
469#elif defined(BOTAN_SIMD_USE_NEON)
470 return SIMD_4x32(vshlq_n_u32(m_simd, SHIFT));
471#endif
472 }
473
474 template<int SHIFT> SIMD_4x32 shr() const noexcept
475 {
476#if defined(BOTAN_SIMD_USE_SSE2)
477 return SIMD_4x32(_mm_srli_epi32(m_simd, SHIFT));
478
479#elif defined(BOTAN_SIMD_USE_ALTIVEC)
480 const unsigned int s = static_cast<unsigned int>(SHIFT);
481 const __vector unsigned int shifts = {s, s, s, s};
482 return SIMD_4x32(vec_sr(m_simd, shifts));
483#elif defined(BOTAN_SIMD_USE_NEON)
484 return SIMD_4x32(vshrq_n_u32(m_simd, SHIFT));
485#endif
486 }
487
488 SIMD_4x32 operator~() const noexcept
489 {
490#if defined(BOTAN_SIMD_USE_SSE2)
491 return SIMD_4x32(_mm_xor_si128(m_simd, _mm_set1_epi32(0xFFFFFFFF)));
492#elif defined(BOTAN_SIMD_USE_ALTIVEC)
493 return SIMD_4x32(vec_nor(m_simd, m_simd));
494#elif defined(BOTAN_SIMD_USE_NEON)
495 return SIMD_4x32(vmvnq_u32(m_simd));
496#endif
497 }
498
499 // (~reg) & other
500 SIMD_4x32 andc(const SIMD_4x32& other) const noexcept
501 {
502#if defined(BOTAN_SIMD_USE_SSE2)
503 return SIMD_4x32(_mm_andnot_si128(m_simd, other.m_simd));
504#elif defined(BOTAN_SIMD_USE_ALTIVEC)
505 /*
506 AltiVec does arg1 & ~arg2 rather than SSE's ~arg1 & arg2
507 so swap the arguments
508 */
509 return SIMD_4x32(vec_andc(other.m_simd, m_simd));
510#elif defined(BOTAN_SIMD_USE_NEON)
511 // NEON is also a & ~b
512 return SIMD_4x32(vbicq_u32(other.m_simd, m_simd));
513#endif
514 }
515
516 /**
517 * Return copy *this with each word byte swapped
518 */
519 SIMD_4x32 bswap() const noexcept
520 {
521#if defined(BOTAN_SIMD_USE_SSE2)
522
523 __m128i T = m_simd;
524 T = _mm_shufflehi_epi16(T, _MM_SHUFFLE(2, 3, 0, 1));
525 T = _mm_shufflelo_epi16(T, _MM_SHUFFLE(2, 3, 0, 1));
526 return SIMD_4x32(_mm_or_si128(_mm_srli_epi16(T, 8), _mm_slli_epi16(T, 8)));
527
528#elif defined(BOTAN_SIMD_USE_ALTIVEC)
529
530 union {
531 __vector unsigned int V;
532 uint32_t R[4];
533 } vec;
534
535 vec.V = m_simd;
536 bswap_4(vec.R);
537 return SIMD_4x32(vec.R[0], vec.R[1], vec.R[2], vec.R[3]);
538
539#elif defined(BOTAN_SIMD_USE_NEON)
540 return SIMD_4x32(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(m_simd))));
541#endif
542 }
543
544 template<size_t I>
546 requires (I <= 3)
547 {
548#if defined(BOTAN_SIMD_USE_SSE2)
549 return SIMD_4x32(_mm_slli_si128(raw(), 4*I));
550#elif defined(BOTAN_SIMD_USE_NEON)
551 return SIMD_4x32(vextq_u32(vdupq_n_u32(0), raw(), 4-I));
552#elif defined(BOTAN_SIMD_USE_ALTIVEC)
553 const __vector unsigned int zero = vec_splat_u32(0);
554
555 const __vector unsigned char shuf[3] = {
556 { 16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 },
557 { 16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7 },
558 { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3 },
559 };
560
561 return SIMD_4x32(vec_perm(raw(), zero, shuf[I-1]));
562#endif
563 }
564
565 template<size_t I>
567 requires (I <= 3)
568 {
569#if defined(BOTAN_SIMD_USE_SSE2)
570 return SIMD_4x32(_mm_srli_si128(raw(), 4*I));
571#elif defined(BOTAN_SIMD_USE_NEON)
572 return SIMD_4x32(vextq_u32(raw(), vdupq_n_u32(0), I));
573#elif defined(BOTAN_SIMD_USE_ALTIVEC)
574 const __vector unsigned int zero = vec_splat_u32(0);
575
576 const __vector unsigned char shuf[3] = {
577 { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 },
578 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 },
579 { 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 },
580 };
581
582 return SIMD_4x32(vec_perm(raw(), zero, shuf[I-1]));
583#endif
584 }
585
586 /**
587 * 4x4 Transposition on SIMD registers
588 */
589 static void transpose(SIMD_4x32& B0, SIMD_4x32& B1,
590 SIMD_4x32& B2, SIMD_4x32& B3) noexcept
591 {
592#if defined(BOTAN_SIMD_USE_SSE2)
593 const __m128i T0 = _mm_unpacklo_epi32(B0.m_simd, B1.m_simd);
594 const __m128i T1 = _mm_unpacklo_epi32(B2.m_simd, B3.m_simd);
595 const __m128i T2 = _mm_unpackhi_epi32(B0.m_simd, B1.m_simd);
596 const __m128i T3 = _mm_unpackhi_epi32(B2.m_simd, B3.m_simd);
597
598 B0.m_simd = _mm_unpacklo_epi64(T0, T1);
599 B1.m_simd = _mm_unpackhi_epi64(T0, T1);
600 B2.m_simd = _mm_unpacklo_epi64(T2, T3);
601 B3.m_simd = _mm_unpackhi_epi64(T2, T3);
602#elif defined(BOTAN_SIMD_USE_ALTIVEC)
603 const __vector unsigned int T0 = vec_mergeh(B0.m_simd, B2.m_simd);
604 const __vector unsigned int T1 = vec_mergeh(B1.m_simd, B3.m_simd);
605 const __vector unsigned int T2 = vec_mergel(B0.m_simd, B2.m_simd);
606 const __vector unsigned int T3 = vec_mergel(B1.m_simd, B3.m_simd);
607
608 B0.m_simd = vec_mergeh(T0, T1);
609 B1.m_simd = vec_mergel(T0, T1);
610 B2.m_simd = vec_mergeh(T2, T3);
611 B3.m_simd = vec_mergel(T2, T3);
612
613#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM32)
614 const uint32x4x2_t T0 = vzipq_u32(B0.m_simd, B2.m_simd);
615 const uint32x4x2_t T1 = vzipq_u32(B1.m_simd, B3.m_simd);
616 const uint32x4x2_t O0 = vzipq_u32(T0.val[0], T1.val[0]);
617 const uint32x4x2_t O1 = vzipq_u32(T0.val[1], T1.val[1]);
618
619 B0.m_simd = O0.val[0];
620 B1.m_simd = O0.val[1];
621 B2.m_simd = O1.val[0];
622 B3.m_simd = O1.val[1];
623
624#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
625 const uint32x4_t T0 = vzip1q_u32(B0.m_simd, B2.m_simd);
626 const uint32x4_t T2 = vzip2q_u32(B0.m_simd, B2.m_simd);
627 const uint32x4_t T1 = vzip1q_u32(B1.m_simd, B3.m_simd);
628 const uint32x4_t T3 = vzip2q_u32(B1.m_simd, B3.m_simd);
629
630 B0.m_simd = vzip1q_u32(T0, T1);
631 B1.m_simd = vzip2q_u32(T0, T1);
632 B2.m_simd = vzip1q_u32(T2, T3);
633 B3.m_simd = vzip2q_u32(T2, T3);
634#endif
635 }
636
637 static inline SIMD_4x32 choose(const SIMD_4x32& mask, const SIMD_4x32& a, const SIMD_4x32& b) noexcept
638 {
639#if defined(BOTAN_SIMD_USE_ALTIVEC)
640 return SIMD_4x32(vec_sel(b.raw(), a.raw(), mask.raw()));
641#elif defined(BOTAN_SIMD_USE_NEON)
642 return SIMD_4x32(vbslq_u32(mask.raw(), a.raw(), b.raw()));
643#else
644 return (mask & a) ^ mask.andc(b);
645#endif
646 }
647
648 static inline SIMD_4x32 majority(const SIMD_4x32& x, const SIMD_4x32& y, const SIMD_4x32& z) noexcept
649 {
650 return SIMD_4x32::choose(x ^ y, z, y);
651 }
652
653 native_simd_type raw() const noexcept { return m_simd; }
654
655 explicit SIMD_4x32(native_simd_type x) noexcept : m_simd(x) {}
656 private:
657 native_simd_type m_simd;
658 };
659
660template<size_t R>
662 {
663 return input.rotl<R>();
664 }
665
666template<size_t R>
668 {
669 return input.rotr<R>();
670 }
671
672// For Serpent:
673template<size_t S>
675 {
676 return input.shl<S>();
677 }
678
679}
680
681#endif
static SIMD_4x64 y
static bool is_little_endian()
Definition: cpuid.h:60
static bool is_big_endian()
Definition: cpuid.h:71
static SIMD_4x32 load_be(const void *in) noexcept
Definition: simd_32.h:179
SIMD_4x32 andc(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:500
void store_le(uint8_t out[]) const noexcept
Definition: simd_32.h:213
SIMD_4x32(SIMD_4x32 &&other)=default
SIMD_4x32 & operator=(SIMD_4x32 &&other)=default
SIMD_4x32 operator|(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:381
SIMD_4x32 operator^(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:371
SIMD_4x32(native_simd_type x) noexcept
Definition: simd_32.h:655
static void transpose(SIMD_4x32 &B0, SIMD_4x32 &B1, SIMD_4x32 &B2, SIMD_4x32 &B3) noexcept
Definition: simd_32.h:589
SIMD_4x32 bswap() const noexcept
Definition: simd_32.h:519
SIMD_4x32 operator+(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:351
SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept
Definition: simd_32.h:116
native_simd_type raw() const noexcept
Definition: simd_32.h:653
SIMD_4x32() noexcept
Definition: simd_32.h:87
void store_le(uint32_t out[4]) const noexcept
Definition: simd_32.h:195
static SIMD_4x32 load_le(const void *in) noexcept
Definition: simd_32.h:162
SIMD_4x32 sigma1() const noexcept
Definition: simd_32.h:288
void store_be(uint8_t out[]) const noexcept
Definition: simd_32.h:243
SIMD_4x32(const SIMD_4x32 &other)=default
~SIMD_4x32()=default
void operator^=(uint32_t other) noexcept
Definition: simd_32.h:431
void operator^=(const SIMD_4x32 &other) noexcept
Definition: simd_32.h:420
void operator+=(const SIMD_4x32 &other) noexcept
Definition: simd_32.h:398
SIMD_4x32 operator~() const noexcept
Definition: simd_32.h:488
void store_le(uint64_t out[2]) const noexcept
Definition: simd_32.h:205
void operator|=(const SIMD_4x32 &other) noexcept
Definition: simd_32.h:436
SIMD_4x32 shift_elems_left() const noexcept
Definition: simd_32.h:545
void store_be(uint32_t out[4]) const noexcept
Definition: simd_32.h:200
SIMD_4x32(const uint32_t B[4]) noexcept
Definition: simd_32.h:101
SIMD_4x32 sigma0() const noexcept
Definition: simd_32.h:273
SIMD_4x32 operator-(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:361
SIMD_4x32 shr() const noexcept
Definition: simd_32.h:474
static SIMD_4x32 splat_u8(uint8_t B) noexcept
Definition: simd_32.h:147
SIMD_4x32 rotr() const noexcept
Definition: simd_32.h:343
SIMD_4x32 shl() const noexcept
Definition: simd_32.h:459
SIMD_4x32 shift_elems_right() const noexcept
Definition: simd_32.h:566
SIMD_4x32 rotl() const noexcept
Definition: simd_32.h:304
void operator&=(const SIMD_4x32 &other) noexcept
Definition: simd_32.h:447
SIMD_4x32 operator&(const SIMD_4x32 &other) const noexcept
Definition: simd_32.h:391
SIMD_4x32 & operator=(const SIMD_4x32 &other)=default
static SIMD_4x32 choose(const SIMD_4x32 &mask, const SIMD_4x32 &a, const SIMD_4x32 &b) noexcept
Definition: simd_32.h:637
void operator-=(const SIMD_4x32 &other) noexcept
Definition: simd_32.h:409
static SIMD_4x32 majority(const SIMD_4x32 &x, const SIMD_4x32 &y, const SIMD_4x32 &z) noexcept
Definition: simd_32.h:648
static SIMD_4x32 splat(uint32_t B) noexcept
Definition: simd_32.h:133
int(* final)(unsigned char *, CTX *)
FE_25519 T
Definition: ge.cpp:36
Definition: alg_id.cpp:12
constexpr void store_le(uint16_t in, uint8_t out[2])
Definition: loadstor.h:465
constexpr T rotr(T input)
Definition: rotate.h:33
constexpr void bswap_4(T x[4])
Definition: bswap.h:72
constexpr uint32_t make_uint32(uint8_t i0, uint8_t i1, uint8_t i2, uint8_t i3)
Definition: loadstor.h:78
constexpr T load_le(const uint8_t in[], size_t off)
Definition: loadstor.h:134
constexpr void store_be(uint16_t in, uint8_t out[2])
Definition: loadstor.h:449
SIMD_4x32 shl(SIMD_4x32 input)
Definition: simd_32.h:674
constexpr T rotl(T input)
Definition: rotate.h:21
constexpr T load_be(const uint8_t in[], size_t off)
Definition: loadstor.h:118