8#ifndef BOTAN_SIMD_32_H_
9#define BOTAN_SIMD_32_H_
11#include <botan/types.h>
13#if defined(BOTAN_TARGET_SUPPORTS_SSE2)
14 #include <emmintrin.h>
15 #define BOTAN_SIMD_USE_SSE2
17#elif defined(BOTAN_TARGET_SUPPORTS_ALTIVEC)
18 #include <botan/internal/bswap.h>
19 #include <botan/internal/loadstor.h>
23 #define BOTAN_SIMD_USE_ALTIVEC
25#elif defined(BOTAN_TARGET_SUPPORTS_NEON)
26 #include <botan/internal/cpuid.h>
28 #define BOTAN_SIMD_USE_NEON
31 #error "No SIMD instruction set enabled"
34#if defined(BOTAN_SIMD_USE_SSE2)
35 #define BOTAN_SIMD_ISA "sse2"
36 #define BOTAN_VPERM_ISA "ssse3"
37 #define BOTAN_CLMUL_ISA "pclmul"
38#elif defined(BOTAN_SIMD_USE_NEON)
39 #if defined(BOTAN_TARGET_ARCH_IS_ARM64)
40 #define BOTAN_SIMD_ISA "+simd"
41 #define BOTAN_CLMUL_ISA "+crypto"
43 #define BOTAN_SIMD_ISA "fpu=neon"
45 #define BOTAN_VPERM_ISA BOTAN_SIMD_ISA
46#elif defined(BOTAN_SIMD_USE_ALTIVEC)
47 #define BOTAN_SIMD_ISA "altivec"
48 #define BOTAN_VPERM_ISA "altivec"
49 #define BOTAN_CLMUL_ISA "crypto"
54#if defined(BOTAN_SIMD_USE_SSE2)
55 using native_simd_type = __m128i;
56#elif defined(BOTAN_SIMD_USE_ALTIVEC)
57 using native_simd_type = __vector
unsigned int;
58#elif defined(BOTAN_SIMD_USE_NEON)
59 using native_simd_type = uint32x4_t;
89#if defined(BOTAN_SIMD_USE_SSE2)
90 m_simd = _mm_setzero_si128();
91#elif defined(BOTAN_SIMD_USE_ALTIVEC)
92 m_simd = vec_splat_u32(0);
93#elif defined(BOTAN_SIMD_USE_NEON)
94 m_simd = vdupq_n_u32(0);
103#if defined(BOTAN_SIMD_USE_SSE2)
104 m_simd = _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(B));
105#elif defined(BOTAN_SIMD_USE_ALTIVEC)
106 __vector
unsigned int val = { B[0], B[1], B[2], B[3]};
108#elif defined(BOTAN_SIMD_USE_NEON)
109 m_simd = vld1q_u32(B);
116 SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3)
noexcept
118#if defined(BOTAN_SIMD_USE_SSE2)
119 m_simd = _mm_set_epi32(B3, B2, B1, B0);
120#elif defined(BOTAN_SIMD_USE_ALTIVEC)
121 __vector
unsigned int val = {B0, B1, B2, B3};
123#elif defined(BOTAN_SIMD_USE_NEON)
125 const uint32_t B[4] = { B0, B1, B2, B3 };
126 m_simd = vld1q_u32(B);
135#if defined(BOTAN_SIMD_USE_SSE2)
137#elif defined(BOTAN_SIMD_USE_NEON)
149#if defined(BOTAN_SIMD_USE_SSE2)
151#elif defined(BOTAN_SIMD_USE_NEON)
152 return SIMD_4x32(vreinterpretq_u32_u8(vdupq_n_u8(B)));
164#if defined(BOTAN_SIMD_USE_SSE2)
165 return SIMD_4x32(_mm_loadu_si128(
reinterpret_cast<const __m128i*
>(in)));
166#elif defined(BOTAN_SIMD_USE_ALTIVEC)
170#elif defined(BOTAN_SIMD_USE_NEON)
171 SIMD_4x32 l(vld1q_u32(
static_cast<const uint32_t*
>(in)));
181#if defined(BOTAN_SIMD_USE_SSE2)
184#elif defined(BOTAN_SIMD_USE_ALTIVEC)
189#elif defined(BOTAN_SIMD_USE_NEON)
190 SIMD_4x32 l(vld1q_u32(
static_cast<const uint32_t*
>(in)));
197 this->
store_le(
reinterpret_cast<uint8_t*
>(out));
202 this->
store_be(
reinterpret_cast<uint8_t*
>(out));
207 this->
store_le(
reinterpret_cast<uint8_t*
>(out));
215#if defined(BOTAN_SIMD_USE_SSE2)
217 _mm_storeu_si128(
reinterpret_cast<__m128i*
>(out),
raw());
219#elif defined(BOTAN_SIMD_USE_ALTIVEC)
222 __vector
unsigned int V;
228#elif defined(BOTAN_SIMD_USE_NEON)
231 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
235 vst1q_u8(out, vreinterpretq_u8_u32(
bswap().m_simd));
245#if defined(BOTAN_SIMD_USE_SSE2)
249#elif defined(BOTAN_SIMD_USE_ALTIVEC)
252 __vector
unsigned int V;
258#elif defined(BOTAN_SIMD_USE_NEON)
261 vst1q_u8(out, vreinterpretq_u8_u32(
bswap().m_simd));
265 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
275#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
276 return SIMD_4x32(__builtin_crypto_vshasigmaw(
raw(), 1, 0));
281 return (rot1 ^ rot2 ^ rot3);
290#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
291 return SIMD_4x32(__builtin_crypto_vshasigmaw(
raw(), 1, 0xF));
296 return (rot1 ^ rot2 ^ rot3);
305 requires (ROT > 0 && ROT < 32)
308#if defined(BOTAN_SIMD_USE_SSE2)
310 return SIMD_4x32(_mm_or_si128(_mm_slli_epi32(m_simd,
static_cast<int>(ROT)),
311 _mm_srli_epi32(m_simd,
static_cast<int>(32-ROT))));
313#elif defined(BOTAN_SIMD_USE_ALTIVEC)
315 const unsigned int r =
static_cast<unsigned int>(ROT);
316 __vector
unsigned int rot = {r, r, r, r};
319#elif defined(BOTAN_SIMD_USE_NEON)
321#if defined(BOTAN_TARGET_ARCH_IS_ARM64)
323 if constexpr(ROT == 8)
325 const uint8_t maskb[16] = { 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 };
326 const uint8x16_t mask = vld1q_u8(maskb);
327 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(m_simd), mask)));
329 else if constexpr(ROT == 16)
331 return SIMD_4x32(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(m_simd))));
334 return SIMD_4x32(vorrq_u32(vshlq_n_u32(m_simd,
static_cast<int>(ROT)),
335 vshrq_n_u32(m_simd,
static_cast<int>(32-ROT))));
345 return this->
rotl<32-ROT>();
400#if defined(BOTAN_SIMD_USE_SSE2)
401 m_simd = _mm_add_epi32(m_simd, other.m_simd);
402#elif defined(BOTAN_SIMD_USE_ALTIVEC)
403 m_simd = vec_add(m_simd, other.m_simd);
404#elif defined(BOTAN_SIMD_USE_NEON)
405 m_simd = vaddq_u32(m_simd, other.m_simd);
411#if defined(BOTAN_SIMD_USE_SSE2)
412 m_simd = _mm_sub_epi32(m_simd, other.m_simd);
413#elif defined(BOTAN_SIMD_USE_ALTIVEC)
414 m_simd = vec_sub(m_simd, other.m_simd);
415#elif defined(BOTAN_SIMD_USE_NEON)
416 m_simd = vsubq_u32(m_simd, other.m_simd);
422#if defined(BOTAN_SIMD_USE_SSE2)
423 m_simd = _mm_xor_si128(m_simd, other.m_simd);
424#elif defined(BOTAN_SIMD_USE_ALTIVEC)
425 m_simd = vec_xor(m_simd, other.m_simd);
426#elif defined(BOTAN_SIMD_USE_NEON)
427 m_simd = veorq_u32(m_simd, other.m_simd);
438#if defined(BOTAN_SIMD_USE_SSE2)
439 m_simd = _mm_or_si128(m_simd, other.m_simd);
440#elif defined(BOTAN_SIMD_USE_ALTIVEC)
441 m_simd = vec_or(m_simd, other.m_simd);
442#elif defined(BOTAN_SIMD_USE_NEON)
443 m_simd = vorrq_u32(m_simd, other.m_simd);
449#if defined(BOTAN_SIMD_USE_SSE2)
450 m_simd = _mm_and_si128(m_simd, other.m_simd);
451#elif defined(BOTAN_SIMD_USE_ALTIVEC)
452 m_simd = vec_and(m_simd, other.m_simd);
453#elif defined(BOTAN_SIMD_USE_NEON)
454 m_simd = vandq_u32(m_simd, other.m_simd);
460 requires (SHIFT > 0 && SHIFT < 32)
462#if defined(BOTAN_SIMD_USE_SSE2)
463 return SIMD_4x32(_mm_slli_epi32(m_simd, SHIFT));
465#elif defined(BOTAN_SIMD_USE_ALTIVEC)
466 const unsigned int s =
static_cast<unsigned int>(SHIFT);
467 const __vector
unsigned int shifts = {s, s, s, s};
468 return SIMD_4x32(vec_sl(m_simd, shifts));
469#elif defined(BOTAN_SIMD_USE_NEON)
470 return SIMD_4x32(vshlq_n_u32(m_simd, SHIFT));
476#if defined(BOTAN_SIMD_USE_SSE2)
477 return SIMD_4x32(_mm_srli_epi32(m_simd, SHIFT));
479#elif defined(BOTAN_SIMD_USE_ALTIVEC)
480 const unsigned int s =
static_cast<unsigned int>(SHIFT);
481 const __vector
unsigned int shifts = {s, s, s, s};
482 return SIMD_4x32(vec_sr(m_simd, shifts));
483#elif defined(BOTAN_SIMD_USE_NEON)
484 return SIMD_4x32(vshrq_n_u32(m_simd, SHIFT));
490#if defined(BOTAN_SIMD_USE_SSE2)
491 return SIMD_4x32(_mm_xor_si128(m_simd, _mm_set1_epi32(0xFFFFFFFF)));
492#elif defined(BOTAN_SIMD_USE_ALTIVEC)
493 return SIMD_4x32(vec_nor(m_simd, m_simd));
494#elif defined(BOTAN_SIMD_USE_NEON)
502#if defined(BOTAN_SIMD_USE_SSE2)
503 return SIMD_4x32(_mm_andnot_si128(m_simd, other.m_simd));
504#elif defined(BOTAN_SIMD_USE_ALTIVEC)
509 return SIMD_4x32(vec_andc(other.m_simd, m_simd));
510#elif defined(BOTAN_SIMD_USE_NEON)
512 return SIMD_4x32(vbicq_u32(other.m_simd, m_simd));
521#if defined(BOTAN_SIMD_USE_SSE2)
524 T = _mm_shufflehi_epi16(
T, _MM_SHUFFLE(2, 3, 0, 1));
525 T = _mm_shufflelo_epi16(
T, _MM_SHUFFLE(2, 3, 0, 1));
526 return SIMD_4x32(_mm_or_si128(_mm_srli_epi16(
T, 8), _mm_slli_epi16(
T, 8)));
528#elif defined(BOTAN_SIMD_USE_ALTIVEC)
531 __vector
unsigned int V;
537 return SIMD_4x32(vec.R[0], vec.R[1], vec.R[2], vec.R[3]);
539#elif defined(BOTAN_SIMD_USE_NEON)
540 return SIMD_4x32(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(m_simd))));
548#if defined(BOTAN_SIMD_USE_SSE2)
550#elif defined(BOTAN_SIMD_USE_NEON)
551 return SIMD_4x32(vextq_u32(vdupq_n_u32(0),
raw(), 4-I));
552#elif defined(BOTAN_SIMD_USE_ALTIVEC)
553 const __vector
unsigned int zero = vec_splat_u32(0);
555 const __vector
unsigned char shuf[3] = {
556 { 16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 },
557 { 16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7 },
558 { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3 },
569#if defined(BOTAN_SIMD_USE_SSE2)
571#elif defined(BOTAN_SIMD_USE_NEON)
573#elif defined(BOTAN_SIMD_USE_ALTIVEC)
574 const __vector
unsigned int zero = vec_splat_u32(0);
576 const __vector
unsigned char shuf[3] = {
577 { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 },
578 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 },
579 { 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 },
592#if defined(BOTAN_SIMD_USE_SSE2)
593 const __m128i T0 = _mm_unpacklo_epi32(B0.m_simd, B1.m_simd);
594 const __m128i T1 = _mm_unpacklo_epi32(B2.m_simd, B3.m_simd);
595 const __m128i T2 = _mm_unpackhi_epi32(B0.m_simd, B1.m_simd);
596 const __m128i T3 = _mm_unpackhi_epi32(B2.m_simd, B3.m_simd);
598 B0.m_simd = _mm_unpacklo_epi64(T0, T1);
599 B1.m_simd = _mm_unpackhi_epi64(T0, T1);
600 B2.m_simd = _mm_unpacklo_epi64(T2, T3);
601 B3.m_simd = _mm_unpackhi_epi64(T2, T3);
602#elif defined(BOTAN_SIMD_USE_ALTIVEC)
603 const __vector
unsigned int T0 = vec_mergeh(B0.m_simd, B2.m_simd);
604 const __vector
unsigned int T1 = vec_mergeh(B1.m_simd, B3.m_simd);
605 const __vector
unsigned int T2 = vec_mergel(B0.m_simd, B2.m_simd);
606 const __vector
unsigned int T3 = vec_mergel(B1.m_simd, B3.m_simd);
608 B0.m_simd = vec_mergeh(T0, T1);
609 B1.m_simd = vec_mergel(T0, T1);
610 B2.m_simd = vec_mergeh(T2, T3);
611 B3.m_simd = vec_mergel(T2, T3);
613#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM32)
614 const uint32x4x2_t T0 = vzipq_u32(B0.m_simd, B2.m_simd);
615 const uint32x4x2_t T1 = vzipq_u32(B1.m_simd, B3.m_simd);
616 const uint32x4x2_t O0 = vzipq_u32(T0.val[0], T1.val[0]);
617 const uint32x4x2_t O1 = vzipq_u32(T0.val[1], T1.val[1]);
619 B0.m_simd = O0.val[0];
620 B1.m_simd = O0.val[1];
621 B2.m_simd = O1.val[0];
622 B3.m_simd = O1.val[1];
624#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
625 const uint32x4_t T0 = vzip1q_u32(B0.m_simd, B2.m_simd);
626 const uint32x4_t T2 = vzip2q_u32(B0.m_simd, B2.m_simd);
627 const uint32x4_t T1 = vzip1q_u32(B1.m_simd, B3.m_simd);
628 const uint32x4_t T3 = vzip2q_u32(B1.m_simd, B3.m_simd);
630 B0.m_simd = vzip1q_u32(T0, T1);
631 B1.m_simd = vzip2q_u32(T0, T1);
632 B2.m_simd = vzip1q_u32(T2, T3);
633 B3.m_simd = vzip2q_u32(T2, T3);
639#if defined(BOTAN_SIMD_USE_ALTIVEC)
640 return SIMD_4x32(vec_sel(b.raw(), a.raw(), mask.raw()));
641#elif defined(BOTAN_SIMD_USE_NEON)
642 return SIMD_4x32(vbslq_u32(mask.raw(), a.raw(), b.raw()));
644 return (mask & a) ^ mask.andc(b);
653 native_simd_type
raw() const noexcept {
return m_simd; }
655 explicit SIMD_4x32(native_simd_type x) noexcept : m_simd(x) {}
657 native_simd_type m_simd;
663 return input.
rotl<R>();
669 return input.
rotr<R>();
676 return input.
shl<S>();
static bool is_little_endian()
static bool is_big_endian()
static SIMD_4x32 load_be(const void *in) noexcept
SIMD_4x32 andc(const SIMD_4x32 &other) const noexcept
void store_le(uint8_t out[]) const noexcept
SIMD_4x32(SIMD_4x32 &&other)=default
SIMD_4x32 & operator=(SIMD_4x32 &&other)=default
SIMD_4x32 operator|(const SIMD_4x32 &other) const noexcept
SIMD_4x32 operator^(const SIMD_4x32 &other) const noexcept
SIMD_4x32(native_simd_type x) noexcept
static void transpose(SIMD_4x32 &B0, SIMD_4x32 &B1, SIMD_4x32 &B2, SIMD_4x32 &B3) noexcept
SIMD_4x32 bswap() const noexcept
SIMD_4x32 operator+(const SIMD_4x32 &other) const noexcept
SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept
native_simd_type raw() const noexcept
void store_le(uint32_t out[4]) const noexcept
static SIMD_4x32 load_le(const void *in) noexcept
SIMD_4x32 sigma1() const noexcept
void store_be(uint8_t out[]) const noexcept
SIMD_4x32(const SIMD_4x32 &other)=default
void operator^=(uint32_t other) noexcept
void operator^=(const SIMD_4x32 &other) noexcept
void operator+=(const SIMD_4x32 &other) noexcept
SIMD_4x32 operator~() const noexcept
void store_le(uint64_t out[2]) const noexcept
void operator|=(const SIMD_4x32 &other) noexcept
SIMD_4x32 shift_elems_left() const noexcept
void store_be(uint32_t out[4]) const noexcept
SIMD_4x32(const uint32_t B[4]) noexcept
SIMD_4x32 sigma0() const noexcept
SIMD_4x32 operator-(const SIMD_4x32 &other) const noexcept
SIMD_4x32 shr() const noexcept
static SIMD_4x32 splat_u8(uint8_t B) noexcept
SIMD_4x32 rotr() const noexcept
SIMD_4x32 shl() const noexcept
SIMD_4x32 shift_elems_right() const noexcept
SIMD_4x32 rotl() const noexcept
void operator&=(const SIMD_4x32 &other) noexcept
SIMD_4x32 operator&(const SIMD_4x32 &other) const noexcept
SIMD_4x32 & operator=(const SIMD_4x32 &other)=default
static SIMD_4x32 choose(const SIMD_4x32 &mask, const SIMD_4x32 &a, const SIMD_4x32 &b) noexcept
void operator-=(const SIMD_4x32 &other) noexcept
static SIMD_4x32 majority(const SIMD_4x32 &x, const SIMD_4x32 &y, const SIMD_4x32 &z) noexcept
static SIMD_4x32 splat(uint32_t B) noexcept
int(* final)(unsigned char *, CTX *)
constexpr void store_le(uint16_t in, uint8_t out[2])
constexpr T rotr(T input)
constexpr void bswap_4(T x[4])
constexpr uint32_t make_uint32(uint8_t i0, uint8_t i1, uint8_t i2, uint8_t i3)
constexpr T load_le(const uint8_t in[], size_t off)
constexpr void store_be(uint16_t in, uint8_t out[2])
SIMD_4x32 shl(SIMD_4x32 input)
constexpr T rotl(T input)
constexpr T load_be(const uint8_t in[], size_t off)