8#ifndef BOTAN_SIMD_32_H_
9#define BOTAN_SIMD_32_H_
11#include <botan/types.h>
13#if defined(BOTAN_TARGET_SUPPORTS_SSE2)
14 #include <emmintrin.h>
15 #define BOTAN_SIMD_USE_SSE2
17#elif defined(BOTAN_TARGET_SUPPORTS_ALTIVEC)
18 #include <botan/bswap.h>
19 #include <botan/loadstor.h>
23 #define BOTAN_SIMD_USE_ALTIVEC
25#elif defined(BOTAN_TARGET_SUPPORTS_NEON)
26 #include <botan/cpuid.h>
28 #define BOTAN_SIMD_USE_NEON
31 #error "No SIMD instruction set enabled"
34#if defined(BOTAN_SIMD_USE_SSE2)
35 #define BOTAN_SIMD_ISA "sse2"
36 #define BOTAN_VPERM_ISA "ssse3"
37 #define BOTAN_CLMUL_ISA "pclmul"
38#elif defined(BOTAN_SIMD_USE_NEON)
39 #if defined(BOTAN_TARGET_ARCH_IS_ARM64)
40 #define BOTAN_SIMD_ISA "+simd"
41 #define BOTAN_CLMUL_ISA "+crypto"
43 #define BOTAN_SIMD_ISA "fpu=neon"
45 #define BOTAN_VPERM_ISA BOTAN_SIMD_ISA
46#elif defined(BOTAN_SIMD_USE_ALTIVEC)
47 #define BOTAN_SIMD_ISA "altivec"
48 #define BOTAN_VPERM_ISA "altivec"
49 #define BOTAN_CLMUL_ISA "crypto"
54#if defined(BOTAN_SIMD_USE_SSE2)
55 typedef __m128i native_simd_type;
56#elif defined(BOTAN_SIMD_USE_ALTIVEC)
57 typedef __vector
unsigned int native_simd_type;
58#elif defined(BOTAN_SIMD_USE_NEON)
59 typedef uint32x4_t native_simd_type;
87#if defined(BOTAN_SIMD_USE_SSE2)
88 m_simd = _mm_setzero_si128();
89#elif defined(BOTAN_SIMD_USE_ALTIVEC)
90 m_simd = vec_splat_u32(0);
91#elif defined(BOTAN_SIMD_USE_NEON)
92 m_simd = vdupq_n_u32(0);
101#if defined(BOTAN_SIMD_USE_SSE2)
102 m_simd = _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(B));
103#elif defined(BOTAN_SIMD_USE_ALTIVEC)
104 __vector
unsigned int val = { B[0], B[1], B[2], B[3]};
106#elif defined(BOTAN_SIMD_USE_NEON)
107 m_simd = vld1q_u32(B);
114 SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3)
116#if defined(BOTAN_SIMD_USE_SSE2)
117 m_simd = _mm_set_epi32(B3, B2, B1, B0);
118#elif defined(BOTAN_SIMD_USE_ALTIVEC)
119 __vector
unsigned int val = {B0, B1, B2, B3};
121#elif defined(BOTAN_SIMD_USE_NEON)
123 const uint32_t B[4] = { B0, B1, B2, B3 };
124 m_simd = vld1q_u32(B);
133#if defined(BOTAN_SIMD_USE_SSE2)
135#elif defined(BOTAN_SIMD_USE_NEON)
147#if defined(BOTAN_SIMD_USE_SSE2)
149#elif defined(BOTAN_SIMD_USE_NEON)
150 return SIMD_4x32(vreinterpretq_u32_u8(vdupq_n_u8(B)));
162#if defined(BOTAN_SIMD_USE_SSE2)
163 return SIMD_4x32(_mm_loadu_si128(
reinterpret_cast<const __m128i*
>(in)));
164#elif defined(BOTAN_SIMD_USE_ALTIVEC)
168#elif defined(BOTAN_SIMD_USE_NEON)
169 SIMD_4x32 l(vld1q_u32(
static_cast<const uint32_t*
>(in)));
179#if defined(BOTAN_SIMD_USE_SSE2)
182#elif defined(BOTAN_SIMD_USE_ALTIVEC)
187#elif defined(BOTAN_SIMD_USE_NEON)
188 SIMD_4x32 l(vld1q_u32(
static_cast<const uint32_t*
>(in)));
195 this->
store_le(
reinterpret_cast<uint8_t*
>(out));
200 this->
store_le(
reinterpret_cast<uint8_t*
>(out));
208#if defined(BOTAN_SIMD_USE_SSE2)
210 _mm_storeu_si128(
reinterpret_cast<__m128i*
>(out),
raw());
212#elif defined(BOTAN_SIMD_USE_ALTIVEC)
215 __vector
unsigned int V;
221#elif defined(BOTAN_SIMD_USE_NEON)
224 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
228 vst1q_u8(out, vreinterpretq_u8_u32(
bswap().m_simd));
238#if defined(BOTAN_SIMD_USE_SSE2)
242#elif defined(BOTAN_SIMD_USE_ALTIVEC)
245 __vector
unsigned int V;
251#elif defined(BOTAN_SIMD_USE_NEON)
254 vst1q_u8(out, vreinterpretq_u8_u32(
bswap().m_simd));
258 vst1q_u8(out, vreinterpretq_u8_u32(m_simd));
267 template<
size_t ROT1,
size_t ROT2,
size_t ROT3>
270 const SIMD_4x32 rot1 = this->rotr<ROT1>();
271 const SIMD_4x32 rot2 = this->rotr<ROT2>();
272 const SIMD_4x32 rot3 = this->rotr<ROT3>();
273 return (rot1 ^ rot2 ^ rot3);
282 static_assert(ROT > 0 && ROT < 32,
"Invalid rotation constant");
284#if defined(BOTAN_SIMD_USE_SSE2)
286 return SIMD_4x32(_mm_or_si128(_mm_slli_epi32(m_simd,
static_cast<int>(ROT)),
287 _mm_srli_epi32(m_simd,
static_cast<int>(32-ROT))));
289#elif defined(BOTAN_SIMD_USE_ALTIVEC)
291 const unsigned int r =
static_cast<unsigned int>(ROT);
292 __vector
unsigned int rot = {r, r, r, r};
295#elif defined(BOTAN_SIMD_USE_NEON)
297#if defined(BOTAN_TARGET_ARCH_IS_ARM64)
301 const uint8_t maskb[16] = { 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 };
302 const uint8x16_t mask = vld1q_u8(maskb);
303 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(m_simd), mask)));
307 return SIMD_4x32(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(m_simd))));
310 return SIMD_4x32(vorrq_u32(vshlq_n_u32(m_simd,
static_cast<int>(ROT)),
311 vshrq_n_u32(m_simd,
static_cast<int>(32-ROT))));
321 return this->
rotl<32-ROT>();
376#if defined(BOTAN_SIMD_USE_SSE2)
377 m_simd = _mm_add_epi32(m_simd, other.m_simd);
378#elif defined(BOTAN_SIMD_USE_ALTIVEC)
379 m_simd = vec_add(m_simd, other.m_simd);
380#elif defined(BOTAN_SIMD_USE_NEON)
381 m_simd = vaddq_u32(m_simd, other.m_simd);
387#if defined(BOTAN_SIMD_USE_SSE2)
388 m_simd = _mm_sub_epi32(m_simd, other.m_simd);
389#elif defined(BOTAN_SIMD_USE_ALTIVEC)
390 m_simd = vec_sub(m_simd, other.m_simd);
391#elif defined(BOTAN_SIMD_USE_NEON)
392 m_simd = vsubq_u32(m_simd, other.m_simd);
398#if defined(BOTAN_SIMD_USE_SSE2)
399 m_simd = _mm_xor_si128(m_simd, other.m_simd);
401#elif defined(BOTAN_SIMD_USE_ALTIVEC)
402 m_simd = vec_xor(m_simd, other.m_simd);
403#elif defined(BOTAN_SIMD_USE_NEON)
404 m_simd = veorq_u32(m_simd, other.m_simd);
410#if defined(BOTAN_SIMD_USE_SSE2)
411 m_simd = _mm_or_si128(m_simd, other.m_simd);
412#elif defined(BOTAN_SIMD_USE_ALTIVEC)
413 m_simd = vec_or(m_simd, other.m_simd);
414#elif defined(BOTAN_SIMD_USE_NEON)
415 m_simd = vorrq_u32(m_simd, other.m_simd);
421#if defined(BOTAN_SIMD_USE_SSE2)
422 m_simd = _mm_and_si128(m_simd, other.m_simd);
423#elif defined(BOTAN_SIMD_USE_ALTIVEC)
424 m_simd = vec_and(m_simd, other.m_simd);
425#elif defined(BOTAN_SIMD_USE_NEON)
426 m_simd = vandq_u32(m_simd, other.m_simd);
433 static_assert(SHIFT > 0 && SHIFT <= 31,
"Invalid shift count");
435#if defined(BOTAN_SIMD_USE_SSE2)
436 return SIMD_4x32(_mm_slli_epi32(m_simd, SHIFT));
438#elif defined(BOTAN_SIMD_USE_ALTIVEC)
439 const unsigned int s =
static_cast<unsigned int>(SHIFT);
440 const __vector
unsigned int shifts = {s, s, s, s};
441 return SIMD_4x32(vec_sl(m_simd, shifts));
442#elif defined(BOTAN_SIMD_USE_NEON)
443 return SIMD_4x32(vshlq_n_u32(m_simd, SHIFT));
449#if defined(BOTAN_SIMD_USE_SSE2)
450 return SIMD_4x32(_mm_srli_epi32(m_simd, SHIFT));
452#elif defined(BOTAN_SIMD_USE_ALTIVEC)
453 const unsigned int s =
static_cast<unsigned int>(SHIFT);
454 const __vector
unsigned int shifts = {s, s, s, s};
455 return SIMD_4x32(vec_sr(m_simd, shifts));
456#elif defined(BOTAN_SIMD_USE_NEON)
457 return SIMD_4x32(vshrq_n_u32(m_simd, SHIFT));
463#if defined(BOTAN_SIMD_USE_SSE2)
464 return SIMD_4x32(_mm_xor_si128(m_simd, _mm_set1_epi32(0xFFFFFFFF)));
465#elif defined(BOTAN_SIMD_USE_ALTIVEC)
466 return SIMD_4x32(vec_nor(m_simd, m_simd));
467#elif defined(BOTAN_SIMD_USE_NEON)
475#if defined(BOTAN_SIMD_USE_SSE2)
476 return SIMD_4x32(_mm_andnot_si128(m_simd, other.m_simd));
477#elif defined(BOTAN_SIMD_USE_ALTIVEC)
482 return SIMD_4x32(vec_andc(other.m_simd, m_simd));
483#elif defined(BOTAN_SIMD_USE_NEON)
485 return SIMD_4x32(vbicq_u32(other.m_simd, m_simd));
494#if defined(BOTAN_SIMD_USE_SSE2)
497 T = _mm_shufflehi_epi16(
T, _MM_SHUFFLE(2, 3, 0, 1));
498 T = _mm_shufflelo_epi16(
T, _MM_SHUFFLE(2, 3, 0, 1));
499 return SIMD_4x32(_mm_or_si128(_mm_srli_epi16(
T, 8), _mm_slli_epi16(
T, 8)));
501#elif defined(BOTAN_SIMD_USE_ALTIVEC)
504 __vector
unsigned int V;
510 return SIMD_4x32(vec.R[0], vec.R[1], vec.R[2], vec.R[3]);
512#elif defined(BOTAN_SIMD_USE_NEON)
513 return SIMD_4x32(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(m_simd))));
520 static_assert(I <= 3,
"Invalid shift count");
522#if defined(BOTAN_SIMD_USE_SSE2)
524#elif defined(BOTAN_SIMD_USE_NEON)
525 return SIMD_4x32(vextq_u32(vdupq_n_u32(0),
raw(), 4-I));
526#elif defined(BOTAN_SIMD_USE_ALTIVEC)
527 const __vector
unsigned int zero = vec_splat_u32(0);
529 const __vector
unsigned char shuf[3] = {
530 { 16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 },
531 { 16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7 },
532 { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3 },
542 static_assert(I <= 3,
"Invalid shift count");
544#if defined(BOTAN_SIMD_USE_SSE2)
546#elif defined(BOTAN_SIMD_USE_NEON)
548#elif defined(BOTAN_SIMD_USE_ALTIVEC)
549 const __vector
unsigned int zero = vec_splat_u32(0);
551 const __vector
unsigned char shuf[3] = {
552 { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 },
553 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 },
554 { 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 },
567#if defined(BOTAN_SIMD_USE_SSE2)
568 const __m128i T0 = _mm_unpacklo_epi32(B0.m_simd, B1.m_simd);
569 const __m128i T1 = _mm_unpacklo_epi32(B2.m_simd, B3.m_simd);
570 const __m128i T2 = _mm_unpackhi_epi32(B0.m_simd, B1.m_simd);
571 const __m128i T3 = _mm_unpackhi_epi32(B2.m_simd, B3.m_simd);
573 B0.m_simd = _mm_unpacklo_epi64(T0, T1);
574 B1.m_simd = _mm_unpackhi_epi64(T0, T1);
575 B2.m_simd = _mm_unpacklo_epi64(T2, T3);
576 B3.m_simd = _mm_unpackhi_epi64(T2, T3);
577#elif defined(BOTAN_SIMD_USE_ALTIVEC)
578 const __vector
unsigned int T0 = vec_mergeh(B0.m_simd, B2.m_simd);
579 const __vector
unsigned int T1 = vec_mergeh(B1.m_simd, B3.m_simd);
580 const __vector
unsigned int T2 = vec_mergel(B0.m_simd, B2.m_simd);
581 const __vector
unsigned int T3 = vec_mergel(B1.m_simd, B3.m_simd);
583 B0.m_simd = vec_mergeh(T0, T1);
584 B1.m_simd = vec_mergel(T0, T1);
585 B2.m_simd = vec_mergeh(T2, T3);
586 B3.m_simd = vec_mergel(T2, T3);
588#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM32)
589 const uint32x4x2_t T0 = vzipq_u32(B0.m_simd, B2.m_simd);
590 const uint32x4x2_t T1 = vzipq_u32(B1.m_simd, B3.m_simd);
591 const uint32x4x2_t O0 = vzipq_u32(T0.val[0], T1.val[0]);
592 const uint32x4x2_t O1 = vzipq_u32(T0.val[1], T1.val[1]);
594 B0.m_simd = O0.val[0];
595 B1.m_simd = O0.val[1];
596 B2.m_simd = O1.val[0];
597 B3.m_simd = O1.val[1];
599#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
600 const uint32x4_t T0 = vzip1q_u32(B0.m_simd, B2.m_simd);
601 const uint32x4_t T2 = vzip2q_u32(B0.m_simd, B2.m_simd);
602 const uint32x4_t T1 = vzip1q_u32(B1.m_simd, B3.m_simd);
603 const uint32x4_t T3 = vzip2q_u32(B1.m_simd, B3.m_simd);
605 B0.m_simd = vzip1q_u32(T0, T1);
606 B1.m_simd = vzip2q_u32(T0, T1);
607 B2.m_simd = vzip1q_u32(T2, T3);
608 B3.m_simd = vzip2q_u32(T2, T3);
616 native_simd_type m_simd;
static bool is_little_endian()
static bool is_big_endian()
native_simd_type raw() const BOTAN_FUNC_ISA(BOTAN_SIMD_ISA)
SIMD_4x32(SIMD_4x32 &&other)=default
void operator-=(const SIMD_4x32 &other)
void operator+=(const SIMD_4x32 &other)
SIMD_4x32 & operator=(SIMD_4x32 &&other)=default
static SIMD_4x32 splat(uint32_t B)
void store_le(uint8_t out[]) const
SIMD_4x32 operator&(const SIMD_4x32 &other) const
void store_le(uint32_t out[4]) const
SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3)
void operator&=(const SIMD_4x32 &other)
SIMD_4x32(const SIMD_4x32 &other)=default
SIMD_4x32 operator|(const SIMD_4x32 &other) const
SIMD_4x32 shift_elems_left() const
static SIMD_4x32 splat_u8(uint8_t B)
SIMD_4x32 andc(const SIMD_4x32 &other) const
static void transpose(SIMD_4x32 &B0, SIMD_4x32 &B1, SIMD_4x32 &B2, SIMD_4x32 &B3)
SIMD_4x32(const uint32_t B[4])
static SIMD_4x32 load_be(const void *in)
void store_le(uint64_t out[2]) const
SIMD_4x32 operator-(const SIMD_4x32 &other) const
void store_be(uint8_t out[]) const
void operator|=(const SIMD_4x32 &other)
SIMD_4x32 & operator=(const SIMD_4x32 &other)=default
SIMD_4x32 operator^(const SIMD_4x32 &other) const
SIMD_4x32 operator+(const SIMD_4x32 &other) const
SIMD_4x32 operator~() const
void operator^=(const SIMD_4x32 &other)
static SIMD_4x32 load_le(const void *in)
SIMD_4x32(native_simd_type x)
SIMD_4x32 shift_elems_right() const
int(* final)(unsigned char *, CTX *)
#define BOTAN_FUNC_ISA(isa)
#define BOTAN_IF_CONSTEXPR
void store_be(uint16_t in, uint8_t out[2])
T load_be(const uint8_t in[], size_t off)
constexpr uint32_t make_uint32(uint8_t i0, uint8_t i1, uint8_t i2, uint8_t i3)
T load_le(const uint8_t in[], size_t off)
void store_le(uint16_t in, uint8_t out[2])