7#ifndef BOTAN_SIMD_8X32_H_
8#define BOTAN_SIMD_8X32_H_
10#include <botan/compiler.h>
11#include <botan/types.h>
12#include <botan/internal/isa_extn.h>
29 BOTAN_FN_ISA_SIMD_8X32
32 BOTAN_FN_ISA_SIMD_8X32
35 m_avx2 = _mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(B));
38 BOTAN_FN_ISA_SIMD_8X32
46 uint32_t B7)
noexcept {
48 m_avx2 = _mm256_set_epi32(B7, B6, B5, B4, B3, B2, B1, B0);
51 BOTAN_FN_ISA_SIMD_8X32
52 explicit SIMD_8x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3)
noexcept {
54 m_avx2 = _mm256_set_epi32(B3, B2, B1, B0, B3, B2, B1, B0);
57 BOTAN_FN_ISA_SIMD_8X32
60 BOTAN_FN_ISA_SIMD_8X32
62 return SIMD_8x32(_mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(in)));
65 BOTAN_FN_ISA_SIMD_8X32
67 return SIMD_8x32(_mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(in)));
70 BOTAN_FN_ISA_SIMD_8X32
72 return SIMD_8x32(_mm256_broadcastsi128_si256(_mm_loadu_si128(
reinterpret_cast<const __m128i*
>(in))));
75 BOTAN_FN_ISA_SIMD_8X32
77 return SIMD_8x32(_mm256_broadcastsi128_si256(_mm_loadu_si128(
reinterpret_cast<const __m128i*
>(in))));
80 BOTAN_FN_ISA_SIMD_8X32
87 template <
size_t Str
ide>
89 load_be(
const uint8_t*
const* ptrs,
size_t i,
size_t n,
size_t offset = 0) {
90 return load_be(ptrs[i] + Stride * n + offset);
93 BOTAN_FN_ISA_SIMD_8X32
94 void store_le(uint8_t out[])
const noexcept { _mm256_storeu_si256(
reinterpret_cast<__m256i*
>(out), m_avx2); }
96 BOTAN_FN_ISA_SIMD_8X32
97 void store_le(uint32_t out[])
const noexcept { _mm256_storeu_si256(
reinterpret_cast<__m256i*
>(out), m_avx2); }
99 BOTAN_FN_ISA_SIMD_8X32
101 _mm_storeu_si128(
reinterpret_cast<__m128i*
>(out), _mm256_extracti128_si256(
raw(), 0));
104 BOTAN_FN_ISA_SIMD_8X32
107 _mm256_loadu2_m128i(
reinterpret_cast<const __m128i*
>(in2),
reinterpret_cast<const __m128i*
>(in1)));
110 BOTAN_FN_ISA_SIMD_8X32
113 _mm256_loadu2_m128i(
reinterpret_cast<const __m128i*
>(in2),
reinterpret_cast<const __m128i*
>(in1)))
117 BOTAN_FN_ISA_SIMD_8X32
118 void store_le128(uint32_t out1[], uint32_t out2[])
const noexcept {
119 _mm256_storeu2_m128i(
reinterpret_cast<__m128i*
>(out2),
reinterpret_cast<__m128i*
>(out1),
raw());
122 BOTAN_FN_ISA_SIMD_8X32
125 template <
size_t ROT>
127 requires(ROT > 0 && ROT < 32)
129#if defined(__AVX512VL__)
130 return SIMD_8x32(_mm256_rol_epi32(m_avx2, ROT));
132 if constexpr(ROT == 8) {
133 const __m256i shuf_rotl_8 =
134 _mm256_set_epi64x(0x0e0d0c0f'0a09080b, 0x06050407'02010003, 0x0e0d0c0f'0a09080b, 0x06050407'02010003);
136 return SIMD_8x32(_mm256_shuffle_epi8(m_avx2, shuf_rotl_8));
137 }
else if constexpr(ROT == 16) {
138 const __m256i shuf_rotl_16 =
139 _mm256_set_epi64x(0x0d0c0f0e'09080b0a, 0x05040706'01000302, 0x0d0c0f0e'09080b0a, 0x05040706'01000302);
141 return SIMD_8x32(_mm256_shuffle_epi8(m_avx2, shuf_rotl_16));
142 }
else if constexpr(ROT == 24) {
143 const __m256i shuf_rotl_24 =
144 _mm256_set_epi64x(0x0c0f0e0d'080b0a09, 0x04070605'00030201, 0x0c0f0e0d'080b0a09, 0x04070605'00030201);
146 return SIMD_8x32(_mm256_shuffle_epi8(m_avx2, shuf_rotl_24));
148 return SIMD_8x32(_mm256_xor_si256(_mm256_slli_epi32(m_avx2,
static_cast<int>(ROT)),
149 _mm256_srli_epi32(m_avx2,
static_cast<int>(32 - ROT))));
154 template <
size_t ROT>
156 return this->
rotl<32 - ROT>();
173 BOTAN_FN_ISA_SIMD_8X32
180 BOTAN_FN_ISA_SIMD_8X32
187 BOTAN_FN_ISA_SIMD_8X32
194 BOTAN_FN_ISA_SIMD_8X32
201 BOTAN_FN_ISA_SIMD_8X32
208 BOTAN_FN_ISA_SIMD_8X32
211 BOTAN_FN_ISA_SIMD_8X32
214 BOTAN_FN_ISA_SIMD_8X32
217 BOTAN_FN_ISA_SIMD_8X32
220 BOTAN_FN_ISA_SIMD_8X32
223 BOTAN_FN_ISA_SIMD_8X32
228 return SIMD_8x32(_mm256_slli_epi32(m_avx2, SHIFT));
233 return SIMD_8x32(_mm256_srli_epi32(m_avx2, SHIFT));
236 BOTAN_FN_ISA_SIMD_8X32
238 return SIMD_8x32(_mm256_xor_si256(m_avx2, _mm256_set1_epi32(0xFFFFFFFF)));
242 BOTAN_FN_ISA_SIMD_8X32
244 return SIMD_8x32(_mm256_andnot_si256(m_avx2, other.m_avx2));
247 BOTAN_FN_ISA_SIMD_8X32
249 alignas(32)
const uint8_t BSWAP_TBL[32] = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12,
250 19, 18, 17, 16, 23, 22, 21, 20, 27, 26, 25, 24, 31, 30, 29, 28};
252 const __m256i
bswap = _mm256_load_si256(
reinterpret_cast<const __m256i*
>(BSWAP_TBL));
254 const __m256i output = _mm256_shuffle_epi8(m_avx2,
bswap);
260 BOTAN_FN_ISA_SIMD_8X32
262 alignas(32)
const uint8_t REV_TBL[32] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
263 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
265 const __m256i
bswap = _mm256_load_si256(
reinterpret_cast<const __m256i*
>(REV_TBL));
266 const __m256i output = _mm256_shuffle_epi8(m_avx2,
bswap);
270 BOTAN_FN_ISA_SIMD_8X32
273 BOTAN_FN_ISA_SIMD_8X32
275 const __m256i T0 = _mm256_unpacklo_epi32(B0.m_avx2, B1.m_avx2);
276 const __m256i T1 = _mm256_unpacklo_epi32(B2.m_avx2, B3.m_avx2);
277 const __m256i T2 = _mm256_unpackhi_epi32(B0.m_avx2, B1.m_avx2);
278 const __m256i T3 = _mm256_unpackhi_epi32(B2.m_avx2, B3.m_avx2);
280 B0.m_avx2 = _mm256_unpacklo_epi64(T0, T1);
281 B1.m_avx2 = _mm256_unpackhi_epi64(T0, T1);
282 B2.m_avx2 = _mm256_unpacklo_epi64(T2, T3);
283 B3.m_avx2 = _mm256_unpackhi_epi64(T2, T3);
292 requires(I > 0 && I <= 3)
299 requires(I > 0 && I <= 3)
304 BOTAN_FN_ISA_SIMD_8X32
329 const __m256i bias = _mm256_set1_epi32(
static_cast<int32_t
>(0x80000000));
330 return SIMD_8x32(_mm256_cmpgt_epi32(_mm256_xor_si256(other.raw(), bias), _mm256_xor_si256(
raw(), bias)));
333 BOTAN_FN_ISA_SIMD_8X32
335#if defined(__AVX512VL__)
336 return SIMD_8x32(_mm256_ternarylogic_epi32(mask.raw(), a.raw(), b.raw(), 0xca));
338 return (mask & a) ^ mask.andc(b);
342 BOTAN_FN_ISA_SIMD_8X32
344#if defined(__AVX512VL__)
345 return SIMD_8x32(_mm256_ternarylogic_epi32(x.raw(), y.raw(), z.raw(), 0xe8));
376 template <u
int8_t POLY>
378 const auto poly = _mm256_set1_epi8(
static_cast<char>(POLY));
379 const auto shifted = _mm256_add_epi8(
raw(),
raw());
381 return SIMD_8x32(_mm256_blendv_epi8(shifted, _mm256_xor_si256(shifted, poly),
raw()));
384 BOTAN_FN_ISA_SIMD_8X32
387 BOTAN_FN_ISA_SIMD_8X32
390 __m256i BOTAN_FN_ISA_SIMD_8X32
raw() const noexcept {
return m_avx2; }
392 BOTAN_FN_ISA_SIMD_8X32
396 BOTAN_FN_ISA_SIMD_8X32
398 const auto T0 =
SIMD_8x32(_mm256_permute2x128_si256(A.
raw(), B.
raw(), 0 + (2 << 4)));
399 const auto T1 =
SIMD_8x32(_mm256_permute2x128_si256(A.
raw(), B.
raw(), 1 + (3 << 4)));
411 return input.
rotl<R>();
416 return input.
rotr<R>();
422 return input.
shl<S>();
BOTAN_FN_ISA_SIMD_8X32 void store_le(uint32_t out[]) const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 shift_elems_right() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 andc(const SIMD_8x32 &other) const noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le128(const uint32_t *in) noexcept
SIMD_8x32 & operator=(SIMD_8x32 &&other)=default
SIMD_8x32(const SIMD_8x32 &other)=default
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le128(const uint32_t in1[], const uint32_t in2[]) noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 shr() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator-(const SIMD_8x32 &other) const noexcept
static BOTAN_FN_ISA_SIMD_8X32 void transpose(SIMD_8x32 &B0, SIMD_8x32 &B1, SIMD_8x32 &B2, SIMD_8x32 &B3, SIMD_8x32 &B4, SIMD_8x32 &B5, SIMD_8x32 &B6, SIMD_8x32 &B7) noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 splat(uint32_t B) noexcept
static BOTAN_FN_ISA_SIMD_8X32 void transpose(SIMD_8x32 &B0, SIMD_8x32 &B1, SIMD_8x32 &B2, SIMD_8x32 &B3) noexcept
BOTAN_FN_ISA_SIMD_8X32 void operator^=(uint32_t other)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator~() const noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le(const uint8_t *in) noexcept
static BOTAN_FN_ISA_SIMD_8X32 void zero_registers() noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le128(const uint8_t *in) noexcept
SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 unsigned_lt(const SIMD_8x32 &other) const noexcept
BOTAN_FN_ISA_SIMD_8X32 void operator-=(const SIMD_8x32 &other)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 rotr() const noexcept
SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 swap_halves() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 rotl() const noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 choose(const SIMD_8x32 &mask, const SIMD_8x32 &a, const SIMD_8x32 &b) noexcept
BOTAN_FN_ISA_SIMD_8X32 void operator&=(const SIMD_8x32 &other)
BOTAN_FN_ISA_SIMD_8X32 void store_le128(uint32_t out1[], uint32_t out2[]) const noexcept
static SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 byte_shuffle(const SIMD_8x32 &tbl, const SIMD_8x32 &idx)
static BOTAN_FN_ISA_SIMD_8X32 void reset_registers() noexcept
BOTAN_FN_ISA_SIMD_8X32 void operator+=(const SIMD_8x32 &other)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 shift_elems_left() const noexcept
SIMD_8x32(SIMD_8x32 &&other)=default
BOTAN_FN_ISA_SIMD_8X32 void operator^=(const SIMD_8x32 &other)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator|(const SIMD_8x32 &other) const noexcept
SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 sigma0() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator^(const SIMD_8x32 &other) const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 reverse() const noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le(const uint32_t *in) noexcept
BOTAN_FN_ISA_SIMD_8X32 BOTAN_FORCE_INLINE SIMD_8x32() noexcept
BOTAN_FN_ISA_SIMD_8X32 void store_le128(uint8_t out[]) const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 bswap() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 rev_words() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3, uint32_t B4, uint32_t B5, uint32_t B6, uint32_t B7) noexcept
SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 sigma1() const noexcept
static SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 alignr8(const SIMD_8x32 &a, const SIMD_8x32 &b)
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_be(const uint8_t *in) noexcept
BOTAN_FN_ISA_SIMD_8X32 void store_be(uint8_t out[]) const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator&(const SIMD_8x32 &other) const noexcept
SIMD_8x32 & operator=(const SIMD_8x32 &other)=default
SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 xtime() const noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_be128(const uint8_t in1[], const uint8_t in2[]) noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 majority(const SIMD_8x32 &x, const SIMD_8x32 &y, const SIMD_8x32 &z) noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_be(const uint8_t *const *ptrs, size_t i, size_t n, size_t offset=0)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 operator+(const SIMD_8x32 &other) const noexcept
__m256i BOTAN_FN_ISA_SIMD_8X32 raw() const noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32(__m256i x) noexcept
static SIMD_8x32 BOTAN_FN_ISA_SIMD_8X32 masked_byte_shuffle(const SIMD_8x32 &tbl, const SIMD_8x32 &idx)
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32(const uint32_t B[8]) noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept
BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 shl() const noexcept
BOTAN_FN_ISA_SIMD_8X32 void operator|=(const SIMD_8x32 &other)
BOTAN_FN_ISA_SIMD_8X32 void store_le(uint8_t out[]) const noexcept
#define BOTAN_FORCE_INLINE
BOTAN_FORCE_INLINE constexpr T rotr(T input)
BOTAN_FORCE_INLINE constexpr T rotl(T input)
SIMD_4x32 shl(SIMD_4x32 input)