7#ifndef BOTAN_SIMD_4X64_H_
8#define BOTAN_SIMD_4X64_H_
10#include <botan/types.h>
11#include <botan/internal/isa_extn.h>
12#include <botan/internal/target_info.h>
14#if defined(BOTAN_TARGET_ARCH_SUPPORTS_AVX2)
15 #include <immintrin.h>
33 BOTAN_FN_ISA_SIMD_4X64
SIMD_4x64() : m_simd(_mm256_setzero_si256()) {}
38 _mm256_loadu2_m128i(
reinterpret_cast<const __m128i*
>(lo),
reinterpret_cast<const __m128i*
>(hi)));
46 return SIMD_4x64(_mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(in)));
55 template <
size_t Str
ide>
57 load_be(
const uint8_t*
const* ptrs,
size_t i,
size_t n,
size_t offset = 0) {
58 return load_be(ptrs[i] + Stride * n + offset);
62 return SIMD_4x64(_mm256_broadcastsi128_si256(_mm_loadu_si128(
reinterpret_cast<const __m128i*
>(in))));
66 const auto idx = _mm256_set_epi8(
67 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7);
69 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, idx));
72 void BOTAN_FN_ISA_SIMD_4X64
store_le(uint64_t out[4])
const { this->
store_le(
reinterpret_cast<uint8_t*
>(out)); }
74 BOTAN_FN_ISA_SIMD_4X64
void store_le(uint8_t out[])
const {
75 _mm256_storeu_si256(
reinterpret_cast<__m256i*
>(out), m_simd);
78 BOTAN_FN_ISA_SIMD_4X64
void store_le2(
void* outh,
void* outl) {
79 _mm256_storeu2_m128i(
reinterpret_cast<__m128i*
>(outh),
reinterpret_cast<__m128i*
>(outl), m_simd);
109 m_simd = _mm256_add_epi64(m_simd, other.m_simd);
113 m_simd = _mm256_xor_si256(m_simd, other.m_simd);
117 m_simd = _mm256_and_si256(m_simd, other.m_simd);
120 BOTAN_FN_ISA_SIMD_4X64
void operator|=(
const SIMD_4x64& other) { m_simd = _mm256_or_si256(m_simd, other.m_simd); }
124 return SIMD_4x64(_mm256_andnot_si256(m_simd, other.m_simd));
127 template <
size_t ROT>
129 requires(ROT > 0 && ROT < 64)
131#if defined(__AVX512VL__)
132 return SIMD_4x64(_mm256_ror_epi64(m_simd, ROT));
134 if constexpr(ROT == 8) {
135 const auto shuf_rot_8 =
136 _mm256_set_epi64x(0x080f0e0d0c0b0a09, 0x0007060504030201, 0x080f0e0d0c0b0a09, 0x0007060504030201);
138 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_8));
139 }
else if constexpr(ROT == 16) {
140 const auto shuf_rot_16 =
141 _mm256_set_epi64x(0x09080f0e0d0c0b0a, 0x0100070605040302, 0x09080f0e0d0c0b0a, 0x0100070605040302);
143 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_16));
144 }
else if constexpr(ROT == 24) {
145 const auto shuf_rot_24 =
146 _mm256_set_epi64x(0x0a09080f0e0d0c0b, 0x0201000706050403, 0x0a09080f0e0d0c0b, 0x0201000706050403);
148 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_24));
149 }
else if constexpr(ROT == 32) {
150 const auto shuf_rot_32 =
151 _mm256_set_epi64x(0x0b0a09080f0e0d0c, 0x0302010007060504, 0x0b0a09080f0e0d0c, 0x0302010007060504);
153 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_32));
155 return SIMD_4x64(_mm256_or_si256(_mm256_srli_epi64(m_simd,
static_cast<int>(ROT)),
156 _mm256_slli_epi64(m_simd,
static_cast<int>(64 - ROT))));
161 template <
size_t ROT>
163 return this->
rotr<64 - ROT>();
184#if defined(__AVX512VL__)
185 constexpr uint8_t xor_not_and = 0b11010010;
188 return x ^ y.
andc(z);
192 BOTAN_FN_ISA_SIMD_4X64
194#if defined(__AVX512VL__)
197 return (mask & a) ^ mask.
andc(b);
201 BOTAN_FN_ISA_SIMD_4X64
203#if defined(__AVX512VL__)
210 BOTAN_FN_ISA_SIMD_4X64
212 const __m256i T0 = _mm256_unpacklo_epi64(B0.m_simd, B1.m_simd);
213 const __m256i T1 = _mm256_unpackhi_epi64(B0.m_simd, B1.m_simd);
214 const __m256i T2 = _mm256_unpacklo_epi64(B2.m_simd, B3.m_simd);
215 const __m256i T3 = _mm256_unpackhi_epi64(B2.m_simd, B3.m_simd);
217 B0.m_simd = _mm256_permute2x128_si256(T0, T2, 0 + (2 << 4));
218 B1.m_simd = _mm256_permute2x128_si256(T1, T3, 0 + (2 << 4));
219 B2.m_simd = _mm256_permute2x128_si256(T0, T2, 1 + (3 << 4));
220 B3.m_simd = _mm256_permute2x128_si256(T1, T3, 1 + (3 << 4));
225 return SIMD_4x64(_mm256_srli_epi64(m_simd, SHIFT));
230 return SIMD_4x64(_mm256_slli_epi64(m_simd, SHIFT));
234 return SIMD_4x64(_mm256_alignr_epi8(a.m_simd, b.m_simd, 8));
239 const __m256i m = _mm256_mul_epu32(x.m_simd, y.m_simd);
240 return SIMD_4x64(_mm256_add_epi64(m, m));
243 template <u
int8_t CTRL>
245 return SIMD_4x64(_mm256_permute4x64_epi64(x.m_simd, CTRL));
262 BOTAN_FN_ISA_SIMD_4X64
265 __m256i BOTAN_FN_ISA_SIMD_4X64
raw() const noexcept {
return m_simd; }
267 explicit BOTAN_FN_ISA_SIMD_4X64
SIMD_4x64(__m256i x) : m_simd(x) {}
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 permute_4x64(SIMD_4x64 x)
void BOTAN_FN_ISA_SIMD_4X64 store_le(uint64_t out[4]) const
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator&(const SIMD_4x64 &other) const
static SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 alignr8(const SIMD_4x64 &a, const SIMD_4x64 &b)
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64()
BOTAN_FN_ISA_SIMD_4X64 void operator&=(const SIMD_4x64 &other)
SIMD_4x64(SIMD_4x64 &&other)=default
SIMD_4x64(const SIMD_4x64 &other)=default
BOTAN_FN_ISA_SIMD_4X64 void operator^=(const SIMD_4x64 &other)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t *in)
SIMD_4x64 & operator=(SIMD_4x64 &&other)=default
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 majority(const SIMD_4x64 &x, const SIMD_4x64 &y, const SIMD_4x64 &z)
BOTAN_FN_ISA_SIMD_4X64 void operator|=(const SIMD_4x64 &other)
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shl() const noexcept
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void *lo, const void *hi)
BOTAN_FN_ISA_SIMD_4X64 void store_be(uint8_t out[]) const
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma0() const
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 bswap() const
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 choose(const SIMD_4x64 &mask, const SIMD_4x64 &a, const SIMD_4x64 &b)
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 andc(const SIMD_4x64 &other) const
BOTAN_FN_ISA_SIMD_4X64 void store_le2(void *outh, void *outl)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be(const void *in)
static BOTAN_FN_ISA_SIMD_4X64 void transpose(SIMD_4x64 &B0, SIMD_4x64 &B1, SIMD_4x64 &B2, SIMD_4x64 &B3)
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator+(const SIMD_4x64 &other) const
static void BOTAN_FN_ISA_SIMD_4X64 untwist(SIMD_4x64 &B, SIMD_4x64 &C, SIMD_4x64 &D)
BOTAN_FN_ISA_SIMD_4X64 void operator+=(const SIMD_4x64 &other)
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator|(const SIMD_4x64 &other) const
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le2(const void *lo, const void *hi)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le(const void *in)
__m256i BOTAN_FN_ISA_SIMD_4X64 raw() const noexcept
SIMD_4x64 & operator=(const SIMD_4x64 &other)=default
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator^(const SIMD_4x64 &other) const
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64(__m256i x)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 chi(const SIMD_4x64 &x, const SIMD_4x64 &y, const SIMD_4x64 &z)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 mul2_32(SIMD_4x64 x, SIMD_4x64 y)
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shr() const noexcept
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 rotl() const
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 splat(uint64_t v)
BOTAN_FN_ISA_SIMD_4X64 void store_le(uint8_t out[]) const
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma1() const
static void BOTAN_FN_ISA_SIMD_4X64 twist(SIMD_4x64 &B, SIMD_4x64 &C, SIMD_4x64 &D)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be(const uint8_t *const *ptrs, size_t i, size_t n, size_t offset=0)
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 rotr() const