Botan 3.14.1
Crypto and TLS for C&
simd_4x64.h
Go to the documentation of this file.
1/*
2* (C) 2022,2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#ifndef BOTAN_SIMD_4X64_H_
8#define BOTAN_SIMD_4X64_H_
9
10#include <botan/types.h>
11#include <botan/internal/isa_extn.h>
12#include <botan/internal/target_info.h>
13
14#if defined(BOTAN_TARGET_ARCH_SUPPORTS_AVX2)
15 #include <immintrin.h>
16#endif
17
18namespace Botan {
19
20// NOLINTBEGIN(portability-simd-intrinsics)
21
22class SIMD_4x64 final {
23 public:
24 SIMD_4x64& operator=(const SIMD_4x64& other) = default;
25 SIMD_4x64(const SIMD_4x64& other) = default;
26
27 SIMD_4x64& operator=(SIMD_4x64&& other) = default;
28 SIMD_4x64(SIMD_4x64&& other) = default;
29
30 ~SIMD_4x64() = default;
31
32 // zero initialized
33 BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64() : m_simd(_mm256_setzero_si256()) {}
34
35 // Load two halves at different addresses
36 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le2(const void* lo, const void* hi) {
37 return SIMD_4x64(
38 _mm256_loadu2_m128i(reinterpret_cast<const __m128i*>(lo), reinterpret_cast<const __m128i*>(hi)));
39 }
40
41 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void* lo, const void* hi) {
42 return SIMD_4x64::load_le2(lo, hi).bswap();
43 }
44
45 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le(const void* in) {
46 return SIMD_4x64(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(in)));
47 }
48
49 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be(const void* in) { return SIMD_4x64::load_le(in).bswap(); }
50
51 /**
52 * Load in big endian order from ptrs[i] + Stride * n + offset,
53 * gathering lane i's n'th block out of a set of per lane buffers
54 */
55 template <size_t Stride>
56 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64
57 load_be(const uint8_t* const* ptrs, size_t i, size_t n, size_t offset = 0) {
58 return load_be(ptrs[i] + Stride * n + offset);
59 }
60
61 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t* in) {
62 return SIMD_4x64(_mm256_broadcastsi128_si256(_mm_loadu_si128(reinterpret_cast<const __m128i*>(in))));
63 }
64
65 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 bswap() const {
66 const auto idx = _mm256_set_epi8(
67 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7);
68
69 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, idx));
70 }
71
72 void BOTAN_FN_ISA_SIMD_4X64 store_le(uint64_t out[4]) const { this->store_le(reinterpret_cast<uint8_t*>(out)); }
73
74 BOTAN_FN_ISA_SIMD_4X64 void store_le(uint8_t out[]) const {
75 _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), m_simd);
76 }
77
78 BOTAN_FN_ISA_SIMD_4X64 void store_le2(void* outh, void* outl) {
79 _mm256_storeu2_m128i(reinterpret_cast<__m128i*>(outh), reinterpret_cast<__m128i*>(outl), m_simd);
80 }
81
82 BOTAN_FN_ISA_SIMD_4X64 void store_be(uint8_t out[]) const { bswap().store_le(out); }
83
84 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator+(const SIMD_4x64& other) const {
85 SIMD_4x64 retval(*this);
86 retval += other;
87 return retval;
88 }
89
90 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator^(const SIMD_4x64& other) const {
91 SIMD_4x64 retval(*this);
92 retval ^= other;
93 return retval;
94 }
95
96 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator&(const SIMD_4x64& other) const {
97 SIMD_4x64 retval(*this);
98 retval &= other;
99 return retval;
100 }
101
102 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator|(const SIMD_4x64& other) const {
103 SIMD_4x64 retval(*this);
104 retval |= other;
105 return retval;
106 }
107
108 BOTAN_FN_ISA_SIMD_4X64 void operator+=(const SIMD_4x64& other) {
109 m_simd = _mm256_add_epi64(m_simd, other.m_simd);
110 }
111
112 BOTAN_FN_ISA_SIMD_4X64 void operator^=(const SIMD_4x64& other) {
113 m_simd = _mm256_xor_si256(m_simd, other.m_simd);
114 }
115
116 BOTAN_FN_ISA_SIMD_4X64 void operator&=(const SIMD_4x64& other) {
117 m_simd = _mm256_and_si256(m_simd, other.m_simd);
118 }
119
120 BOTAN_FN_ISA_SIMD_4X64 void operator|=(const SIMD_4x64& other) { m_simd = _mm256_or_si256(m_simd, other.m_simd); }
121
122 // (~reg) & other
123 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 andc(const SIMD_4x64& other) const {
124 return SIMD_4x64(_mm256_andnot_si256(m_simd, other.m_simd));
125 }
126
127 template <size_t ROT>
128 BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 rotr() const
129 requires(ROT > 0 && ROT < 64)
130 {
131#if defined(__AVX512VL__)
132 return SIMD_4x64(_mm256_ror_epi64(m_simd, ROT));
133#else
134 if constexpr(ROT == 8) {
135 const auto shuf_rot_8 =
136 _mm256_set_epi64x(0x080f0e0d0c0b0a09, 0x0007060504030201, 0x080f0e0d0c0b0a09, 0x0007060504030201);
137
138 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_8));
139 } else if constexpr(ROT == 16) {
140 const auto shuf_rot_16 =
141 _mm256_set_epi64x(0x09080f0e0d0c0b0a, 0x0100070605040302, 0x09080f0e0d0c0b0a, 0x0100070605040302);
142
143 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_16));
144 } else if constexpr(ROT == 24) {
145 const auto shuf_rot_24 =
146 _mm256_set_epi64x(0x0a09080f0e0d0c0b, 0x0201000706050403, 0x0a09080f0e0d0c0b, 0x0201000706050403);
147
148 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_24));
149 } else if constexpr(ROT == 32) {
150 const auto shuf_rot_32 =
151 _mm256_set_epi64x(0x0b0a09080f0e0d0c, 0x0302010007060504, 0x0b0a09080f0e0d0c, 0x0302010007060504);
152
153 return SIMD_4x64(_mm256_shuffle_epi8(m_simd, shuf_rot_32));
154 } else {
155 return SIMD_4x64(_mm256_or_si256(_mm256_srli_epi64(m_simd, static_cast<int>(ROT)),
156 _mm256_slli_epi64(m_simd, static_cast<int>(64 - ROT))));
157 }
158#endif
159 }
160
161 template <size_t ROT>
162 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 rotl() const {
163 return this->rotr<64 - ROT>();
164 }
165
166 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma0() const {
167 const SIMD_4x64 r1 = this->rotr<28>();
168 const SIMD_4x64 r2 = this->rotr<34>();
169 const SIMD_4x64 r3 = this->rotr<39>();
170 return r1 ^ r2 ^ r3;
171 }
172
173 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma1() const {
174 const SIMD_4x64 r1 = this->rotr<14>();
175 const SIMD_4x64 r2 = this->rotr<18>();
176 const SIMD_4x64 r3 = this->rotr<41>();
177 return r1 ^ r2 ^ r3;
178 }
179
180 /**
181 * The Keccak chi operation, x ^ (~y & z)
182 */
183 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 chi(const SIMD_4x64& x, const SIMD_4x64& y, const SIMD_4x64& z) {
184#if defined(__AVX512VL__)
185 constexpr uint8_t xor_not_and = 0b11010010;
186 return SIMD_4x64(_mm256_ternarylogic_epi64(x.raw(), y.raw(), z.raw(), xor_not_and));
187#else
188 return x ^ y.andc(z);
189#endif
190 }
191
192 BOTAN_FN_ISA_SIMD_4X64
193 static SIMD_4x64 choose(const SIMD_4x64& mask, const SIMD_4x64& a, const SIMD_4x64& b) {
194#if defined(__AVX512VL__)
195 return SIMD_4x64(_mm256_ternarylogic_epi64(mask.raw(), a.raw(), b.raw(), 0xca));
196#else
197 return (mask & a) ^ mask.andc(b);
198#endif
199 }
200
201 BOTAN_FN_ISA_SIMD_4X64
202 static SIMD_4x64 majority(const SIMD_4x64& x, const SIMD_4x64& y, const SIMD_4x64& z) {
203#if defined(__AVX512VL__)
204 return SIMD_4x64(_mm256_ternarylogic_epi64(x.raw(), y.raw(), z.raw(), 0xe8));
205#else
206 return SIMD_4x64::choose(x ^ y, z, y);
207#endif
208 }
209
210 BOTAN_FN_ISA_SIMD_4X64
211 static void transpose(SIMD_4x64& B0, SIMD_4x64& B1, SIMD_4x64& B2, SIMD_4x64& B3) {
212 const __m256i T0 = _mm256_unpacklo_epi64(B0.m_simd, B1.m_simd);
213 const __m256i T1 = _mm256_unpackhi_epi64(B0.m_simd, B1.m_simd);
214 const __m256i T2 = _mm256_unpacklo_epi64(B2.m_simd, B3.m_simd);
215 const __m256i T3 = _mm256_unpackhi_epi64(B2.m_simd, B3.m_simd);
216
217 B0.m_simd = _mm256_permute2x128_si256(T0, T2, 0 + (2 << 4));
218 B1.m_simd = _mm256_permute2x128_si256(T1, T3, 0 + (2 << 4));
219 B2.m_simd = _mm256_permute2x128_si256(T0, T2, 1 + (3 << 4));
220 B3.m_simd = _mm256_permute2x128_si256(T1, T3, 1 + (3 << 4));
221 }
222
223 template <int SHIFT>
224 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shr() const noexcept {
225 return SIMD_4x64(_mm256_srli_epi64(m_simd, SHIFT));
226 }
227
228 template <int SHIFT>
229 SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shl() const noexcept {
230 return SIMD_4x64(_mm256_slli_epi64(m_simd, SHIFT));
231 }
232
233 static SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 alignr8(const SIMD_4x64& a, const SIMD_4x64& b) {
234 return SIMD_4x64(_mm256_alignr_epi8(a.m_simd, b.m_simd, 8));
235 }
236
237 // Argon2 specific operation
238 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 mul2_32(SIMD_4x64 x, SIMD_4x64 y) {
239 const __m256i m = _mm256_mul_epu32(x.m_simd, y.m_simd);
240 return SIMD_4x64(_mm256_add_epi64(m, m));
241 }
242
243 template <uint8_t CTRL>
244 static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 permute_4x64(SIMD_4x64 x) {
245 return SIMD_4x64(_mm256_permute4x64_epi64(x.m_simd, CTRL));
246 }
247
248 // Argon2 specific
254
255 // Argon2 specific
261
262 BOTAN_FN_ISA_SIMD_4X64
263 static SIMD_4x64 splat(uint64_t v) { return SIMD_4x64(_mm256_set1_epi64x(v)); }
264
265 __m256i BOTAN_FN_ISA_SIMD_4X64 raw() const noexcept { return m_simd; }
266
267 explicit BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64(__m256i x) : m_simd(x) {}
268
269 private:
270 __m256i m_simd;
271};
272
273// NOLINTEND(portability-simd-intrinsics)
274
275} // namespace Botan
276
277#endif
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 permute_4x64(SIMD_4x64 x)
Definition simd_4x64.h:244
void BOTAN_FN_ISA_SIMD_4X64 store_le(uint64_t out[4]) const
Definition simd_4x64.h:72
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator&(const SIMD_4x64 &other) const
Definition simd_4x64.h:96
static SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 alignr8(const SIMD_4x64 &a, const SIMD_4x64 &b)
Definition simd_4x64.h:233
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64()
Definition simd_4x64.h:33
BOTAN_FN_ISA_SIMD_4X64 void operator&=(const SIMD_4x64 &other)
Definition simd_4x64.h:116
SIMD_4x64(SIMD_4x64 &&other)=default
SIMD_4x64(const SIMD_4x64 &other)=default
BOTAN_FN_ISA_SIMD_4X64 void operator^=(const SIMD_4x64 &other)
Definition simd_4x64.h:112
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t *in)
Definition simd_4x64.h:61
SIMD_4x64 & operator=(SIMD_4x64 &&other)=default
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 majority(const SIMD_4x64 &x, const SIMD_4x64 &y, const SIMD_4x64 &z)
Definition simd_4x64.h:202
BOTAN_FN_ISA_SIMD_4X64 void operator|=(const SIMD_4x64 &other)
Definition simd_4x64.h:120
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shl() const noexcept
Definition simd_4x64.h:229
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void *lo, const void *hi)
Definition simd_4x64.h:41
BOTAN_FN_ISA_SIMD_4X64 void store_be(uint8_t out[]) const
Definition simd_4x64.h:82
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma0() const
Definition simd_4x64.h:166
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 bswap() const
Definition simd_4x64.h:65
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 choose(const SIMD_4x64 &mask, const SIMD_4x64 &a, const SIMD_4x64 &b)
Definition simd_4x64.h:193
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 andc(const SIMD_4x64 &other) const
Definition simd_4x64.h:123
BOTAN_FN_ISA_SIMD_4X64 void store_le2(void *outh, void *outl)
Definition simd_4x64.h:78
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be(const void *in)
Definition simd_4x64.h:49
static BOTAN_FN_ISA_SIMD_4X64 void transpose(SIMD_4x64 &B0, SIMD_4x64 &B1, SIMD_4x64 &B2, SIMD_4x64 &B3)
Definition simd_4x64.h:211
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator+(const SIMD_4x64 &other) const
Definition simd_4x64.h:84
static void BOTAN_FN_ISA_SIMD_4X64 untwist(SIMD_4x64 &B, SIMD_4x64 &C, SIMD_4x64 &D)
Definition simd_4x64.h:256
BOTAN_FN_ISA_SIMD_4X64 void operator+=(const SIMD_4x64 &other)
Definition simd_4x64.h:108
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator|(const SIMD_4x64 &other) const
Definition simd_4x64.h:102
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le2(const void *lo, const void *hi)
Definition simd_4x64.h:36
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_le(const void *in)
Definition simd_4x64.h:45
__m256i BOTAN_FN_ISA_SIMD_4X64 raw() const noexcept
Definition simd_4x64.h:265
SIMD_4x64 & operator=(const SIMD_4x64 &other)=default
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 operator^(const SIMD_4x64 &other) const
Definition simd_4x64.h:90
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64(__m256i x)
Definition simd_4x64.h:267
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 chi(const SIMD_4x64 &x, const SIMD_4x64 &y, const SIMD_4x64 &z)
Definition simd_4x64.h:183
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 mul2_32(SIMD_4x64 x, SIMD_4x64 y)
Definition simd_4x64.h:238
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 shr() const noexcept
Definition simd_4x64.h:224
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 rotl() const
Definition simd_4x64.h:162
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 splat(uint64_t v)
Definition simd_4x64.h:263
~SIMD_4x64()=default
BOTAN_FN_ISA_SIMD_4X64 void store_le(uint8_t out[]) const
Definition simd_4x64.h:74
SIMD_4x64 BOTAN_FN_ISA_SIMD_4X64 sigma1() const
Definition simd_4x64.h:173
static void BOTAN_FN_ISA_SIMD_4X64 twist(SIMD_4x64 &B, SIMD_4x64 &C, SIMD_4x64 &D)
Definition simd_4x64.h:249
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be(const uint8_t *const *ptrs, size_t i, size_t n, size_t offset=0)
Definition simd_4x64.h:57
BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 rotr() const
Definition simd_4x64.h:128