Botan 3.14.1
Crypto and TLS for C&
simd_avx512.h
Go to the documentation of this file.
1/*
2* (C) 2023 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#ifndef BOTAN_SIMD_AVX512_H_
8#define BOTAN_SIMD_AVX512_H_
9
10#include <botan/compiler.h>
11#include <botan/types.h>
12#include <botan/internal/isa_extn.h>
13#include <immintrin.h>
14
15namespace Botan {
16
17// NOLINTBEGIN(portability-simd-intrinsics)
18
19class SIMD_16x32 final {
20 public:
21 SIMD_16x32& operator=(const SIMD_16x32& other) = default;
22 SIMD_16x32(const SIMD_16x32& other) = default;
23
24 SIMD_16x32& operator=(SIMD_16x32&& other) = default;
25 SIMD_16x32(SIMD_16x32&& other) = default;
26
27 ~SIMD_16x32() = default;
28
29 BOTAN_FN_ISA_AVX512
30 BOTAN_FORCE_INLINE SIMD_16x32() : m_avx512(_mm512_setzero_si512()) {}
31
32 BOTAN_FN_ISA_AVX512
33 explicit SIMD_16x32(const uint32_t B[16]) {
34 // NOLINTNEXTLINE(*-prefer-member-initializer)
35 m_avx512 = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(B));
36 }
37
38 BOTAN_FN_ISA_AVX512
39 explicit SIMD_16x32(uint32_t B0,
40 uint32_t B1,
41 uint32_t B2,
42 uint32_t B3,
43 uint32_t B4,
44 uint32_t B5,
45 uint32_t B6,
46 uint32_t B7,
47 uint32_t B8,
48 uint32_t B9,
49 uint32_t BA,
50 uint32_t BB,
51 uint32_t BC,
52 uint32_t BD,
53 uint32_t BE,
54 uint32_t BF) {
55 // NOLINTNEXTLINE(*-prefer-member-initializer)
56 m_avx512 = _mm512_set_epi32(BF, BE, BD, BC, BB, BA, B9, B8, B7, B6, B5, B4, B3, B2, B1, B0);
57 }
58
59 BOTAN_FN_ISA_AVX512
60 static SIMD_16x32 splat(uint32_t B) { return SIMD_16x32(_mm512_set1_epi32(B)); }
61
62 BOTAN_FN_ISA_AVX512
63 static SIMD_16x32 load_le(const uint8_t* in) {
64 return SIMD_16x32(_mm512_loadu_si512(reinterpret_cast<const __m512i*>(in)));
65 }
66
67 BOTAN_FN_ISA_AVX512
68 static SIMD_16x32 load_be(const uint8_t* in) { return load_le(in).bswap(); }
69
70 /**
71 * Load in big endian order from ptrs[i] + Stride * n + offset,
72 * gathering lane i's n'th block out of a set of per lane buffers
73 */
74 template <size_t Stride>
75 static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_be(const uint8_t* const* ptrs, size_t i, size_t n, size_t offset = 0) {
76 return load_be(ptrs[i] + Stride * n + offset);
77 }
78
79 BOTAN_FN_ISA_AVX512
80 void store_le(uint8_t out[]) const { _mm512_storeu_si512(reinterpret_cast<__m512i*>(out), m_avx512); }
81
82 BOTAN_FN_ISA_AVX512
83 void store_be(uint8_t out[]) const { bswap().store_le(out); }
84
85 template <size_t ROT>
86 BOTAN_FN_ISA_AVX512 SIMD_16x32 rotl() const
87 requires(ROT > 0 && ROT < 32)
88 {
89 return SIMD_16x32(_mm512_rol_epi32(m_avx512, ROT));
90 }
91
92 template <size_t ROT>
93 BOTAN_FN_ISA_AVX512 SIMD_16x32 rotr() const {
94 return this->rotl<32 - ROT>();
95 }
96
97 SIMD_16x32 BOTAN_FN_ISA_AVX512 sigma0() const {
98 const SIMD_16x32 r1 = this->rotr<2>();
99 const SIMD_16x32 r2 = this->rotr<13>();
100 const SIMD_16x32 r3 = this->rotr<22>();
101 return r1 ^ r2 ^ r3;
102 }
103
104 SIMD_16x32 BOTAN_FN_ISA_AVX512 sigma1() const {
105 const SIMD_16x32 r1 = this->rotr<6>();
106 const SIMD_16x32 r2 = this->rotr<11>();
107 const SIMD_16x32 r3 = this->rotr<25>();
108 return r1 ^ r2 ^ r3;
109 }
110
111 BOTAN_FN_ISA_AVX512
112 SIMD_16x32 operator+(const SIMD_16x32& other) const {
113 SIMD_16x32 retval(*this);
114 retval += other;
115 return retval;
116 }
117
118 BOTAN_FN_ISA_AVX512
119 SIMD_16x32 operator-(const SIMD_16x32& other) const {
120 SIMD_16x32 retval(*this);
121 retval -= other;
122 return retval;
123 }
124
125 BOTAN_FN_ISA_AVX512
126 SIMD_16x32 operator^(const SIMD_16x32& other) const {
127 SIMD_16x32 retval(*this);
128 retval ^= other;
129 return retval;
130 }
131
132 BOTAN_FN_ISA_AVX512
133 SIMD_16x32 operator|(const SIMD_16x32& other) const {
134 SIMD_16x32 retval(*this);
135 retval |= other;
136 return retval;
137 }
138
139 BOTAN_FN_ISA_AVX512
140 SIMD_16x32 operator&(const SIMD_16x32& other) const {
141 SIMD_16x32 retval(*this);
142 retval &= other;
143 return retval;
144 }
145
146 BOTAN_FN_ISA_AVX512
147 void operator+=(const SIMD_16x32& other) { m_avx512 = _mm512_add_epi32(m_avx512, other.m_avx512); }
148
149 BOTAN_FN_ISA_AVX512
150 void operator-=(const SIMD_16x32& other) { m_avx512 = _mm512_sub_epi32(m_avx512, other.m_avx512); }
151
152 BOTAN_FN_ISA_AVX512
153 void operator^=(const SIMD_16x32& other) { m_avx512 = _mm512_xor_si512(m_avx512, other.m_avx512); }
154
155 BOTAN_FN_ISA_AVX512
156 void operator^=(uint32_t other) { *this ^= SIMD_16x32::splat(other); }
157
158 BOTAN_FN_ISA_AVX512
159 void operator|=(const SIMD_16x32& other) { m_avx512 = _mm512_or_si512(m_avx512, other.m_avx512); }
160
161 BOTAN_FN_ISA_AVX512
162 void operator&=(const SIMD_16x32& other) { m_avx512 = _mm512_and_si512(m_avx512, other.m_avx512); }
163
164 template <int SHIFT>
165 BOTAN_FN_ISA_AVX512 SIMD_16x32 shl() const {
166 return SIMD_16x32(_mm512_slli_epi32(m_avx512, SHIFT));
167 }
168
169 template <int SHIFT>
170 BOTAN_FN_ISA_AVX512 SIMD_16x32 shr() const {
171 return SIMD_16x32(_mm512_srli_epi32(m_avx512, SHIFT));
172 }
173
174 BOTAN_FN_ISA_AVX512
175 SIMD_16x32 operator~() const { return SIMD_16x32(_mm512_xor_si512(m_avx512, _mm512_set1_epi32(0xFFFFFFFF))); }
176
177 // (~reg) & other
178 BOTAN_FN_ISA_AVX512
179 SIMD_16x32 andc(const SIMD_16x32& other) const {
180 return SIMD_16x32(_mm512_andnot_si512(m_avx512, other.m_avx512));
181 }
182
183 template <uint8_t TBL>
184 BOTAN_FN_ISA_AVX512 static SIMD_16x32 ternary_fn(const SIMD_16x32& a, const SIMD_16x32& b, const SIMD_16x32& c) {
185 return SIMD_16x32(_mm512_ternarylogic_epi32(a.raw(), b.raw(), c.raw(), TBL));
186 }
187
188 BOTAN_FN_ISA_AVX512
190 const uint8_t BSWAP_MASK[64] = {
191 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12, 19, 18, 17, 16, 23, 22,
192 21, 20, 27, 26, 25, 24, 31, 30, 29, 28, 35, 34, 33, 32, 39, 38, 37, 36, 43, 42, 41, 40,
193 47, 46, 45, 44, 51, 50, 49, 48, 55, 54, 53, 52, 59, 58, 57, 56, 63, 62, 61, 60,
194 };
195
196 const __m512i bswap = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(BSWAP_MASK));
197
198 const __m512i output = _mm512_shuffle_epi8(m_avx512, bswap);
199
200 return SIMD_16x32(output);
201 }
202
203 BOTAN_FN_ISA_AVX512
204 SIMD_16x32 rev_words() const noexcept { return SIMD_16x32(_mm512_shuffle_epi32(raw(), _MM_PERM_ABCD)); }
205
206 BOTAN_FN_ISA_AVX512
207 static void transpose(SIMD_16x32& B0, SIMD_16x32& B1, SIMD_16x32& B2, SIMD_16x32& B3) {
208 const __m512i T0 = _mm512_unpacklo_epi32(B0.m_avx512, B1.m_avx512);
209 const __m512i T1 = _mm512_unpacklo_epi32(B2.m_avx512, B3.m_avx512);
210 const __m512i T2 = _mm512_unpackhi_epi32(B0.m_avx512, B1.m_avx512);
211 const __m512i T3 = _mm512_unpackhi_epi32(B2.m_avx512, B3.m_avx512);
212
213 B0.m_avx512 = _mm512_unpacklo_epi64(T0, T1);
214 B1.m_avx512 = _mm512_unpackhi_epi64(T0, T1);
215 B2.m_avx512 = _mm512_unpacklo_epi64(T2, T3);
216 B3.m_avx512 = _mm512_unpackhi_epi64(T2, T3);
217 }
218
219 BOTAN_FN_ISA_AVX512
220 static void transpose(SIMD_16x32& B0,
221 SIMD_16x32& B1,
222 SIMD_16x32& B2,
223 SIMD_16x32& B3,
224 SIMD_16x32& B4,
225 SIMD_16x32& B5,
226 SIMD_16x32& B6,
227 SIMD_16x32& B7,
228 SIMD_16x32& B8,
229 SIMD_16x32& B9,
230 SIMD_16x32& BA,
231 SIMD_16x32& BB,
232 SIMD_16x32& BC,
233 SIMD_16x32& BD,
234 SIMD_16x32& BE,
235 SIMD_16x32& BF) {
236 auto t0 = _mm512_unpacklo_epi32(B0.raw(), B1.raw());
237 auto t1 = _mm512_unpackhi_epi32(B0.raw(), B1.raw());
238 auto t2 = _mm512_unpacklo_epi32(B2.raw(), B3.raw());
239 auto t3 = _mm512_unpackhi_epi32(B2.raw(), B3.raw());
240 auto t4 = _mm512_unpacklo_epi32(B4.raw(), B5.raw());
241 auto t5 = _mm512_unpackhi_epi32(B4.raw(), B5.raw());
242 auto t6 = _mm512_unpacklo_epi32(B6.raw(), B7.raw());
243 auto t7 = _mm512_unpackhi_epi32(B6.raw(), B7.raw());
244 auto t8 = _mm512_unpacklo_epi32(B8.raw(), B9.raw());
245 auto t9 = _mm512_unpackhi_epi32(B8.raw(), B9.raw());
246 auto ta = _mm512_unpacklo_epi32(BA.raw(), BB.raw());
247 auto tb = _mm512_unpackhi_epi32(BA.raw(), BB.raw());
248 auto tc = _mm512_unpacklo_epi32(BC.raw(), BD.raw());
249 auto td = _mm512_unpackhi_epi32(BC.raw(), BD.raw());
250 auto te = _mm512_unpacklo_epi32(BE.raw(), BF.raw());
251 auto tf = _mm512_unpackhi_epi32(BE.raw(), BF.raw());
252
253 const auto r0 = _mm512_unpacklo_epi64(t0, t2);
254 const auto r1 = _mm512_unpackhi_epi64(t0, t2);
255 const auto r2 = _mm512_unpacklo_epi64(t1, t3);
256 const auto r3 = _mm512_unpackhi_epi64(t1, t3);
257 const auto r4 = _mm512_unpacklo_epi64(t4, t6);
258 const auto r5 = _mm512_unpackhi_epi64(t4, t6);
259 const auto r6 = _mm512_unpacklo_epi64(t5, t7);
260 const auto r7 = _mm512_unpackhi_epi64(t5, t7);
261 const auto r8 = _mm512_unpacklo_epi64(t8, ta);
262 const auto r9 = _mm512_unpackhi_epi64(t8, ta);
263 const auto ra = _mm512_unpacklo_epi64(t9, tb);
264 const auto rb = _mm512_unpackhi_epi64(t9, tb);
265 const auto rc = _mm512_unpacklo_epi64(tc, te);
266 const auto rd = _mm512_unpackhi_epi64(tc, te);
267 const auto re = _mm512_unpacklo_epi64(td, tf);
268 const auto rf = _mm512_unpackhi_epi64(td, tf);
269
270 t0 = _mm512_shuffle_i32x4(r0, r4, 0x88);
271 t1 = _mm512_shuffle_i32x4(r1, r5, 0x88);
272 t2 = _mm512_shuffle_i32x4(r2, r6, 0x88);
273 t3 = _mm512_shuffle_i32x4(r3, r7, 0x88);
274 t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd);
275 t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd);
276 t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd);
277 t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd);
278 t8 = _mm512_shuffle_i32x4(r8, rc, 0x88);
279 t9 = _mm512_shuffle_i32x4(r9, rd, 0x88);
280 ta = _mm512_shuffle_i32x4(ra, re, 0x88);
281 tb = _mm512_shuffle_i32x4(rb, rf, 0x88);
282 tc = _mm512_shuffle_i32x4(r8, rc, 0xdd);
283 td = _mm512_shuffle_i32x4(r9, rd, 0xdd);
284 te = _mm512_shuffle_i32x4(ra, re, 0xdd);
285 tf = _mm512_shuffle_i32x4(rb, rf, 0xdd);
286
287 B0.m_avx512 = _mm512_shuffle_i32x4(t0, t8, 0x88);
288 B1.m_avx512 = _mm512_shuffle_i32x4(t1, t9, 0x88);
289 B2.m_avx512 = _mm512_shuffle_i32x4(t2, ta, 0x88);
290 B3.m_avx512 = _mm512_shuffle_i32x4(t3, tb, 0x88);
291 B4.m_avx512 = _mm512_shuffle_i32x4(t4, tc, 0x88);
292 B5.m_avx512 = _mm512_shuffle_i32x4(t5, td, 0x88);
293 B6.m_avx512 = _mm512_shuffle_i32x4(t6, te, 0x88);
294 B7.m_avx512 = _mm512_shuffle_i32x4(t7, tf, 0x88);
295 B8.m_avx512 = _mm512_shuffle_i32x4(t0, t8, 0xdd);
296 B9.m_avx512 = _mm512_shuffle_i32x4(t1, t9, 0xdd);
297 BA.m_avx512 = _mm512_shuffle_i32x4(t2, ta, 0xdd);
298 BB.m_avx512 = _mm512_shuffle_i32x4(t3, tb, 0xdd);
299 BC.m_avx512 = _mm512_shuffle_i32x4(t4, tc, 0xdd);
300 BD.m_avx512 = _mm512_shuffle_i32x4(t5, td, 0xdd);
301 BE.m_avx512 = _mm512_shuffle_i32x4(t6, te, 0xdd);
302 BF.m_avx512 = _mm512_shuffle_i32x4(t7, tf, 0xdd);
303 }
304
305 /**
306 * Unsigned lane comparison; returns a mask with all bits set in each
307 * 32-bit lane that is (unsigned) less than the corresponding lane of
308 * @p other, and all bits cleared otherwise.
309 */
310 SIMD_16x32 BOTAN_FN_ISA_AVX512 unsigned_lt(const SIMD_16x32& other) const noexcept {
311 return SIMD_16x32(_mm512_movm_epi32(_mm512_cmplt_epu32_mask(raw(), other.raw())));
312 }
313
314 BOTAN_FN_ISA_AVX512
315 static SIMD_16x32 choose(const SIMD_16x32& mask, const SIMD_16x32& a, const SIMD_16x32& b) {
316 return SIMD_16x32::ternary_fn<0xca>(mask, a, b);
317 }
318
319 BOTAN_FN_ISA_AVX512
320 static SIMD_16x32 majority(const SIMD_16x32& x, const SIMD_16x32& y, const SIMD_16x32& z) {
321 return SIMD_16x32::ternary_fn<0xe8>(x, y, z);
322 }
323
324 BOTAN_FN_ISA_AVX512 static void zero_registers() {
325 // Unfortunately this only zeros zmm0-zmm15 and not zmm16-zmm32
326 _mm256_zeroall();
327 }
328
329 __m512i BOTAN_FN_ISA_AVX512 raw() const { return m_avx512; }
330
331 BOTAN_FN_ISA_AVX512
332 explicit SIMD_16x32(__m512i x) noexcept : m_avx512(x) {}
333
334 private:
335 __m512i m_avx512;
336};
337
338// NOLINTEND(portability-simd-intrinsics)
339
340template <size_t R>
341inline SIMD_16x32 BOTAN_FN_ISA_AVX512 rotl(SIMD_16x32 input) {
342 return input.rotl<R>();
343}
344
345template <size_t R>
346inline SIMD_16x32 BOTAN_FN_ISA_AVX512 rotr(SIMD_16x32 input) {
347 return input.rotr<R>();
348}
349
350// For Serpent:
351template <size_t S>
352inline SIMD_16x32 BOTAN_FN_ISA_AVX512 shl(SIMD_16x32 input) {
353 return input.shl<S>();
354}
355
356} // namespace Botan
357
358#endif
SIMD_16x32(SIMD_16x32 &&other)=default
static BOTAN_FN_ISA_AVX512 void transpose(SIMD_16x32 &B0, SIMD_16x32 &B1, SIMD_16x32 &B2, SIMD_16x32 &B3)
BOTAN_FN_ISA_AVX512 void operator^=(const SIMD_16x32 &other)
BOTAN_FN_ISA_AVX512 void store_le(uint8_t out[]) const
Definition simd_avx512.h:80
~SIMD_16x32()=default
static BOTAN_FN_ISA_AVX512 SIMD_16x32 choose(const SIMD_16x32 &mask, const SIMD_16x32 &a, const SIMD_16x32 &b)
BOTAN_FN_ISA_AVX512 SIMD_16x32 andc(const SIMD_16x32 &other) const
SIMD_16x32 BOTAN_FN_ISA_AVX512 sigma0() const
Definition simd_avx512.h:97
BOTAN_FN_ISA_AVX512 void operator|=(const SIMD_16x32 &other)
BOTAN_FN_ISA_AVX512 BOTAN_FORCE_INLINE SIMD_16x32()
Definition simd_avx512.h:30
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator|(const SIMD_16x32 &other) const
BOTAN_FN_ISA_AVX512 void operator-=(const SIMD_16x32 &other)
BOTAN_FN_ISA_AVX512 SIMD_16x32 rev_words() const noexcept
SIMD_16x32 & operator=(SIMD_16x32 &&other)=default
BOTAN_FN_ISA_AVX512 SIMD_16x32 rotl() const
Definition simd_avx512.h:86
static BOTAN_FN_ISA_AVX512 SIMD_16x32 ternary_fn(const SIMD_16x32 &a, const SIMD_16x32 &b, const SIMD_16x32 &c)
BOTAN_FN_ISA_AVX512 SIMD_16x32 bswap() const
BOTAN_FN_ISA_AVX512 void operator&=(const SIMD_16x32 &other)
BOTAN_FN_ISA_AVX512 SIMD_16x32 shr() const
static BOTAN_FN_ISA_AVX512 SIMD_16x32 majority(const SIMD_16x32 &x, const SIMD_16x32 &y, const SIMD_16x32 &z)
BOTAN_FN_ISA_AVX512 void operator+=(const SIMD_16x32 &other)
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_le(const uint8_t *in)
Definition simd_avx512.h:63
SIMD_16x32 BOTAN_FN_ISA_AVX512 unsigned_lt(const SIMD_16x32 &other) const noexcept
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator-(const SIMD_16x32 &other) const
BOTAN_FN_ISA_AVX512 void store_be(uint8_t out[]) const
Definition simd_avx512.h:83
static BOTAN_FN_ISA_AVX512 void zero_registers()
SIMD_16x32 & operator=(const SIMD_16x32 &other)=default
BOTAN_FN_ISA_AVX512 SIMD_16x32 rotr() const
Definition simd_avx512.h:93
BOTAN_FN_ISA_AVX512 SIMD_16x32(const uint32_t B[16])
Definition simd_avx512.h:33
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_be(const uint8_t *const *ptrs, size_t i, size_t n, size_t offset=0)
Definition simd_avx512.h:75
BOTAN_FN_ISA_AVX512 SIMD_16x32(__m512i x) noexcept
__m512i BOTAN_FN_ISA_AVX512 raw() const
SIMD_16x32(const SIMD_16x32 &other)=default
BOTAN_FN_ISA_AVX512 SIMD_16x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3, uint32_t B4, uint32_t B5, uint32_t B6, uint32_t B7, uint32_t B8, uint32_t B9, uint32_t BA, uint32_t BB, uint32_t BC, uint32_t BD, uint32_t BE, uint32_t BF)
Definition simd_avx512.h:39
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator+(const SIMD_16x32 &other) const
static BOTAN_FN_ISA_AVX512 void transpose(SIMD_16x32 &B0, SIMD_16x32 &B1, SIMD_16x32 &B2, SIMD_16x32 &B3, SIMD_16x32 &B4, SIMD_16x32 &B5, SIMD_16x32 &B6, SIMD_16x32 &B7, SIMD_16x32 &B8, SIMD_16x32 &B9, SIMD_16x32 &BA, SIMD_16x32 &BB, SIMD_16x32 &BC, SIMD_16x32 &BD, SIMD_16x32 &BE, SIMD_16x32 &BF)
SIMD_16x32 BOTAN_FN_ISA_AVX512 sigma1() const
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator^(const SIMD_16x32 &other) const
static BOTAN_FN_ISA_AVX512 SIMD_16x32 splat(uint32_t B)
Definition simd_avx512.h:60
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_be(const uint8_t *in)
Definition simd_avx512.h:68
BOTAN_FN_ISA_AVX512 SIMD_16x32 shl() const
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator&(const SIMD_16x32 &other) const
BOTAN_FN_ISA_AVX512 SIMD_16x32 operator~() const
BOTAN_FN_ISA_AVX512 void operator^=(uint32_t other)
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35
BOTAN_FORCE_INLINE constexpr T rotl(T input)
Definition rotate.h:23
SIMD_4x32 shl(SIMD_4x32 input)
Definition simd_4x32.h:1004