Botan 3.14.1
Crypto and TLS for C&
simd_4x32.h
Go to the documentation of this file.
1/*
2* Lightweight wrappers for SIMD (4x32 bit) operations
3* (C) 2009,2011,2016,2017,2019,2025 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#ifndef BOTAN_SIMD_4X32_H_
9#define BOTAN_SIMD_4X32_H_
10
11#include <botan/compiler.h>
12#include <botan/types.h>
13#include <botan/internal/isa_extn.h>
14#include <botan/internal/target_info.h>
15#include <bit>
16#include <span>
17
18#if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3)
19 #include <tmmintrin.h>
20 #define BOTAN_SIMD_USE_SSSE3
21
22#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_VSX)
23 #include <altivec.h>
24 #undef vector
25 #undef bool
26 #define BOTAN_SIMD_USE_VSX
27
28#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_NEON)
29 #include <arm_neon.h>
30 #define BOTAN_SIMD_USE_NEON
31
32#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_LSX)
33 #include <lsxintrin.h>
34 #define BOTAN_SIMD_USE_LSX
35
36#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_SIMD128)
37 #include <wasm_simd128.h>
38 #define BOTAN_SIMD_USE_SIMD128
39
40#else
41 #error "No SIMD instruction set enabled"
42#endif
43
44namespace Botan {
45
46// NOLINTBEGIN(portability-simd-intrinsics)
47
48/**
49* 4x32 bit SIMD register
50*
51* This class is not a general purpose SIMD type, and only offers instructions
52* needed for evaluation of specific crypto primitives. For example it does not
53* currently have equality operators of any kind.
54*
55* Implemented for SSE2, POWER8 VSX, ARMv7/Aarch64 NEON, LoongArch LSX and Wasm SIMD128
56*/
57class SIMD_4x32 final {
58 public:
59#if defined(BOTAN_SIMD_USE_SSSE3) || defined(BOTAN_SIMD_USE_LSX)
60 using native_simd_type = __m128i;
61#elif defined(BOTAN_SIMD_USE_VSX)
62 using native_simd_type = __vector unsigned int;
63#elif defined(BOTAN_SIMD_USE_NEON)
64 using native_simd_type = uint32x4_t;
65#elif defined(BOTAN_SIMD_USE_SIMD128)
66 using native_simd_type = v128_t;
67#endif
68
69 SIMD_4x32& operator=(const SIMD_4x32& other) = default;
70 SIMD_4x32(const SIMD_4x32& other) = default;
71
72 SIMD_4x32& operator=(SIMD_4x32&& other) = default;
73 SIMD_4x32(SIMD_4x32&& other) = default;
74
75 ~SIMD_4x32() = default;
76
77 /* NOLINTBEGIN(*-prefer-member-initializer) */
78
79 /**
80 * Zero initialize SIMD register with 4 32-bit elements
81 */
82 BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32() noexcept {
83#if defined(BOTAN_SIMD_USE_SSSE3)
84 m_simd = _mm_setzero_si128();
85#elif defined(BOTAN_SIMD_USE_VSX)
86 m_simd = vec_splat_u32(0);
87#elif defined(BOTAN_SIMD_USE_NEON)
88 m_simd = vdupq_n_u32(0);
89#elif defined(BOTAN_SIMD_USE_LSX)
90 m_simd = __lsx_vldi(0);
91#elif defined(BOTAN_SIMD_USE_SIMD128)
92 m_simd = wasm_u32x4_const_splat(0);
93#endif
94 }
95
96 /**
97 * Load SIMD register with 4 32-bit elements
98 */
99 BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept {
100#if defined(BOTAN_SIMD_USE_SSSE3)
101 m_simd = _mm_set_epi32(B3, B2, B1, B0);
102#elif defined(BOTAN_SIMD_USE_VSX)
103 __vector unsigned int val = {B0, B1, B2, B3};
104 m_simd = val;
105#elif defined(BOTAN_SIMD_USE_NEON)
106 // Better way to do this?
107 const uint32_t B[4] = {B0, B1, B2, B3};
108 m_simd = vld1q_u32(B);
109#elif defined(BOTAN_SIMD_USE_LSX)
110 // Better way to do this?
111 const uint32_t B[4] = {B0, B1, B2, B3};
112 m_simd = __lsx_vld(B, 0);
113#elif defined(BOTAN_SIMD_USE_SIMD128)
114 m_simd = wasm_u32x4_make(B0, B1, B2, B3);
115#endif
116 }
117
118 /* NOLINTEND(*-prefer-member-initializer) */
119
120 /**
121 * Load SIMD register with one 32-bit element repeated
122 */
123 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat(uint32_t B) noexcept {
124#if defined(BOTAN_SIMD_USE_SSSE3)
125 return SIMD_4x32(_mm_set1_epi32(B));
126#elif defined(BOTAN_SIMD_USE_NEON)
127 return SIMD_4x32(vdupq_n_u32(B));
128#elif defined(BOTAN_SIMD_USE_LSX)
129 return SIMD_4x32(__lsx_vreplgr2vr_w(B));
130#elif defined(BOTAN_SIMD_USE_SIMD128)
131 return SIMD_4x32(wasm_u32x4_splat(B));
132#elif defined(BOTAN_SIMD_USE_VSX)
133 return SIMD_4x32(vec_splats(static_cast<unsigned int>(B)));
134#endif
135 }
136
137 /**
138 * Load SIMD register with one 8-bit element repeated
139 */
140 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat_u8(uint8_t B) noexcept {
141#if defined(BOTAN_SIMD_USE_SSSE3)
142 return SIMD_4x32(_mm_set1_epi8(B));
143#elif defined(BOTAN_SIMD_USE_NEON)
144 /*
145 * Do not use vdupq_n_u8 here. MSVC ARM64 (VS 17.13 through at least
146 * VS 18.6) can fold vand(vdupq_n_u8(k), vdupq_n_u32(x)) into a scalar
147 * and+dup which wrongly uses the 8-bit k as the 32-bit immediate,
148 * computing dup(x & k) instead of dup(x) & splat_u8(k). This
149 * miscompiled the SM4 key schedule. Broadcasting an explicitly
150 * replicated 32-bit value avoids the bad fold; GCC and Clang generate
151 * identical code either way.
152 */
153 return SIMD_4x32(vdupq_n_u32(static_cast<uint32_t>(B) * 0x01010101U));
154#elif defined(BOTAN_SIMD_USE_LSX)
155 return SIMD_4x32(__lsx_vreplgr2vr_b(B));
156#elif defined(BOTAN_SIMD_USE_SIMD128)
157 return SIMD_4x32(wasm_u8x16_splat(B));
158#elif defined(BOTAN_SIMD_USE_VSX)
159 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(vec_splats(static_cast<unsigned char>(B))));
160#endif
161 }
162
163 /**
164 * Load a SIMD register from four 32-bit words stored in native byte order
165 */
166 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_native(const void* in) noexcept {
167#if defined(BOTAN_SIMD_USE_SSSE3)
168 return SIMD_4x32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(in)));
169#elif defined(BOTAN_SIMD_USE_VSX)
170 return SIMD_4x32(reinterpret_cast<native_simd_type>(vec_xl(0, static_cast<const unsigned char*>(in))));
171#elif defined(BOTAN_SIMD_USE_NEON)
172 return SIMD_4x32(vld1q_u32(static_cast<const uint32_t*>(in)));
173#elif defined(BOTAN_SIMD_USE_LSX)
174 return SIMD_4x32(__lsx_vld(in, 0));
175#elif defined(BOTAN_SIMD_USE_SIMD128)
176 return SIMD_4x32(wasm_v128_load(in));
177#endif
178 }
179
180 /**
181 * Load a SIMD register with little-endian convention
182 */
183 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void* in) noexcept {
184 if constexpr(std::endian::native == std::endian::little) {
185 return SIMD_4x32::load_native(in);
186 } else {
187 return SIMD_4x32::load_native(in).bswap();
188 }
189 }
190
191 /**
192 * Load a SIMD register with big-endian convention
193 */
194 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(const void* in) noexcept {
195 if constexpr(std::endian::native == std::endian::big) {
196 return SIMD_4x32::load_native(in);
197 } else {
198 return SIMD_4x32::load_native(in).bswap();
199 }
200 }
201
202 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(std::span<const uint8_t, 16> in) {
203 return SIMD_4x32::load_le(in.data());
204 }
205
206 static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(std::span<const uint8_t, 16> in) {
207 return SIMD_4x32::load_be(in.data());
208 }
209
210 void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept {
211 this->store_le(reinterpret_cast<uint8_t*>(out));
212 }
213
214 void BOTAN_FN_ISA_SIMD_4X32 store_be(uint32_t out[4]) const noexcept {
215 this->store_be(reinterpret_cast<uint8_t*>(out));
216 }
217
218 void BOTAN_FN_ISA_SIMD_4X32 store_le(uint64_t out[2]) const noexcept {
219 this->store_le(reinterpret_cast<uint8_t*>(out));
220 }
221
222 /**
223 * Store four 32-bit words in native byte order
224 */
225 void BOTAN_FN_ISA_SIMD_4X32 store_native(void* out) const noexcept {
226#if defined(BOTAN_SIMD_USE_SSSE3)
227 _mm_storeu_si128(static_cast<__m128i*>(out), raw());
228#elif defined(BOTAN_SIMD_USE_VSX)
229 vec_xst(reinterpret_cast<__vector unsigned char>(m_simd), 0, static_cast<unsigned char*>(out));
230#elif defined(BOTAN_SIMD_USE_NEON)
231 vst1q_u32(static_cast<uint32_t*>(out), m_simd);
232#elif defined(BOTAN_SIMD_USE_LSX)
233 __lsx_vst(raw(), out, 0);
234#elif defined(BOTAN_SIMD_USE_SIMD128)
235 wasm_v128_store(out, m_simd);
236#endif
237 }
238
239 /**
240 * Store a SIMD register with little-endian convention
241 */
242 void BOTAN_FN_ISA_SIMD_4X32 store_le(uint8_t out[]) const noexcept {
243 if constexpr(std::endian::native == std::endian::little) {
244 this->store_native(out);
245 } else {
246 this->bswap().store_native(out);
247 }
248 }
249
250 /**
251 * Store a SIMD register with big-endian convention
252 */
253 void BOTAN_FN_ISA_SIMD_4X32 store_be(uint8_t out[]) const noexcept {
254 if constexpr(std::endian::native == std::endian::big) {
255 this->store_native(out);
256 } else {
257 this->bswap().store_native(out);
258 }
259 }
260
261 void BOTAN_FN_ISA_SIMD_4X32 store_be(std::span<uint8_t, 16> out) const { this->store_be(out.data()); }
262
263 void BOTAN_FN_ISA_SIMD_4X32 store_le(std::span<uint8_t, 16> out) const { this->store_le(out.data()); }
264
265 /*
266 * This is used for SHA-2/SHACAL2
267 */
268 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 sigma0() const noexcept {
269#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
270 return SIMD_4x32(__builtin_crypto_vshasigmaw(raw(), 1, 0));
271#else
272 const SIMD_4x32 r1 = this->rotr<2>();
273 const SIMD_4x32 r2 = this->rotr<13>();
274 const SIMD_4x32 r3 = this->rotr<22>();
275 return (r1 ^ r2 ^ r3);
276#endif
277 }
278
279 /*
280 * This is used for SHA-2/SHACAL2
281 */
282 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 sigma1() const noexcept {
283#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vshasigmaw) && defined(_ARCH_PWR8)
284 return SIMD_4x32(__builtin_crypto_vshasigmaw(raw(), 1, 0xF));
285#else
286 const SIMD_4x32 r1 = this->rotr<6>();
287 const SIMD_4x32 r2 = this->rotr<11>();
288 const SIMD_4x32 r3 = this->rotr<25>();
289 return (r1 ^ r2 ^ r3);
290#endif
291 }
292
293 /**
294 * Left rotation by a compile time constant
295 */
296 template <size_t ROT>
297 BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 rotl() const noexcept
298 requires(ROT > 0 && ROT < 32)
299 {
300#if defined(BOTAN_SIMD_USE_SSSE3)
301 if constexpr(ROT == 8) {
302 const auto shuf_rotl_8 = _mm_set_epi64x(0x0e0d0c0f0a09080b, 0x0605040702010003);
303 return SIMD_4x32(_mm_shuffle_epi8(raw(), shuf_rotl_8));
304 } else if constexpr(ROT == 16) {
305 const auto shuf_rotl_16 = _mm_set_epi64x(0x0d0c0f0e09080b0a, 0x0504070601000302);
306 return SIMD_4x32(_mm_shuffle_epi8(raw(), shuf_rotl_16));
307 } else if constexpr(ROT == 24) {
308 const auto shuf_rotl_24 = _mm_set_epi64x(0x0c0f0e0d080b0a09, 0x0407060500030201);
309 return SIMD_4x32(_mm_shuffle_epi8(raw(), shuf_rotl_24));
310 } else {
311 return SIMD_4x32(_mm_xor_si128(_mm_slli_epi32(raw(), static_cast<int>(ROT)),
312 _mm_srli_epi32(raw(), static_cast<int>(32 - ROT))));
313 }
314
315#elif defined(BOTAN_SIMD_USE_VSX)
316
317 return SIMD_4x32(vec_rl(m_simd, vec_splats(static_cast<unsigned int>(ROT))));
318
319#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
320
321 if constexpr(ROT == 8) {
322 const uint8_t maskb[16] = {3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14};
323 const uint8x16_t mask = vld1q_u8(maskb);
324 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(m_simd), mask)));
325 } else if constexpr(ROT == 16) {
326 return SIMD_4x32(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(m_simd))));
327 } else {
328 return SIMD_4x32(
329 vorrq_u32(vshlq_n_u32(m_simd, static_cast<int>(ROT)), vshrq_n_u32(m_simd, static_cast<int>(32 - ROT))));
330 }
331#elif defined(BOTAN_SIMD_USE_NEON)
332 return SIMD_4x32(
333 vorrq_u32(vshlq_n_u32(m_simd, static_cast<int>(ROT)), vshrq_n_u32(m_simd, static_cast<int>(32 - ROT))));
334#elif defined(BOTAN_SIMD_USE_LSX)
335 return SIMD_4x32(__lsx_vrotri_w(raw(), 32 - ROT));
336#elif defined(BOTAN_SIMD_USE_SIMD128)
337 return SIMD_4x32(wasm_v128_or(wasm_i32x4_shl(m_simd, ROT), wasm_u32x4_shr(m_simd, 32 - ROT)));
338#endif
339 }
340
341 /**
342 * Right rotation by a compile time constant
343 */
344 template <size_t ROT>
345 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 rotr() const noexcept {
346 return this->rotl<32 - ROT>();
347 }
348
349 /**
350 * Add elements of a SIMD vector
351 */
352 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator+(const SIMD_4x32& other) const noexcept {
353 SIMD_4x32 retval(*this);
354 retval += other;
355 return retval;
356 }
357
358 /**
359 * Subtract elements of a SIMD vector
360 */
361 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator-(const SIMD_4x32& other) const noexcept {
362 SIMD_4x32 retval(*this);
363 retval -= other;
364 return retval;
365 }
366
367 /**
368 * XOR elements of a SIMD vector
369 */
370 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator^(const SIMD_4x32& other) const noexcept {
371 SIMD_4x32 retval(*this);
372 retval ^= other;
373 return retval;
374 }
375
376 /**
377 * Binary OR elements of a SIMD vector
378 */
379 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator|(const SIMD_4x32& other) const noexcept {
380 SIMD_4x32 retval(*this);
381 retval |= other;
382 return retval;
383 }
384
385 /**
386 * Binary AND elements of a SIMD vector
387 */
388 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator&(const SIMD_4x32& other) const noexcept {
389 SIMD_4x32 retval(*this);
390 retval &= other;
391 return retval;
392 }
393
394 void BOTAN_FN_ISA_SIMD_4X32 operator+=(const SIMD_4x32& other) noexcept {
395#if defined(BOTAN_SIMD_USE_SSSE3)
396 m_simd = _mm_add_epi32(m_simd, other.m_simd);
397#elif defined(BOTAN_SIMD_USE_VSX)
398 m_simd = vec_add(m_simd, other.m_simd);
399#elif defined(BOTAN_SIMD_USE_NEON)
400 m_simd = vaddq_u32(m_simd, other.m_simd);
401#elif defined(BOTAN_SIMD_USE_LSX)
402 m_simd = __lsx_vadd_w(m_simd, other.m_simd);
403#elif defined(BOTAN_SIMD_USE_SIMD128)
404 m_simd = wasm_i32x4_add(m_simd, other.m_simd);
405#endif
406 }
407
408 void BOTAN_FN_ISA_SIMD_4X32 operator-=(const SIMD_4x32& other) noexcept {
409#if defined(BOTAN_SIMD_USE_SSSE3)
410 m_simd = _mm_sub_epi32(m_simd, other.m_simd);
411#elif defined(BOTAN_SIMD_USE_VSX)
412 m_simd = vec_sub(m_simd, other.m_simd);
413#elif defined(BOTAN_SIMD_USE_NEON)
414 m_simd = vsubq_u32(m_simd, other.m_simd);
415#elif defined(BOTAN_SIMD_USE_LSX)
416 m_simd = __lsx_vsub_w(m_simd, other.m_simd);
417#elif defined(BOTAN_SIMD_USE_SIMD128)
418 m_simd = wasm_i32x4_sub(m_simd, other.m_simd);
419#endif
420 }
421
422 void BOTAN_FN_ISA_SIMD_4X32 operator^=(const SIMD_4x32& other) noexcept {
423#if defined(BOTAN_SIMD_USE_SSSE3)
424 m_simd = _mm_xor_si128(m_simd, other.m_simd);
425#elif defined(BOTAN_SIMD_USE_VSX)
426 m_simd = vec_xor(m_simd, other.m_simd);
427#elif defined(BOTAN_SIMD_USE_NEON)
428 m_simd = veorq_u32(m_simd, other.m_simd);
429#elif defined(BOTAN_SIMD_USE_LSX)
430 m_simd = __lsx_vxor_v(m_simd, other.m_simd);
431#elif defined(BOTAN_SIMD_USE_SIMD128)
432 m_simd = wasm_v128_xor(m_simd, other.m_simd);
433#endif
434 }
435
436 void BOTAN_FN_ISA_SIMD_4X32 operator^=(uint32_t other) noexcept { *this ^= SIMD_4x32::splat(other); }
437
438 void BOTAN_FN_ISA_SIMD_4X32 operator|=(const SIMD_4x32& other) noexcept {
439#if defined(BOTAN_SIMD_USE_SSSE3)
440 m_simd = _mm_or_si128(m_simd, other.m_simd);
441#elif defined(BOTAN_SIMD_USE_VSX)
442 m_simd = vec_or(m_simd, other.m_simd);
443#elif defined(BOTAN_SIMD_USE_NEON)
444 m_simd = vorrq_u32(m_simd, other.m_simd);
445#elif defined(BOTAN_SIMD_USE_LSX)
446 m_simd = __lsx_vor_v(m_simd, other.m_simd);
447#elif defined(BOTAN_SIMD_USE_SIMD128)
448 m_simd = wasm_v128_or(m_simd, other.m_simd);
449#endif
450 }
451
452 void BOTAN_FN_ISA_SIMD_4X32 operator&=(const SIMD_4x32& other) noexcept {
453#if defined(BOTAN_SIMD_USE_SSSE3)
454 m_simd = _mm_and_si128(m_simd, other.m_simd);
455#elif defined(BOTAN_SIMD_USE_VSX)
456 m_simd = vec_and(m_simd, other.m_simd);
457#elif defined(BOTAN_SIMD_USE_NEON)
458 m_simd = vandq_u32(m_simd, other.m_simd);
459#elif defined(BOTAN_SIMD_USE_LSX)
460 m_simd = __lsx_vand_v(m_simd, other.m_simd);
461#elif defined(BOTAN_SIMD_USE_SIMD128)
462 m_simd = wasm_v128_and(m_simd, other.m_simd);
463#endif
464 }
465
466 template <int SHIFT>
467 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shl() const noexcept
468 requires(SHIFT > 0 && SHIFT < 32)
469 {
470#if defined(BOTAN_SIMD_USE_SSSE3)
471 return SIMD_4x32(_mm_slli_epi32(m_simd, SHIFT));
472
473#elif defined(BOTAN_SIMD_USE_VSX)
474 return SIMD_4x32(vec_sl(m_simd, vec_splats(static_cast<unsigned int>(SHIFT))));
475#elif defined(BOTAN_SIMD_USE_NEON)
476 return SIMD_4x32(vshlq_n_u32(m_simd, SHIFT));
477#elif defined(BOTAN_SIMD_USE_LSX)
478 return SIMD_4x32(__lsx_vslli_w(m_simd, SHIFT));
479#elif defined(BOTAN_SIMD_USE_SIMD128)
480 return SIMD_4x32(wasm_i32x4_shl(m_simd, SHIFT));
481#endif
482 }
483
484 template <int SHIFT>
485 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shr() const noexcept {
486#if defined(BOTAN_SIMD_USE_SSSE3)
487 return SIMD_4x32(_mm_srli_epi32(m_simd, SHIFT));
488
489#elif defined(BOTAN_SIMD_USE_VSX)
490 return SIMD_4x32(vec_sr(m_simd, vec_splats(static_cast<unsigned int>(SHIFT))));
491#elif defined(BOTAN_SIMD_USE_NEON)
492 return SIMD_4x32(vshrq_n_u32(m_simd, SHIFT));
493#elif defined(BOTAN_SIMD_USE_LSX)
494 return SIMD_4x32(__lsx_vsrli_w(m_simd, SHIFT));
495#elif defined(BOTAN_SIMD_USE_SIMD128)
496 return SIMD_4x32(wasm_u32x4_shr(m_simd, SHIFT));
497#endif
498 }
499
500 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator~() const noexcept {
501#if defined(BOTAN_SIMD_USE_SSSE3)
502 return SIMD_4x32(_mm_xor_si128(m_simd, _mm_set1_epi32(0xFFFFFFFF)));
503#elif defined(BOTAN_SIMD_USE_VSX)
504 return SIMD_4x32(vec_nor(m_simd, m_simd));
505#elif defined(BOTAN_SIMD_USE_NEON)
506 return SIMD_4x32(vmvnq_u32(m_simd));
507#elif defined(BOTAN_SIMD_USE_LSX)
508 return SIMD_4x32(__lsx_vnor_v(m_simd, m_simd));
509#elif defined(BOTAN_SIMD_USE_SIMD128)
510 return SIMD_4x32(wasm_v128_not(m_simd));
511#endif
512 }
513
514 // (~reg) & other
515 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 andc(const SIMD_4x32& other) const noexcept {
516#if defined(BOTAN_SIMD_USE_SSSE3)
517 return SIMD_4x32(_mm_andnot_si128(m_simd, other.m_simd));
518#elif defined(BOTAN_SIMD_USE_VSX)
519 /*
520 AltiVec does arg1 & ~arg2 rather than SSE's ~arg1 & arg2
521 so swap the arguments
522 */
523 return SIMD_4x32(vec_andc(other.m_simd, m_simd));
524#elif defined(BOTAN_SIMD_USE_NEON)
525 // NEON is also a & ~b
526 return SIMD_4x32(vbicq_u32(other.m_simd, m_simd));
527#elif defined(BOTAN_SIMD_USE_LSX)
528 // LSX is ~a & b
529 return SIMD_4x32(__lsx_vandn_v(m_simd, other.m_simd));
530#elif defined(BOTAN_SIMD_USE_SIMD128)
531 // SIMD128 is a & ~b
532 return SIMD_4x32(wasm_v128_andnot(other.m_simd, m_simd));
533#endif
534 }
535
536 BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 reverse_all_bytes() const {
537 const auto rev_idx = SIMD_4x32(0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203);
538 return SIMD_4x32::byte_shuffle(*this, rev_idx);
539 }
540
541 /**
542 * Return copy *this with each word byte swapped
543 */
544 BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 bswap() const noexcept {
545#if defined(BOTAN_SIMD_USE_SSSE3)
546 const auto idx = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3);
547
548 return SIMD_4x32(_mm_shuffle_epi8(raw(), idx));
549#elif defined(BOTAN_SIMD_USE_VSX)
550 return SIMD_4x32(vec_revb(m_simd));
551
552#elif defined(BOTAN_SIMD_USE_NEON)
553 return SIMD_4x32(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(m_simd))));
554#elif defined(BOTAN_SIMD_USE_LSX)
555 return SIMD_4x32(__lsx_vshuf4i_b(m_simd, 0b00011011));
556#elif defined(BOTAN_SIMD_USE_SIMD128)
557 return SIMD_4x32(wasm_i8x16_shuffle(m_simd, m_simd, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
558#endif
559 }
560
561 template <size_t I>
562 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_left() const noexcept
563 requires(I <= 3)
564 {
565#if defined(BOTAN_SIMD_USE_SSSE3)
566 return SIMD_4x32(_mm_slli_si128(raw(), 4 * I));
567#elif defined(BOTAN_SIMD_USE_NEON)
568 return SIMD_4x32(vextq_u32(vdupq_n_u32(0), raw(), 4 - I));
569#elif defined(BOTAN_SIMD_USE_VSX)
570 const __vector unsigned int zero = vec_splat_u32(0);
571
572 const __vector unsigned char shuf[3] = {
573 {16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
574 {16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7},
575 {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3},
576 };
577
578 return SIMD_4x32(vec_perm(raw(), zero, shuf[I - 1]));
579#elif defined(BOTAN_SIMD_USE_LSX)
580 return SIMD_4x32(__lsx_vbsll_v(raw(), 4 * I));
581#elif defined(BOTAN_SIMD_USE_SIMD128)
582 if constexpr(I == 0) {
583 return SIMD_4x32(m_simd);
584 }
585
586 const auto zero = wasm_u32x4_const_splat(0);
587 if constexpr(I == 1) {
588 return SIMD_4x32(wasm_i8x16_shuffle(m_simd, zero, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11));
589 }
590 if constexpr(I == 2) {
591 return SIMD_4x32(wasm_i8x16_shuffle(m_simd, zero, 16, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7));
592 }
593
594 return SIMD_4x32(wasm_i8x16_shuffle(m_simd, zero, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3));
595#endif
596 }
597
598 template <size_t I>
599 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_right() const noexcept
600 requires(I <= 3)
601 {
602#if defined(BOTAN_SIMD_USE_SSSE3)
603 return SIMD_4x32(_mm_srli_si128(raw(), 4 * I));
604#elif defined(BOTAN_SIMD_USE_NEON)
605 return SIMD_4x32(vextq_u32(raw(), vdupq_n_u32(0), I));
606#elif defined(BOTAN_SIMD_USE_VSX)
607 const __vector unsigned int zero = vec_splat_u32(0);
608
609 const __vector unsigned char shuf[3] = {
610 {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
611 {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
612 {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27},
613 };
614
615 return SIMD_4x32(vec_perm(raw(), zero, shuf[I - 1]));
616#elif defined(BOTAN_SIMD_USE_LSX)
617 return SIMD_4x32(__lsx_vbsrl_v(raw(), 4 * I));
618#elif defined(BOTAN_SIMD_USE_SIMD128)
619 if constexpr(I == 0) {
620 return SIMD_4x32(m_simd);
621 }
622
623 const auto zero = wasm_u32x4_const_splat(0);
624 if constexpr(I == 1) {
625 return SIMD_4x32(
626 wasm_i8x16_shuffle(m_simd, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16));
627 }
628 if constexpr(I == 2) {
629 return SIMD_4x32(
630 wasm_i8x16_shuffle(m_simd, zero, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16));
631 }
632
633 return SIMD_4x32(
634 wasm_i8x16_shuffle(m_simd, zero, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16));
635#endif
636 }
637
638 /**
639 * 4x4 Transposition on SIMD registers
640 */
641 static void BOTAN_FN_ISA_SIMD_4X32 transpose(SIMD_4x32& B0,
642 SIMD_4x32& B1,
643 SIMD_4x32& B2,
644 SIMD_4x32& B3) noexcept {
645#if defined(BOTAN_SIMD_USE_SSSE3)
646 const __m128i T0 = _mm_unpacklo_epi32(B0.m_simd, B1.m_simd);
647 const __m128i T1 = _mm_unpacklo_epi32(B2.m_simd, B3.m_simd);
648 const __m128i T2 = _mm_unpackhi_epi32(B0.m_simd, B1.m_simd);
649 const __m128i T3 = _mm_unpackhi_epi32(B2.m_simd, B3.m_simd);
650
651 B0.m_simd = _mm_unpacklo_epi64(T0, T1);
652 B1.m_simd = _mm_unpackhi_epi64(T0, T1);
653 B2.m_simd = _mm_unpacklo_epi64(T2, T3);
654 B3.m_simd = _mm_unpackhi_epi64(T2, T3);
655#elif defined(BOTAN_SIMD_USE_VSX)
656 const __vector unsigned int T0 = vec_mergeh(B0.m_simd, B2.m_simd);
657 const __vector unsigned int T1 = vec_mergeh(B1.m_simd, B3.m_simd);
658 const __vector unsigned int T2 = vec_mergel(B0.m_simd, B2.m_simd);
659 const __vector unsigned int T3 = vec_mergel(B1.m_simd, B3.m_simd);
660
661 B0.m_simd = vec_mergeh(T0, T1);
662 B1.m_simd = vec_mergel(T0, T1);
663 B2.m_simd = vec_mergeh(T2, T3);
664 B3.m_simd = vec_mergel(T2, T3);
665
666#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM32)
667 const uint32x4x2_t T0 = vzipq_u32(B0.m_simd, B2.m_simd);
668 const uint32x4x2_t T1 = vzipq_u32(B1.m_simd, B3.m_simd);
669 const uint32x4x2_t O0 = vzipq_u32(T0.val[0], T1.val[0]);
670 const uint32x4x2_t O1 = vzipq_u32(T0.val[1], T1.val[1]);
671
672 B0.m_simd = O0.val[0];
673 B1.m_simd = O0.val[1];
674 B2.m_simd = O1.val[0];
675 B3.m_simd = O1.val[1];
676
677#elif defined(BOTAN_SIMD_USE_NEON) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
678 const uint32x4_t T0 = vzip1q_u32(B0.m_simd, B2.m_simd);
679 const uint32x4_t T2 = vzip2q_u32(B0.m_simd, B2.m_simd);
680 const uint32x4_t T1 = vzip1q_u32(B1.m_simd, B3.m_simd);
681 const uint32x4_t T3 = vzip2q_u32(B1.m_simd, B3.m_simd);
682
683 B0.m_simd = vzip1q_u32(T0, T1);
684 B1.m_simd = vzip2q_u32(T0, T1);
685 B2.m_simd = vzip1q_u32(T2, T3);
686 B3.m_simd = vzip2q_u32(T2, T3);
687#elif defined(BOTAN_SIMD_USE_LSX)
688 const __m128i T0 = __lsx_vilvl_w(B2.raw(), B0.raw());
689 const __m128i T1 = __lsx_vilvh_w(B2.raw(), B0.raw());
690 const __m128i T2 = __lsx_vilvl_w(B3.raw(), B1.raw());
691 const __m128i T3 = __lsx_vilvh_w(B3.raw(), B1.raw());
692 B0.m_simd = __lsx_vilvl_w(T2, T0);
693 B1.m_simd = __lsx_vilvh_w(T2, T0);
694 B2.m_simd = __lsx_vilvl_w(T3, T1);
695 B3.m_simd = __lsx_vilvh_w(T3, T1);
696#elif defined(BOTAN_SIMD_USE_SIMD128)
697 const auto T0 = wasm_i32x4_shuffle(B0.m_simd, B2.m_simd, 0, 4, 1, 5);
698 const auto T2 = wasm_i32x4_shuffle(B0.m_simd, B2.m_simd, 2, 6, 3, 7);
699 const auto T1 = wasm_i32x4_shuffle(B1.m_simd, B3.m_simd, 0, 4, 1, 5);
700 const auto T3 = wasm_i32x4_shuffle(B1.m_simd, B3.m_simd, 2, 6, 3, 7);
701
702 B0.m_simd = wasm_i32x4_shuffle(T0, T1, 0, 4, 1, 5);
703 B1.m_simd = wasm_i32x4_shuffle(T0, T1, 2, 6, 3, 7);
704 B2.m_simd = wasm_i32x4_shuffle(T2, T3, 0, 4, 1, 5);
705 B3.m_simd = wasm_i32x4_shuffle(T2, T3, 2, 6, 3, 7);
706#endif
707 }
708
709 /**
710 * Unsigned lane comparison; returns a mask with all bits set in each
711 * 32-bit lane that is (unsigned) less than the corresponding lane of
712 * @p other, and all bits cleared otherwise.
713 */
714 SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 unsigned_lt(const SIMD_4x32& other) const noexcept {
715#if defined(BOTAN_SIMD_USE_SSSE3)
716 // No unsigned comparison before AVX-512; bias into the signed domain
717 const __m128i bias = _mm_set1_epi32(static_cast<int32_t>(0x80000000));
718 return SIMD_4x32(_mm_cmpgt_epi32(_mm_xor_si128(other.raw(), bias), _mm_xor_si128(raw(), bias)));
719#elif defined(BOTAN_SIMD_USE_VSX)
720 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(vec_cmplt(raw(), other.raw())));
721#elif defined(BOTAN_SIMD_USE_NEON)
722 return SIMD_4x32(vcltq_u32(raw(), other.raw()));
723#elif defined(BOTAN_SIMD_USE_LSX)
724 return SIMD_4x32(__lsx_vslt_wu(raw(), other.raw()));
725#elif defined(BOTAN_SIMD_USE_SIMD128)
726 return SIMD_4x32(wasm_u32x4_lt(raw(), other.raw()));
727#endif
728 }
729
730 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 choose(const SIMD_4x32& mask,
731 const SIMD_4x32& a,
732 const SIMD_4x32& b) noexcept {
733#if defined(BOTAN_SIMD_USE_VSX)
734 return SIMD_4x32(vec_sel(b.raw(), a.raw(), mask.raw()));
735#elif defined(BOTAN_SIMD_USE_NEON)
736 return SIMD_4x32(vbslq_u32(mask.raw(), a.raw(), b.raw()));
737#elif defined(BOTAN_SIMD_USE_LSX)
738 return SIMD_4x32(__lsx_vbitsel_v(b.raw(), a.raw(), mask.raw()));
739#elif defined(BOTAN_SIMD_USE_SIMD128)
740 return SIMD_4x32(wasm_v128_bitselect(a.raw(), b.raw(), mask.raw()));
741#else
742 return (mask & a) ^ mask.andc(b);
743#endif
744 }
745
746 /**
747 * Byte-granularity blend: for each byte position, select from @p a where
748 * the corresponding mask byte is 0xFF, or from @p b where it is 0x00.
749 *
750 * Each byte of @p mask must be either 0x00 or 0xFF; other values produce
751 * undefined results.
752 */
753 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_blend(const SIMD_4x32& mask,
754 const SIMD_4x32& a,
755 const SIMD_4x32& b) noexcept {
756 // TODO(SSE4.1) could use _mm_blendv_epi8 here
757 return SIMD_4x32::choose(mask, a, b);
758 }
759
760 /**
761 * Byte-granularity blend: for each byte position, select from @p a where
762 * the corresponding mask byte is 0xFF, or from @p b where it is 0x00.
763 *
764 * Each byte of @p mask must be either 0x00 or 0xFF; other values produce
765 * undefined results.
766 */
767 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_blend(uint32_t mask,
768 const SIMD_4x32& a,
769 const SIMD_4x32& b) noexcept {
770 return SIMD_4x32::byte_blend(SIMD_4x32::splat(mask), a, b);
771 }
772
773 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 majority(const SIMD_4x32& x,
774 const SIMD_4x32& y,
775 const SIMD_4x32& z) noexcept {
776#if defined(BOTAN_SIMD_USE_VSX) && defined(_ARCH_PWR10)
777 // vec_ternarylogic is a _Generic macro in Clang, which warns in C++
778 using u64x2 = __vector unsigned long long;
779 const auto r = __builtin_vsx_xxeval(
780 reinterpret_cast<u64x2>(x.raw()), reinterpret_cast<u64x2>(y.raw()), reinterpret_cast<u64x2>(z.raw()), 0x17);
781 return SIMD_4x32(reinterpret_cast<native_simd_type>(r));
782#else
783 return SIMD_4x32::choose(x ^ y, z, y);
784#endif
785 }
786
787 /**
788 * Byte shuffle
789 *
790 * Bytes are numbered in little-endian order (as written by store_le)
791 * regardless of the native endianness.
792 *
793 * This function assumes that each byte of idx is <= 16; it may produce incorrect
794 * results if this does not hold.
795 */
796 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_shuffle(const SIMD_4x32& tbl, const SIMD_4x32& idx) {
797#if defined(BOTAN_SIMD_USE_SSSE3)
798 return SIMD_4x32(_mm_shuffle_epi8(tbl.raw(), idx.raw()));
799#elif defined(BOTAN_SIMD_USE_NEON)
800 const uint8x16_t tbl8 = vreinterpretq_u8_u32(tbl.raw());
801 const uint8x16_t idx8 = vreinterpretq_u8_u32(idx.raw());
802
803 #if defined(BOTAN_TARGET_ARCH_IS_ARM32)
804 const uint8x8x2_t tbl2 = {vget_low_u8(tbl8), vget_high_u8(tbl8)};
805
806 return SIMD_4x32(
807 vreinterpretq_u32_u8(vcombine_u8(vtbl2_u8(tbl2, vget_low_u8(idx8)), vtbl2_u8(tbl2, vget_high_u8(idx8)))));
808 #else
809 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(tbl8, idx8)));
810 #endif
811
812#elif defined(BOTAN_SIMD_USE_VSX)
813 if constexpr(std::endian::native == std::endian::big) {
814 // vec_perm numbers the bytes in big-endian order
815 return SIMD_4x32::altivec_perm(tbl.bswap(), idx.bswap()).bswap();
816 } else {
817 return SIMD_4x32::altivec_perm(tbl, idx);
818 }
819#elif defined(BOTAN_SIMD_USE_LSX)
820 return SIMD_4x32(__lsx_vshuf_b(tbl.raw(), tbl.raw(), idx.raw()));
821#elif defined(BOTAN_SIMD_USE_SIMD128)
822 return SIMD_4x32(wasm_i8x16_swizzle(tbl.raw(), idx.raw()));
823#endif
824 }
825
826 /**
827 * Byte shuffle with masking
828 *
829 * Bytes are numbered as in byte_shuffle. If the index is >= 128 then the
830 * output byte is set to zero.
831 *
832 * Warning: for indices between 16 and 128 this function may have different
833 * behaviors depending on the CPU; possibly the output is zero, tbl[idx % 16],
834 * or even undefined.
835 */
836 inline static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 masked_byte_shuffle(const SIMD_4x32& tbl, const SIMD_4x32& idx) {
837#if defined(BOTAN_SIMD_USE_VSX)
838 if constexpr(std::endian::native == std::endian::big) {
839 // vec_perm numbers the bytes in big-endian order
840 return SIMD_4x32::altivec_masked_perm(tbl.bswap(), idx.bswap()).bswap();
841 } else {
842 return SIMD_4x32::altivec_masked_perm(tbl, idx);
843 }
844#elif defined(BOTAN_SIMD_USE_LSX)
845 /*
846 * The behavior of vshuf.b unfortunately differs among microarchitectures
847 * when the index is larger than the available elements. In LA664 CPUs,
848 * larger indices result in a zero byte, which is exactly what we want.
849 * Unfortunately on LA464 machines, the output is instead undefined.
850 *
851 * So we must use a slower sequence that handles the larger indices.
852 * If we had a way of knowing at compile time that we are on an LA664
853 * or later, we could use __lsx_vshuf_b without the comparison or select.
854 */
855 const auto zero = __lsx_vldi(0);
856 const auto r = __lsx_vshuf_b(zero, tbl.raw(), idx.raw());
857 const auto mask = __lsx_vslti_bu(idx.raw(), 16);
858 return SIMD_4x32(__lsx_vbitsel_v(zero, r, mask));
859#else
860 // ARM, x86 and Wasm byte shuffles have the behavior we want for out of range idx
861 return SIMD_4x32::byte_shuffle(tbl, idx);
862#endif
863 }
864
865 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 alignr4(const SIMD_4x32& a, const SIMD_4x32& b) {
866#if defined(BOTAN_SIMD_USE_SSSE3)
867 return SIMD_4x32(_mm_alignr_epi8(a.raw(), b.raw(), 4));
868#elif defined(BOTAN_SIMD_USE_NEON)
869 return SIMD_4x32(vextq_u32(b.raw(), a.raw(), 1));
870#elif defined(BOTAN_SIMD_USE_VSX)
871 const __vector unsigned char mask = {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
872 return SIMD_4x32(vec_perm(b.raw(), a.raw(), mask));
873#elif defined(BOTAN_SIMD_USE_LSX)
874 const auto mask = SIMD_4x32(0x07060504, 0x0B0A0908, 0x0F0E0D0C, 0x13121110);
875 return SIMD_4x32(__lsx_vshuf_b(a.raw(), b.raw(), mask.raw()));
876#elif defined(BOTAN_SIMD_USE_SIMD128)
877 return SIMD_4x32(
878 wasm_i8x16_shuffle(b.raw(), a.raw(), 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19));
879#endif
880 }
881
882 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 alignr8(const SIMD_4x32& a, const SIMD_4x32& b) {
883#if defined(BOTAN_SIMD_USE_SSSE3)
884 return SIMD_4x32(_mm_alignr_epi8(a.raw(), b.raw(), 8));
885#elif defined(BOTAN_SIMD_USE_NEON)
886 return SIMD_4x32(vextq_u32(b.raw(), a.raw(), 2));
887#elif defined(BOTAN_SIMD_USE_VSX)
888 const __vector unsigned char mask = {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23};
889 return SIMD_4x32(vec_perm(b.raw(), a.raw(), mask));
890#elif defined(BOTAN_SIMD_USE_LSX)
891 return SIMD_4x32(__lsx_vshuf4i_d(a.raw(), b.raw(), 0b0011));
892#elif defined(BOTAN_SIMD_USE_SIMD128)
893 return SIMD_4x32(
894 wasm_i8x16_shuffle(b.raw(), a.raw(), 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23));
895#endif
896 }
897
898 /**
899 If the topmost bit of x is set, return a vector of all ones, otherwise a vector of all zeros
900 ie: (v >> 127) ? splat(0xFFFFFFFF) : zero;
901
902 Most of the implementations work by doing an arithmetic shift of 31 to smear the top bits
903 of each word, followed by a broadcast of the top word.
904 */
905 inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 top_bit_mask() const {
906#if defined(BOTAN_SIMD_USE_SSSE3)
907 return SIMD_4x32(_mm_shuffle_epi32(_mm_srai_epi32(raw(), 31), 0b11111111));
908#elif defined(BOTAN_SIMD_USE_NEON)
909 #if defined(BOTAN_TARGET_ARCH_IS_ARM32)
910 int32x4_t v = vshrq_n_s32(vreinterpretq_s32_u32(raw()), 31);
911 int32x2_t hi = vget_high_s32(v);
912 return SIMD_4x32(vreinterpretq_u32_s32(vdupq_lane_s32(hi, 1)));
913 #else
914 return SIMD_4x32(vreinterpretq_u32_s32(vdupq_laneq_s32(vshrq_n_s32(vreinterpretq_s32_u32(raw()), 31), 3)));
915 #endif
916#elif defined(BOTAN_SIMD_USE_VSX)
917 const __vector unsigned int shift = vec_splats(31U);
918 const __vector signed int shifted = vec_sra(reinterpret_cast<__vector signed int>(raw()), shift);
919 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(vec_splat(shifted, 3)));
920#elif defined(BOTAN_SIMD_USE_LSX)
921 return SIMD_4x32(__lsx_vshuf4i_w(__lsx_vsrai_w(raw(), 31), 0xFF));
922#elif defined(BOTAN_SIMD_USE_SIMD128)
923 return SIMD_4x32(wasm_i32x4_splat(wasm_i32x4_extract_lane(wasm_i32x4_shr(raw(), 31), 3)));
924#endif
925 }
926
927 /**
928 * Swap the upper and lower 64-bit halves of the vector
929 */
930 inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 swap_halves() const {
931#if defined(BOTAN_SIMD_USE_SSSE3)
932 return SIMD_4x32(_mm_shuffle_epi32(raw(), 0b01001110));
933#else
934 return SIMD_4x32::alignr8(*this, *this);
935#endif
936 }
937
938 /**
939 * Extract a single word from the register
940 */
941 template <size_t W>
942 inline uint32_t BOTAN_FN_ISA_SIMD_4X32 extract_word() const noexcept {
943 static_assert(W <= 3);
944
945#if defined(BOTAN_SIMD_USE_NEON)
946 return vgetq_lane_u32(raw(), W);
947#elif defined(BOTAN_SIMD_USE_VSX)
948 return vec_extract(raw(), W);
949#elif defined(BOTAN_SIMD_USE_LSX)
950 return __lsx_vpickve2gr_wu(raw(), W);
951#elif defined(BOTAN_SIMD_USE_SIMD128)
952 return wasm_u32x4_extract_lane(raw(), W);
953#elif defined(BOTAN_SIMD_USE_SSSE3)
954 if constexpr(W == 0) {
955 return static_cast<uint32_t>(_mm_cvtsi128_si32(raw()));
956 } else {
957 // Broadcast the word then extract the low
958 // TODO(SSE4.1) could use _mm_extract_epi32 here
959 return static_cast<uint32_t>(_mm_cvtsi128_si32(_mm_shuffle_epi32(raw(), _MM_SHUFFLE(W, W, W, W))));
960 }
961#endif
962 }
963
964 native_simd_type BOTAN_FN_ISA_SIMD_4X32 raw() const noexcept { return m_simd; }
965
966 explicit BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32(native_simd_type x) noexcept : m_simd(x) {}
967
968 private:
969#if defined(BOTAN_SIMD_USE_VSX)
970 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 altivec_perm(const SIMD_4x32& tbl, const SIMD_4x32& idx) {
971 const auto r = vec_perm(reinterpret_cast<__vector signed char>(tbl.raw()),
972 reinterpret_cast<__vector signed char>(tbl.raw()),
973 reinterpret_cast<__vector unsigned char>(idx.raw()));
974 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(r));
975 }
976
977 static inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 altivec_masked_perm(const SIMD_4x32& tbl, const SIMD_4x32& idx) {
978 const auto zero = vec_splat_s8(0x00);
979 const auto mask = vec_cmplt(reinterpret_cast<__vector signed char>(idx.raw()), zero);
980 const auto r = vec_perm(reinterpret_cast<__vector signed char>(tbl.raw()),
981 reinterpret_cast<__vector signed char>(tbl.raw()),
982 reinterpret_cast<__vector unsigned char>(idx.raw()));
983 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(vec_sel(r, zero, mask)));
984 }
985#endif
986
987 native_simd_type m_simd;
988};
989
990// NOLINTEND(portability-simd-intrinsics)
991
992template <size_t R>
993inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 rotl(SIMD_4x32 input) {
994 return input.rotl<R>();
995}
996
997template <size_t R>
998inline SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 rotr(SIMD_4x32 input) {
999 return input.rotr<R>();
1000}
1001
1002// For Serpent:
1003template <size_t S>
1004inline SIMD_4x32 shl(SIMD_4x32 input) {
1005 return input.shl<S>();
1006}
1007
1008} // namespace Botan
1009
1010#endif
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept
Definition simd_4x32.h:210
uint32_t BOTAN_FN_ISA_SIMD_4X32 extract_word() const noexcept
Definition simd_4x32.h:942
SIMD_4x32(SIMD_4x32 &&other)=default
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator+(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:352
SIMD_4x32 & operator=(SIMD_4x32 &&other)=default
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 top_bit_mask() const
Definition simd_4x32.h:905
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 majority(const SIMD_4x32 &x, const SIMD_4x32 &y, const SIMD_4x32 &z) noexcept
Definition simd_4x32.h:773
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(const void *in) noexcept
Definition simd_4x32.h:194
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_native(const void *in) noexcept
Definition simd_4x32.h:166
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shr() const noexcept
Definition simd_4x32.h:485
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:183
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shl() const noexcept
Definition simd_4x32.h:467
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32(uint32_t B0, uint32_t B1, uint32_t B2, uint32_t B3) noexcept
Definition simd_4x32.h:99
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 reverse_all_bytes() const
Definition simd_4x32.h:536
void BOTAN_FN_ISA_SIMD_4X32 operator+=(const SIMD_4x32 &other) noexcept
Definition simd_4x32.h:394
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_shuffle(const SIMD_4x32 &tbl, const SIMD_4x32 &idx)
Definition simd_4x32.h:796
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 alignr4(const SIMD_4x32 &a, const SIMD_4x32 &b)
Definition simd_4x32.h:865
native_simd_type BOTAN_FN_ISA_SIMD_4X32 raw() const noexcept
Definition simd_4x32.h:964
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator~() const noexcept
Definition simd_4x32.h:500
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_blend(const SIMD_4x32 &mask, const SIMD_4x32 &a, const SIMD_4x32 &b) noexcept
Definition simd_4x32.h:753
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator&(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:388
void BOTAN_FN_ISA_SIMD_4X32 store_le(std::span< uint8_t, 16 > out) const
Definition simd_4x32.h:263
void BOTAN_FN_ISA_SIMD_4X32 operator|=(const SIMD_4x32 &other) noexcept
Definition simd_4x32.h:438
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32(native_simd_type x) noexcept
Definition simd_4x32.h:966
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_right() const noexcept
Definition simd_4x32.h:599
SIMD_4x32(const SIMD_4x32 &other)=default
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator-(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:361
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator^(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:370
~SIMD_4x32()=default
void BOTAN_FN_ISA_SIMD_4X32 operator&=(const SIMD_4x32 &other) noexcept
Definition simd_4x32.h:452
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 alignr8(const SIMD_4x32 &a, const SIMD_4x32 &b)
Definition simd_4x32.h:882
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 choose(const SIMD_4x32 &mask, const SIMD_4x32 &a, const SIMD_4x32 &b) noexcept
Definition simd_4x32.h:730
void BOTAN_FN_ISA_SIMD_4X32 store_native(void *out) const noexcept
Definition simd_4x32.h:225
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 sigma1() const noexcept
Definition simd_4x32.h:282
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat(uint32_t B) noexcept
Definition simd_4x32.h:123
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 rotr() const noexcept
Definition simd_4x32.h:345
void BOTAN_FN_ISA_SIMD_4X32 operator-=(const SIMD_4x32 &other) noexcept
Definition simd_4x32.h:408
static void BOTAN_FN_ISA_SIMD_4X32 transpose(SIMD_4x32 &B0, SIMD_4x32 &B1, SIMD_4x32 &B2, SIMD_4x32 &B3) noexcept
Definition simd_4x32.h:641
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 unsigned_lt(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:714
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_blend(uint32_t mask, const SIMD_4x32 &a, const SIMD_4x32 &b) noexcept
Definition simd_4x32.h:767
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 operator|(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:379
void BOTAN_FN_ISA_SIMD_4X32 store_be(uint8_t out[]) const noexcept
Definition simd_4x32.h:253
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 andc(const SIMD_4x32 &other) const noexcept
Definition simd_4x32.h:515
void BOTAN_FN_ISA_SIMD_4X32 operator^=(const SIMD_4x32 &other) noexcept
Definition simd_4x32.h:422
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32() noexcept
Definition simd_4x32.h:82
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 masked_byte_shuffle(const SIMD_4x32 &tbl, const SIMD_4x32 &idx)
Definition simd_4x32.h:836
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint8_t out[]) const noexcept
Definition simd_4x32.h:242
void BOTAN_FN_ISA_SIMD_4X32 store_be(uint32_t out[4]) const noexcept
Definition simd_4x32.h:214
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat_u8(uint8_t B) noexcept
Definition simd_4x32.h:140
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 bswap() const noexcept
Definition simd_4x32.h:544
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_left() const noexcept
Definition simd_4x32.h:562
SIMD_4x32 & operator=(const SIMD_4x32 &other)=default
BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 rotl() const noexcept
Definition simd_4x32.h:297
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(std::span< const uint8_t, 16 > in)
Definition simd_4x32.h:202
void BOTAN_FN_ISA_SIMD_4X32 store_be(std::span< uint8_t, 16 > out) const
Definition simd_4x32.h:261
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(std::span< const uint8_t, 16 > in)
Definition simd_4x32.h:206
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint64_t out[2]) const noexcept
Definition simd_4x32.h:218
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 sigma0() const noexcept
Definition simd_4x32.h:268
void BOTAN_FN_ISA_SIMD_4X32 operator^=(uint32_t other) noexcept
Definition simd_4x32.h:436
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 swap_halves() const
Definition simd_4x32.h:930
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35
BOTAN_FORCE_INLINE constexpr T rotl(T input)
Definition rotate.h:23
SIMD_4x32 shl(SIMD_4x32 input)
Definition simd_4x32.h:1004