Botan 3.14.1
Crypto and TLS for C&
simd_2x64.h
Go to the documentation of this file.
1/*
2* (C) 2022,2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#ifndef BOTAN_SIMD_2X64_H_
8#define BOTAN_SIMD_2X64_H_
9
10#include <botan/types.h>
11#include <botan/internal/isa_extn.h>
12#include <botan/internal/target_info.h>
13#include <bit>
14#include <span>
15
16// TODO: extend this to support LSX
17
18#if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3)
19 #include <tmmintrin.h>
20 #define BOTAN_SIMD_USE_SSSE3
21#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_NEON)
22 #include <arm_neon.h>
23 #define BOTAN_SIMD_USE_NEON
24#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_SIMD128)
25 #include <wasm_simd128.h>
26 #define BOTAN_SIMD_USE_SIMD128
27#elif defined(BOTAN_TARGET_ARCH_SUPPORTS_VSX)
28 #include <altivec.h>
29 #undef vector
30 #undef bool
31 #define BOTAN_SIMD_USE_VSX
32#endif
33
34namespace Botan {
35
36// NOLINTBEGIN(portability-simd-intrinsics)
37
38class SIMD_2x64 final {
39 public:
40#if defined(BOTAN_SIMD_USE_SSSE3)
41 using native_simd_type = __m128i;
42#elif defined(BOTAN_SIMD_USE_NEON)
43 using native_simd_type = uint64x2_t;
44#elif defined(BOTAN_SIMD_USE_SIMD128)
45 using native_simd_type = v128_t;
46#elif defined(BOTAN_SIMD_USE_VSX)
47 using native_simd_type = __vector unsigned long long;
48#endif
49
50 SIMD_2x64& operator=(const SIMD_2x64& other) = default;
51 SIMD_2x64(const SIMD_2x64& other) = default;
52
53 SIMD_2x64& operator=(SIMD_2x64&& other) = default;
54 SIMD_2x64(SIMD_2x64&& other) = default;
55
56 ~SIMD_2x64() = default;
57
58 // zero initialized
59 BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64() :
60#if defined(BOTAN_SIMD_USE_SSSE3)
61 m_simd(_mm_setzero_si128())
62#elif defined(BOTAN_SIMD_USE_NEON)
63 m_simd(vdupq_n_u64(0))
64#elif defined(BOTAN_SIMD_USE_SIMD128)
65 m_simd(wasm_u64x2_const_splat(0))
66#elif defined(BOTAN_SIMD_USE_VSX)
67 m_simd(vec_splats(static_cast<unsigned long long>(0)))
68#endif
69 {
70 }
71
72 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 splat(uint64_t v) {
73#if defined(BOTAN_SIMD_USE_SSSE3)
74 return SIMD_2x64(_mm_set1_epi64x(v));
75#elif defined(BOTAN_SIMD_USE_NEON)
76 return SIMD_2x64(vdupq_n_u64(v));
77#elif defined(BOTAN_SIMD_USE_SIMD128)
78 return SIMD_2x64(wasm_u64x2_splat(v));
79#elif defined(BOTAN_SIMD_USE_VSX)
80 return SIMD_2x64(vec_splats(static_cast<unsigned long long>(v)));
81#endif
82 }
83
84 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 all_ones() {
85#if defined(BOTAN_SIMD_USE_SSSE3)
86 return SIMD_2x64(_mm_set1_epi8(-1));
87#elif defined(BOTAN_SIMD_USE_NEON)
88 return SIMD_2x64(vdupq_n_u64(~uint64_t(0)));
89#elif defined(BOTAN_SIMD_USE_SIMD128)
90 return SIMD_2x64(wasm_i8x16_splat(0xFF));
91#elif defined(BOTAN_SIMD_USE_VSX)
92 return SIMD_2x64(vec_splats(~static_cast<unsigned long long>(0)));
93#endif
94 }
95
96 BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64(uint64_t low, uint64_t high) :
97#if defined(BOTAN_SIMD_USE_SSSE3)
98 m_simd(_mm_set_epi64x(high, low))
99#elif defined(BOTAN_SIMD_USE_NEON)
100 m_simd(vcombine_u64(vcreate_u64(low), vcreate_u64(high)))
101#elif defined(BOTAN_SIMD_USE_SIMD128)
102 m_simd(wasm_u64x2_make(low, high))
103#elif defined(BOTAN_SIMD_USE_VSX)
104 m_simd(native_simd_type{low, high})
105#endif
106 {
107 }
108
109 /**
110 * Load two 64-bit words stored in native byte order
111 */
112 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_native(const void* in) {
113#if defined(BOTAN_SIMD_USE_SSSE3)
114 return SIMD_2x64(_mm_loadu_si128(reinterpret_cast<const __m128i*>(in)));
115#elif defined(BOTAN_SIMD_USE_NEON)
116 return SIMD_2x64(vld1q_u64(static_cast<const uint64_t*>(in)));
117#elif defined(BOTAN_SIMD_USE_SIMD128)
118 return SIMD_2x64(wasm_v128_load(in));
119#elif defined(BOTAN_SIMD_USE_VSX)
120 return SIMD_2x64(reinterpret_cast<native_simd_type>(vec_xl(0, static_cast<const unsigned char*>(in))));
121#endif
122 }
123
124 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void* in) {
125 if constexpr(std::endian::native == std::endian::little) {
126 return SIMD_2x64::load_native(in);
127 } else {
128 return SIMD_2x64::load_native(in).bswap();
129 }
130 }
131
132 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void* in) {
133 if constexpr(std::endian::native == std::endian::big) {
134 return SIMD_2x64::load_native(in);
135 } else {
136 return SIMD_2x64::load_native(in).bswap();
137 }
138 }
139
140 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(std::span<const uint8_t, 16> in) {
141 return SIMD_2x64::load_le(in.data());
142 }
143
144 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(std::span<const uint8_t, 16> in) {
145 return SIMD_2x64::load_be(in.data());
146 }
147
148 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 bswap() const {
149#if defined(BOTAN_SIMD_USE_SSSE3)
150 const auto idx = _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7);
151 return SIMD_2x64(_mm_shuffle_epi8(m_simd, idx));
152#elif defined(BOTAN_SIMD_USE_NEON)
153 return SIMD_2x64(vreinterpretq_u64_u8(vrev64q_u8(vreinterpretq_u8_u64(m_simd))));
154#elif defined(BOTAN_SIMD_USE_SIMD128)
155 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8));
156#elif defined(BOTAN_SIMD_USE_VSX)
157 return SIMD_2x64(vec_revb(m_simd));
158#endif
159 }
160
161 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 swap_lanes() const {
162#if defined(BOTAN_SIMD_USE_SSSE3)
163 return SIMD_2x64(_mm_shuffle_epi32(m_simd, _MM_SHUFFLE(1, 0, 3, 2)));
164#elif defined(BOTAN_SIMD_USE_NEON)
165 return SIMD_2x64(vextq_u64(m_simd, m_simd, 1));
166#elif defined(BOTAN_SIMD_USE_SIMD128)
167 return SIMD_2x64(wasm_i64x2_shuffle(m_simd, m_simd, 1, 0));
168#elif defined(BOTAN_SIMD_USE_VSX)
169 const __vector unsigned char mask = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
170 return SIMD_2x64(vec_perm(m_simd, m_simd, mask));
171#endif
172 }
173
174 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 reverse_all_bytes() const {
175#if defined(BOTAN_SIMD_USE_SSSE3)
176 const auto idx = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
177 return SIMD_2x64(_mm_shuffle_epi8(m_simd, idx));
178#elif defined(BOTAN_SIMD_USE_NEON) || defined(BOTAN_SIMD_USE_VSX)
179 return this->bswap().swap_lanes();
180#elif defined(BOTAN_SIMD_USE_SIMD128)
181 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
182#endif
183 }
184
185 /**
186 * Store two 64-bit words in native byte order
187 */
188 void BOTAN_FN_ISA_SIMD_2X64 store_native(void* out) const {
189#if defined(BOTAN_SIMD_USE_SSSE3)
190 _mm_storeu_si128(static_cast<__m128i*>(out), m_simd);
191#elif defined(BOTAN_SIMD_USE_NEON)
192 vst1q_u64(static_cast<uint64_t*>(out), m_simd);
193#elif defined(BOTAN_SIMD_USE_SIMD128)
194 wasm_v128_store(out, m_simd);
195#elif defined(BOTAN_SIMD_USE_VSX)
196 vec_xst(reinterpret_cast<__vector unsigned char>(m_simd), 0, static_cast<unsigned char*>(out));
197#endif
198 }
199
200 void BOTAN_FN_ISA_SIMD_2X64 store_le(uint64_t out[2]) const { this->store_le(reinterpret_cast<uint8_t*>(out)); }
201
202 void BOTAN_FN_ISA_SIMD_2X64 store_le(uint8_t out[]) const {
203 if constexpr(std::endian::native == std::endian::little) {
204 this->store_native(out);
205 } else {
206 this->bswap().store_native(out);
207 }
208 }
209
210 void BOTAN_FN_ISA_SIMD_2X64 store_be(uint64_t out[2]) const { this->store_be(reinterpret_cast<uint8_t*>(out)); }
211
212 void BOTAN_FN_ISA_SIMD_2X64 store_be(uint8_t out[]) const {
213 if constexpr(std::endian::native == std::endian::big) {
214 this->store_native(out);
215 } else {
216 this->bswap().store_native(out);
217 }
218 }
219
220 void BOTAN_FN_ISA_SIMD_2X64 store_be(std::span<uint8_t, 16> out) const { this->store_be(out.data()); }
221
222 void BOTAN_FN_ISA_SIMD_2X64 store_le(std::span<uint8_t, 16> out) const { this->store_le(out.data()); }
223
224 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator+(const SIMD_2x64& other) const {
225 SIMD_2x64 retval(*this);
226 retval += other;
227 return retval;
228 }
229
230 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator^(const SIMD_2x64& other) const {
231 SIMD_2x64 retval(*this);
232 retval ^= other;
233 return retval;
234 }
235
236 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator&(const SIMD_2x64& other) const {
237 SIMD_2x64 retval(*this);
238 retval &= other;
239 return retval;
240 }
241
242 void BOTAN_FN_ISA_SIMD_2X64 operator&=(const SIMD_2x64& other) {
243#if defined(BOTAN_SIMD_USE_SSSE3)
244 m_simd = _mm_and_si128(m_simd, other.m_simd);
245#elif defined(BOTAN_SIMD_USE_NEON)
246 m_simd = vandq_u64(m_simd, other.m_simd);
247#elif defined(BOTAN_SIMD_USE_SIMD128)
248 m_simd = wasm_v128_and(m_simd, other.m_simd);
249#elif defined(BOTAN_SIMD_USE_VSX)
250 m_simd = vec_and(m_simd, other.m_simd);
251#endif
252 }
253
254 void BOTAN_FN_ISA_SIMD_2X64 operator+=(const SIMD_2x64& other) {
255#if defined(BOTAN_SIMD_USE_SSSE3)
256 m_simd = _mm_add_epi64(m_simd, other.m_simd);
257#elif defined(BOTAN_SIMD_USE_NEON)
258 m_simd = vaddq_u64(m_simd, other.m_simd);
259#elif defined(BOTAN_SIMD_USE_SIMD128)
260 m_simd = wasm_i64x2_add(m_simd, other.m_simd);
261#elif defined(BOTAN_SIMD_USE_VSX)
262 m_simd = vec_add(m_simd, other.m_simd);
263#endif
264 }
265
266 void BOTAN_FN_ISA_SIMD_2X64 operator^=(const SIMD_2x64& other) {
267#if defined(BOTAN_SIMD_USE_SSSE3)
268 m_simd = _mm_xor_si128(m_simd, other.m_simd);
269#elif defined(BOTAN_SIMD_USE_NEON)
270 m_simd = veorq_u64(m_simd, other.m_simd);
271#elif defined(BOTAN_SIMD_USE_SIMD128)
272 m_simd = wasm_v128_xor(m_simd, other.m_simd);
273#elif defined(BOTAN_SIMD_USE_VSX)
274 m_simd = vec_xor(m_simd, other.m_simd);
275#endif
276 }
277
278 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 andc(const SIMD_2x64& other) const noexcept {
279#if defined(BOTAN_SIMD_USE_SSSE3)
280 return SIMD_2x64(_mm_andnot_si128(m_simd, other.m_simd));
281#elif defined(BOTAN_SIMD_USE_NEON)
282 // NEON vbic is a & ~b
283 return SIMD_2x64(vbicq_u64(other.m_simd, m_simd));
284#elif defined(BOTAN_SIMD_USE_SIMD128)
285 // SIMD128 is a & ~b
286 return SIMD_2x64(wasm_v128_andnot(other.m_simd, m_simd));
287#elif defined(BOTAN_SIMD_USE_VSX)
288 // VSX is a & ~b
289 return SIMD_2x64(vec_andc(other.m_simd, m_simd));
290#endif
291 }
292
293 template <size_t ROT>
294 BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64 rotr() const
295 requires(ROT > 0 && ROT < 64)
296 {
297#if defined(BOTAN_SIMD_USE_SSSE3)
298 if constexpr(ROT == 8) {
299 const auto tab = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 0, 9, 10, 11, 12, 13, 14, 15, 8);
300 return SIMD_2x64(_mm_shuffle_epi8(m_simd, tab));
301 } else if constexpr(ROT == 16) {
302 const auto tab = _mm_setr_epi8(2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9);
303 return SIMD_2x64(_mm_shuffle_epi8(m_simd, tab));
304 } else if constexpr(ROT == 24) {
305 const auto tab = _mm_setr_epi8(3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10);
306 return SIMD_2x64(_mm_shuffle_epi8(m_simd, tab));
307 } else if constexpr(ROT == 32) {
308 const auto tab = _mm_setr_epi8(4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11);
309 return SIMD_2x64(_mm_shuffle_epi8(m_simd, tab));
310 } else {
311 return SIMD_2x64(_mm_or_si128(_mm_srli_epi64(m_simd, static_cast<int>(ROT)),
312 _mm_slli_epi64(m_simd, static_cast<int>(64 - ROT))));
313 }
314#elif defined(BOTAN_SIMD_USE_NEON)
315 return SIMD_2x64(vsriq_n_u64(vshlq_n_u64(m_simd, 64 - ROT), m_simd, ROT));
316#elif defined(BOTAN_SIMD_USE_SIMD128)
317 if constexpr(ROT == 8) {
318 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 1, 2, 3, 4, 5, 6, 7, 0, 9, 10, 11, 12, 13, 14, 15, 8));
319 } else if constexpr(ROT == 16) {
320 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9));
321 } else if constexpr(ROT == 24) {
322 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10));
323 } else if constexpr(ROT == 32) {
324 return SIMD_2x64(wasm_i8x16_shuffle(m_simd, m_simd, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11));
325 } else {
326 return SIMD_2x64(wasm_v128_or(wasm_u64x2_shr(m_simd, ROT), wasm_i64x2_shl(m_simd, 64 - ROT)));
327 }
328#elif defined(BOTAN_SIMD_USE_VSX) && defined(__clang__)
329 // Clang only declares the 64-bit vec_rl if POWER8 is enabled for the whole TU
330 using s64x2 = __vector signed long long;
331 const auto rot = vec_splats(static_cast<unsigned long long>(64 - ROT));
332 const auto r = __builtin_altivec_vrld(reinterpret_cast<s64x2>(m_simd), rot);
333 return SIMD_2x64(reinterpret_cast<native_simd_type>(r));
334#elif defined(BOTAN_SIMD_USE_VSX)
335 return SIMD_2x64(vec_rl(m_simd, vec_splats(static_cast<unsigned long long>(64 - ROT))));
336#endif
337 }
338
339 template <size_t ROT>
340 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 rotl() const {
341 return this->rotr<64 - ROT>();
342 }
343
344 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 sigma0() const {
345 const SIMD_2x64 r1 = this->rotr<28>();
346 const SIMD_2x64 r2 = this->rotr<34>();
347 const SIMD_2x64 r3 = this->rotr<39>();
348 return r1 ^ r2 ^ r3;
349 }
350
351 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 sigma1() const {
352 const SIMD_2x64 r1 = this->rotr<14>();
353 const SIMD_2x64 r2 = this->rotr<18>();
354 const SIMD_2x64 r3 = this->rotr<41>();
355 return r1 ^ r2 ^ r3;
356 }
357
358 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 choose(const SIMD_2x64& mask, const SIMD_2x64& a, const SIMD_2x64& b) {
359#if defined(BOTAN_SIMD_USE_VSX)
360 return SIMD_2x64(vec_sel(b.m_simd, a.m_simd, mask.m_simd));
361#else
362 return (mask & a) ^ mask.andc(b);
363#endif
364 }
365
366 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 majority(const SIMD_2x64& x, const SIMD_2x64& y, const SIMD_2x64& z) {
367#if defined(BOTAN_SIMD_USE_VSX) && defined(_ARCH_PWR10)
368 // vec_ternarylogic is a _Generic macro in Clang, which warns in C++
369 return SIMD_2x64(__builtin_vsx_xxeval(x.m_simd, y.m_simd, z.m_simd, 0x17));
370#else
371 return SIMD_2x64::choose(x ^ y, z, y);
372#endif
373 }
374
375 template <int SHIFT>
376 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 shr() const noexcept {
377#if defined(BOTAN_SIMD_USE_SSSE3)
378 return SIMD_2x64(_mm_srli_epi64(m_simd, SHIFT));
379#elif defined(BOTAN_SIMD_USE_NEON)
380 return SIMD_2x64(vshrq_n_u64(m_simd, SHIFT));
381#elif defined(BOTAN_SIMD_USE_SIMD128)
382 return SIMD_2x64(wasm_u64x2_shr(m_simd, SHIFT));
383#elif defined(BOTAN_SIMD_USE_VSX)
384 return SIMD_2x64(vec_sr(m_simd, vec_splats(static_cast<unsigned long long>(SHIFT))));
385#endif
386 }
387
388 template <int SHIFT>
389 SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 shl() const noexcept {
390#if defined(BOTAN_SIMD_USE_SSSE3)
391 return SIMD_2x64(_mm_slli_epi64(m_simd, SHIFT));
392#elif defined(BOTAN_SIMD_USE_NEON)
393 return SIMD_2x64(vshlq_n_u64(m_simd, SHIFT));
394#elif defined(BOTAN_SIMD_USE_SIMD128)
395 return SIMD_2x64(wasm_i64x2_shl(m_simd, SHIFT));
396#elif defined(BOTAN_SIMD_USE_VSX)
397 return SIMD_2x64(vec_sl(m_simd, vec_splats(static_cast<unsigned long long>(SHIFT))));
398#endif
399 }
400
401 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 alignr8(const SIMD_2x64& a, const SIMD_2x64& b) {
402#if defined(BOTAN_SIMD_USE_SSSE3)
403 return SIMD_2x64(_mm_alignr_epi8(a.m_simd, b.m_simd, 8));
404#elif defined(BOTAN_SIMD_USE_NEON)
405 return SIMD_2x64(vextq_u64(b.m_simd, a.m_simd, 1));
406#elif defined(BOTAN_SIMD_USE_SIMD128)
407 return SIMD_2x64(
408 wasm_i8x16_shuffle(b.m_simd, a.m_simd, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23));
409#elif defined(BOTAN_SIMD_USE_VSX)
410 const __vector unsigned char mask = {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23};
411 return SIMD_2x64(vec_perm(b.m_simd, a.m_simd, mask));
412#endif
413 }
414
415 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 interleave_low(const SIMD_2x64& a, const SIMD_2x64& b) {
416#if defined(BOTAN_SIMD_USE_SSSE3)
417 return SIMD_2x64(_mm_unpacklo_epi64(a.m_simd, b.m_simd));
418#elif defined(BOTAN_SIMD_USE_NEON)
419 return SIMD_2x64(vzip1q_u64(a.m_simd, b.m_simd));
420#elif defined(BOTAN_SIMD_USE_SIMD128)
421 return SIMD_2x64(wasm_u64x2_extract_lane(a.m_simd, 0), wasm_u64x2_extract_lane(b.m_simd, 0));
422#elif defined(BOTAN_SIMD_USE_VSX)
423 const __vector unsigned char mask = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
424 return SIMD_2x64(vec_perm(a.m_simd, b.m_simd, mask));
425#endif
426 }
427
428 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 interleave_high(const SIMD_2x64& a, const SIMD_2x64& b) {
429#if defined(BOTAN_SIMD_USE_SSSE3)
430 return SIMD_2x64(_mm_unpackhi_epi64(a.m_simd, b.m_simd));
431#elif defined(BOTAN_SIMD_USE_NEON)
432 return SIMD_2x64(vzip2q_u64(a.m_simd, b.m_simd));
433#elif defined(BOTAN_SIMD_USE_SIMD128)
434 return SIMD_2x64(wasm_u64x2_extract_lane(a.m_simd, 1), wasm_u64x2_extract_lane(b.m_simd, 1));
435#elif defined(BOTAN_SIMD_USE_VSX)
436 const __vector unsigned char mask = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
437 return SIMD_2x64(vec_perm(a.m_simd, b.m_simd, mask));
438#endif
439 }
440
441 // Argon2 specific operation
442 static void BOTAN_FN_ISA_SIMD_2X64
444 auto T0 = SIMD_2x64::alignr8(B1, B0);
445 auto T1 = SIMD_2x64::alignr8(B0, B1);
446 B0 = T0;
447 B1 = T1;
448
449 T0 = C0;
450 C0 = C1;
451 C1 = T0;
452
453 T0 = SIMD_2x64::alignr8(D0, D1);
454 T1 = SIMD_2x64::alignr8(D1, D0);
455 D0 = T0;
456 D1 = T1;
457 }
458
459 // Argon2 specific operation
460 static void BOTAN_FN_ISA_SIMD_2X64
462 auto T0 = SIMD_2x64::alignr8(B0, B1);
463 auto T1 = SIMD_2x64::alignr8(B1, B0);
464 B0 = T0;
465 B1 = T1;
466
467 T0 = C0;
468 C0 = C1;
469 C1 = T0;
470
471 T0 = SIMD_2x64::alignr8(D1, D0);
472 T1 = SIMD_2x64::alignr8(D0, D1);
473 D0 = T0;
474 D1 = T1;
475 }
476
477 // Argon2 specific operation
478 static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 mul2_32(const SIMD_2x64& x, const SIMD_2x64& y) {
479#if defined(BOTAN_SIMD_USE_SSSE3)
480 const __m128i m = _mm_mul_epu32(x.m_simd, y.m_simd);
481 return SIMD_2x64(_mm_add_epi64(m, m));
482#elif defined(BOTAN_SIMD_USE_NEON)
483 const uint64x2_t m = vmull_u32(vmovn_u64(x.m_simd), vmovn_u64(y.m_simd));
484 return SIMD_2x64(vaddq_u64(m, m));
485#elif defined(BOTAN_SIMD_USE_SIMD128)
486 const auto m = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_shuffle(x.m_simd, x.m_simd, 0, 2, 0, 2),
487 wasm_i32x4_shuffle(y.m_simd, y.m_simd, 0, 2, 0, 2));
488
489 return SIMD_2x64(wasm_i64x2_add(m, m));
490#elif defined(BOTAN_SIMD_USE_VSX)
491 // Multiply the low 32 bits of each 64-bit lane; which 32-bit elements those are depends on endianness
492 const auto x32 = reinterpret_cast<__vector unsigned int>(x.m_simd);
493 const auto y32 = reinterpret_cast<__vector unsigned int>(y.m_simd);
494 #if defined(__clang__)
495 // Clang only declares the 32-bit vec_mule/vec_mulo if POWER8 is enabled for the whole TU;
496 // in big-endian element numbering the low halves are the odd elements
497 const auto m = __builtin_altivec_vmulouw(x32, y32);
498 #else
499 const auto m = (std::endian::native == std::endian::little) ? vec_mule(x32, y32) : vec_mulo(x32, y32);
500 #endif
501 return SIMD_2x64(vec_add(m, m));
502#endif
503 }
504
505 native_simd_type BOTAN_FN_ISA_SIMD_2X64 raw() const noexcept { return m_simd; }
506
507 explicit BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64(native_simd_type x) : m_simd(x) {}
508
509 private:
510 native_simd_type m_simd;
511};
512
513// NOLINTEND(portability-simd-intrinsics)
514
515} // namespace Botan
516
517#endif
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 shr() const noexcept
Definition simd_2x64.h:376
void BOTAN_FN_ISA_SIMD_2X64 store_native(void *out) const
Definition simd_2x64.h:188
void BOTAN_FN_ISA_SIMD_2X64 operator&=(const SIMD_2x64 &other)
Definition simd_2x64.h:242
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(std::span< const uint8_t, 16 > in)
Definition simd_2x64.h:140
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 sigma0() const
Definition simd_2x64.h:344
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_native(const void *in)
Definition simd_2x64.h:112
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator^(const SIMD_2x64 &other) const
Definition simd_2x64.h:230
BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64()
Definition simd_2x64.h:59
native_simd_type BOTAN_FN_ISA_SIMD_2X64 raw() const noexcept
Definition simd_2x64.h:505
void BOTAN_FN_ISA_SIMD_2X64 store_be(uint8_t out[]) const
Definition simd_2x64.h:212
SIMD_2x64 & operator=(const SIMD_2x64 &other)=default
SIMD_2x64(const SIMD_2x64 &other)=default
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(std::span< const uint8_t, 16 > in)
Definition simd_2x64.h:144
void BOTAN_FN_ISA_SIMD_2X64 store_le(uint64_t out[2]) const
Definition simd_2x64.h:200
SIMD_2x64(SIMD_2x64 &&other)=default
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
Definition simd_2x64.h:132
BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64(uint64_t low, uint64_t high)
Definition simd_2x64.h:96
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator&(const SIMD_2x64 &other) const
Definition simd_2x64.h:236
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 reverse_all_bytes() const
Definition simd_2x64.h:174
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 shl() const noexcept
Definition simd_2x64.h:389
void BOTAN_FN_ISA_SIMD_2X64 store_be(std::span< uint8_t, 16 > out) const
Definition simd_2x64.h:220
static void BOTAN_FN_ISA_SIMD_2X64 untwist(SIMD_2x64 &B0, SIMD_2x64 &B1, SIMD_2x64 &C0, SIMD_2x64 &C1, SIMD_2x64 &D0, SIMD_2x64 &D1)
Definition simd_2x64.h:461
BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64(native_simd_type x)
Definition simd_2x64.h:507
~SIMD_2x64()=default
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 swap_lanes() const
Definition simd_2x64.h:161
void BOTAN_FN_ISA_SIMD_2X64 store_be(uint64_t out[2]) const
Definition simd_2x64.h:210
static void BOTAN_FN_ISA_SIMD_2X64 twist(SIMD_2x64 &B0, SIMD_2x64 &B1, SIMD_2x64 &C0, SIMD_2x64 &C1, SIMD_2x64 &D0, SIMD_2x64 &D1)
Definition simd_2x64.h:443
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 andc(const SIMD_2x64 &other) const noexcept
Definition simd_2x64.h:278
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 mul2_32(const SIMD_2x64 &x, const SIMD_2x64 &y)
Definition simd_2x64.h:478
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 sigma1() const
Definition simd_2x64.h:351
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 majority(const SIMD_2x64 &x, const SIMD_2x64 &y, const SIMD_2x64 &z)
Definition simd_2x64.h:366
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 choose(const SIMD_2x64 &mask, const SIMD_2x64 &a, const SIMD_2x64 &b)
Definition simd_2x64.h:358
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 operator+(const SIMD_2x64 &other) const
Definition simd_2x64.h:224
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 all_ones()
Definition simd_2x64.h:84
SIMD_2x64 & operator=(SIMD_2x64 &&other)=default
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
Definition simd_2x64.h:124
void BOTAN_FN_ISA_SIMD_2X64 store_le(std::span< uint8_t, 16 > out) const
Definition simd_2x64.h:222
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 interleave_high(const SIMD_2x64 &a, const SIMD_2x64 &b)
Definition simd_2x64.h:428
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 rotl() const
Definition simd_2x64.h:340
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 interleave_low(const SIMD_2x64 &a, const SIMD_2x64 &b)
Definition simd_2x64.h:415
void BOTAN_FN_ISA_SIMD_2X64 operator+=(const SIMD_2x64 &other)
Definition simd_2x64.h:254
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 splat(uint64_t v)
Definition simd_2x64.h:72
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 alignr8(const SIMD_2x64 &a, const SIMD_2x64 &b)
Definition simd_2x64.h:401
SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 bswap() const
Definition simd_2x64.h:148
BOTAN_FN_ISA_SIMD_2X64 SIMD_2x64 rotr() const
Definition simd_2x64.h:294
void BOTAN_FN_ISA_SIMD_2X64 operator^=(const SIMD_2x64 &other)
Definition simd_2x64.h:266
void BOTAN_FN_ISA_SIMD_2X64 store_le(uint8_t out[]) const
Definition simd_2x64.h:202