Botan 3.14.1
Crypto and TLS for C&
simd_hwaes.h
Go to the documentation of this file.
1/*
2* (C) 2026 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#ifndef BOTAN_SIMD_HWAES_H_
8#define BOTAN_SIMD_HWAES_H_
9
10#include <botan/internal/gfni_utils.h>
11#include <botan/internal/isa_extn.h>
12#include <botan/internal/simd_4x32.h>
13
14#if defined(BOTAN_SIMD_USE_SSSE3)
15 #include <wmmintrin.h>
16#endif
17
18#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
19 #include <bit>
20#endif
21
22namespace Botan {
23
24#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
25
26namespace detail {
27
28 // The vector type taken by the POWER crypto builtins differs between compilers
29 #if defined(__clang__)
30using HwAesVec = __vector unsigned char;
31 #else
32using HwAesVec = __vector unsigned long long;
33 #endif
34
35inline HwAesVec BOTAN_FN_ISA_HWAES hw_aes_vec(SIMD_4x32 x) {
36 return reinterpret_cast<HwAesVec>(x.raw());
37}
38
39inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_simd(HwAesVec x) {
40 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(x));
41}
42
43// POWER numbers the AES state bytes from the most significant end of the register
44inline __vector unsigned char BOTAN_FN_ISA_HWAES hw_aes_state_order(__vector unsigned char v) {
45 if constexpr(std::endian::native == std::endian::little) {
46 const __vector unsigned char rev = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
47 return vec_perm(v, v, rev);
48 } else {
49 return v;
50 }
51}
52
53} // namespace detail
54
55#endif
56
57/**
58* Apply the AES S-box to each byte of the input vector.
59*/
60inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox(SIMD_4x32 x) {
61#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
62 // vsbox is SubBytes alone so no shuffle is required
63 return detail::hw_aes_simd(__builtin_crypto_vsbox(detail::hw_aes_vec(x)));
64#else
65 // Undo the ShiftRows with a byte shuffle implementing InvShiftRows
66 const auto inv_sr = SIMD_4x32(0x070A0D00, 0x0B0E0104, 0x0F020508, 0x0306090C);
67
68 #if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
69 const auto enc = SIMD_4x32(_mm_aesenclast_si128(x.raw(), _mm_setzero_si128()));
70 #elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
71 const auto enc = SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(x.raw()), vdupq_n_u8(0))));
72 #else
73 #error "hw_aes_sbox not implemented for this architecture"
74 #endif
75
76 return SIMD_4x32::byte_shuffle(enc, inv_sr);
77#endif
78}
79
80/**
81* Apply the AES inverse S-box to each byte of the input vector.
82*/
83inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_inv_sbox(SIMD_4x32 x) {
84#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
85 /*
86 * Undo the InvShiftRows with a byte shuffle implementing ShiftRows. POWER
87 * numbers the AES state bytes starting from the most significant byte of the
88 * register, so the indices differ from x86/ARM and depend on endianness.
89 */
90 const auto sr = (std::endian::native == std::endian::little)
91 ? SIMD_4x32(0x030E0904, 0x07020D08, 0x0B06010C, 0x0F0A0500)
92 : SIMD_4x32(0x0306090C, 0x070A0D00, 0x0B0E0104, 0x0F020508);
93#else
94 // Undo the InvShiftRows with a byte shuffle implementing ShiftRows
95 const auto sr = SIMD_4x32(0x0F0A0500, 0x030E0904, 0x07020D08, 0x0B06010C);
96#endif
97
98#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
99 const auto dec = SIMD_4x32(_mm_aesdeclast_si128(x.raw(), _mm_setzero_si128()));
100#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
101 const auto dec = SIMD_4x32(vreinterpretq_u32_u8(vaesdq_u8(vreinterpretq_u8_u32(x.raw()), vdupq_n_u8(0))));
102#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
103 const auto dec =
104 detail::hw_aes_simd(__builtin_crypto_vncipherlast(detail::hw_aes_vec(x), detail::hw_aes_vec(SIMD_4x32())));
105#else
106 #error "hw_aes_inv_sbox not implemented for this architecture"
107#endif
108
109 return SIMD_4x32::byte_shuffle(dec, sr);
110}
111
112/**
113* Compute SubBytes(x) ^ k for a vector x whose four 32-bit words are all equal
114*
115* ShiftRows is the identity on such a vector, so this is exactly what the final
116* round instruction computes. The AES key schedule uses it to apply SubWord to a
117* broadcast word, with k carrying the round constant.
118*/
119inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox_splat(SIMD_4x32 x, SIMD_4x32 k) {
120#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
121 return SIMD_4x32(_mm_aesenclast_si128(x.raw(), k.raw()));
122#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
123 // AESE adds its key before SubBytes so k is added afterwards instead
124 return SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(x.raw()), vdupq_n_u8(0)))) ^ k;
125#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
126 return detail::hw_aes_simd(__builtin_crypto_vcipherlast(detail::hw_aes_vec(x), detail::hw_aes_vec(k)));
127#else
128 #error "hw_aes_sbox_splat not implemented for this architecture"
129#endif
130}
131
132/**
133* True if the round instructions add the round key before SubBytes, as on
134* ARMv8, rather than after MixColumns as on x86 and POWER
135*/
136#if defined(BOTAN_TARGET_ARCH_IS_ARM64)
137constexpr bool HwAesKeyFirst = true;
138#else
139constexpr bool HwAesKeyFirst = false;
140#endif
141
142/**
143* Load a block in the byte order the AES round instructions operate on
144*/
145inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_load_block(const uint8_t in[16]) {
146#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
147 return SIMD_4x32(reinterpret_cast<__vector unsigned int>(detail::hw_aes_state_order(vec_xl(0, in))));
148#else
149 return SIMD_4x32::load_le(in);
150#endif
151}
152
153/**
154* Store a block produced by the AES round instructions
155*/
156inline void BOTAN_FN_ISA_HWAES hw_aes_store_block(SIMD_4x32 B, uint8_t out[16]) {
157#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
158 vec_xst(detail::hw_aes_state_order(reinterpret_cast<__vector unsigned char>(B.raw())), 0, out);
159#else
160 B.store_le(out);
161#endif
162}
163
164/**
165* One encryption round, adding the round key where HwAesKeyFirst says
166*/
167inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_round(SIMD_4x32 B, SIMD_4x32 K) {
168#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
169 return SIMD_4x32(_mm_aesenc_si128(B.raw(), K.raw()));
170#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
171 return SIMD_4x32(
172 vreinterpretq_u32_u8(vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_u32(B.raw()), vreinterpretq_u8_u32(K.raw())))));
173#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
174 return detail::hw_aes_simd(__builtin_crypto_vcipher(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
175#else
176 #error "hw_aes_enc_round not implemented for this architecture"
177#endif
178}
179
180/**
181* The final encryption round, which omits MixColumns
182*/
183inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_last_round(SIMD_4x32 B, SIMD_4x32 K) {
184#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
185 return SIMD_4x32(_mm_aesenclast_si128(B.raw(), K.raw()));
186#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
187 return SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(B.raw()), vreinterpretq_u8_u32(K.raw()))));
188#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
189 return detail::hw_aes_simd(__builtin_crypto_vcipherlast(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
190#else
191 #error "hw_aes_enc_last_round not implemented for this architecture"
192#endif
193}
194
195/**
196* One decryption round, taking a round key prepared by hw_aes_dec_round_key
197*/
198inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round(SIMD_4x32 B, SIMD_4x32 K) {
199#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
200 return SIMD_4x32(_mm_aesdec_si128(B.raw(), K.raw()));
201#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
202 return SIMD_4x32(
203 vreinterpretq_u32_u8(vaesimcq_u8(vaesdq_u8(vreinterpretq_u8_u32(B.raw()), vreinterpretq_u8_u32(K.raw())))));
204#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
205 return detail::hw_aes_simd(__builtin_crypto_vncipher(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
206#else
207 #error "hw_aes_dec_round not implemented for this architecture"
208#endif
209}
210
211/**
212* The final decryption round, which omits InvMixColumns
213*/
214inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_last_round(SIMD_4x32 B, SIMD_4x32 K) {
215#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
216 return SIMD_4x32(_mm_aesdeclast_si128(B.raw(), K.raw()));
217#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
218 return SIMD_4x32(vreinterpretq_u32_u8(vaesdq_u8(vreinterpretq_u8_u32(B.raw()), vreinterpretq_u8_u32(K.raw()))));
219#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
220 return detail::hw_aes_simd(__builtin_crypto_vncipherlast(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
221#else
222 #error "hw_aes_dec_last_round not implemented for this architecture"
223#endif
224}
225
226/**
227* Convert an inner encryption round key to the form hw_aes_dec_round takes
228*
229* x86 and ARMv8 implement the equivalent inverse cipher, which needs
230* InvMixColumns applied to the inner round keys. POWER's vncipher implements
231* the standard inverse cipher and takes the encryption round keys unchanged.
232*/
233inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round_key(SIMD_4x32 K) {
234#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
235 return SIMD_4x32(_mm_aesimc_si128(K.raw()));
236#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
237 return SIMD_4x32(vreinterpretq_u32_u8(vaesimcq_u8(vreinterpretq_u8_u32(K.raw()))));
238#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
239 return K;
240#else
241 #error "hw_aes_dec_round_key not implemented for this architecture"
242#endif
243}
244
245namespace detail {
246
247/*
248* GF(2) matrix-vector multiply: returns M*x where M is a GFNI matrix
249* and x is an 8-bit vector. Both use GFNI bit numbering convention.
250*/
251consteval uint8_t gf2_mat_vec(uint64_t M, uint8_t x) {
252 uint8_t result = 0;
253 for(size_t i = 0; i != 8; ++i) {
254 uint8_t bit = 0;
255 for(size_t j = 0; j != 8; ++j) {
256 if(((M >> (56 - 8 * i + j)) & 1) == 1) {
257 bit ^= (x >> j) & 1;
258 }
259 }
260 result |= bit << i;
261 }
262 return result;
263}
264
265/*
266* GF(2) 8x8 matrix multiplication: returns A*B in GFNI format.
267*/
268consteval uint64_t gf2_mat_mul(uint64_t A, uint64_t B) {
269 uint64_t result = 0;
270 for(size_t i = 0; i != 8; ++i) {
271 for(size_t j = 0; j != 8; ++j) {
272 uint8_t bit = 0;
273 for(size_t k = 0; k != 8; ++k) {
274 auto a_ik = static_cast<uint8_t>((A >> (56 - 8 * i + k)) & 1);
275 auto b_kj = static_cast<uint8_t>((B >> (56 - 8 * k + j)) & 1);
276 bit ^= a_ik & b_kj;
277 }
278 if(bit != 0) {
279 result |= uint64_t(1) << (56 - 8 * i + j);
280 }
281 }
282 }
283 return result;
284}
285
286// AES affine matrix in GFNI format
287constexpr uint64_t AES_AFF = gfni_matrix(R"(
288 1 0 0 0 1 1 1 1
289 1 1 0 0 0 1 1 1
290 1 1 1 0 0 0 1 1
291 1 1 1 1 0 0 0 1
292 1 1 1 1 1 0 0 0
293 0 1 1 1 1 1 0 0
294 0 0 1 1 1 1 1 0
295 0 0 0 1 1 1 1 1)");
296constexpr uint8_t AES_C = 0x63;
297
298// AES inverse affine matrix in GFNI format
299constexpr uint64_t AES_AFF_INV = gfni_matrix(R"(
300 0 0 1 0 0 1 0 1
301 1 0 0 1 0 0 1 0
302 0 1 0 0 1 0 0 1
303 1 0 1 0 0 1 0 0
304 0 1 0 1 0 0 1 0
305 0 0 1 0 1 0 0 1
306 1 0 0 1 0 1 0 0
307 0 1 0 0 1 0 1 0)");
308constexpr uint8_t AES_C_INV = 0x05;
309
310} // namespace detail
311
312/**
313* Lookup tables for GF(2) affine transformations
314*/
316 public:
317 consteval Gf2AffineTransformation(uint64_t M, uint8_t c) : lo{}, hi{} {
318 for(size_t i = 0; i != 16; ++i) {
319 // Low nibble table includes the constant addition
320 const uint8_t lo_val = detail::gf2_mat_vec(M, static_cast<uint8_t>(i)) ^ c;
321 const uint8_t hi_val = detail::gf2_mat_vec(M, static_cast<uint8_t>(i << 4));
322
323 lo[i / 4] |= static_cast<uint32_t>(lo_val) << (8 * (i % 4));
324 hi[i / 4] |= static_cast<uint32_t>(hi_val) << (8 * (i % 4));
325 }
326 }
327
328 /**
329 * Derive tables used for computing an affine transform after the application of an
330 * AES sbox.
331 */
332 static consteval Gf2AffineTransformation post_sbox(uint64_t M, uint8_t c) {
333 const auto comb_M = detail::gf2_mat_mul(M, detail::AES_AFF_INV);
334 const auto comb_c = static_cast<uint8_t>(detail::gf2_mat_vec(comb_M, detail::AES_C) ^ c);
335 return Gf2AffineTransformation(comb_M, comb_c);
336 }
337
338 /**
339 * Derive tables used for computing an affine transform after the application of an
340 * AES inverse sbox.
341 */
342 static consteval Gf2AffineTransformation post_inv_sbox(uint64_t M, uint8_t c) {
343 const auto comb_mat = detail::gf2_mat_mul(detail::AES_AFF, M);
344 const auto comb_c = detail::gf2_mat_vec(detail::AES_AFF, static_cast<uint8_t>(c ^ detail::AES_C_INV));
345 return Gf2AffineTransformation(comb_mat, comb_c);
346 }
347
348 inline SIMD_4x32 BOTAN_FN_ISA_HWAES affine_transform(SIMD_4x32 x) const {
349 const SIMD_4x32 tbl_lo(lo[0], lo[1], lo[2], lo[3]);
350 const SIMD_4x32 tbl_hi(hi[0], hi[1], hi[2], hi[3]);
351 const auto lo_mask = SIMD_4x32::splat_u8(0x0F);
352
353 return SIMD_4x32::byte_shuffle(tbl_lo, lo_mask & x) ^ SIMD_4x32::byte_shuffle(tbl_hi, lo_mask & x.shr<4>());
354 }
355
356 private:
357 uint32_t lo[4];
358 uint32_t hi[4];
359};
360
361} // namespace Botan
362
363#endif
SIMD_4x32 BOTAN_FN_ISA_HWAES affine_transform(SIMD_4x32 x) const
Definition simd_hwaes.h:348
static consteval Gf2AffineTransformation post_sbox(uint64_t M, uint8_t c)
Definition simd_hwaes.h:332
consteval Gf2AffineTransformation(uint64_t M, uint8_t c)
Definition simd_hwaes.h:317
static consteval Gf2AffineTransformation post_inv_sbox(uint64_t M, uint8_t c)
Definition simd_hwaes.h:342
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept
Definition simd_4x32.h:210
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shr() const noexcept
Definition simd_4x32.h:485
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:183
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_shuffle(const SIMD_4x32 &tbl, const SIMD_4x32 &idx)
Definition simd_4x32.h:796
native_simd_type BOTAN_FN_ISA_SIMD_4X32 raw() const noexcept
Definition simd_4x32.h:964
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat_u8(uint8_t B) noexcept
Definition simd_4x32.h:140
constexpr uint64_t AES_AFF
Definition simd_hwaes.h:287
consteval uint8_t gf2_mat_vec(uint64_t M, uint8_t x)
Definition simd_hwaes.h:251
consteval uint64_t gf2_mat_mul(uint64_t A, uint64_t B)
Definition simd_hwaes.h:268
constexpr uint64_t AES_AFF_INV
Definition simd_hwaes.h:299
constexpr uint8_t AES_C_INV
Definition simd_hwaes.h:308
constexpr uint8_t AES_C
Definition simd_hwaes.h:296
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_last_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:183
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox_splat(SIMD_4x32 x, SIMD_4x32 k)
Definition simd_hwaes.h:119
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round_key(SIMD_4x32 K)
Definition simd_hwaes.h:233
consteval uint64_t gfni_matrix(std::string_view s)
Definition gfni_utils.h:17
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_last_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:214
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:167
constexpr bool HwAesKeyFirst
Definition simd_hwaes.h:139
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:198
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_load_block(const uint8_t in[16])
Definition simd_hwaes.h:145
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_inv_sbox(SIMD_4x32 x)
Definition simd_hwaes.h:83
void BOTAN_FN_ISA_HWAES hw_aes_store_block(SIMD_4x32 B, uint8_t out[16])
Definition simd_hwaes.h:156
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox(SIMD_4x32 x)
Definition simd_hwaes.h:60