Botan 3.14.1
Crypto and TLS for C&
aes_hwaes.cpp
Go to the documentation of this file.
1/*
2* AES using hardware AES instructions
3* (C) 2026 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#include <botan/internal/aes.h>
9
10#include <botan/internal/isa_extn.h>
11#include <botan/internal/simd_4x32.h>
12#include <botan/internal/simd_hwaes.h>
13#include <utility>
14
15namespace Botan {
16
17namespace {
18
19template <bool ENC>
20BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 hw_aes_round(SIMD_4x32 B, SIMD_4x32 K) {
21 if constexpr(ENC) {
22 return hw_aes_enc_round(B, K);
23 } else {
24 return hw_aes_dec_round(B, K);
25 }
26}
27
28template <bool ENC>
29BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 hw_aes_last_round(SIMD_4x32 B, SIMD_4x32 K) {
30 if constexpr(ENC) {
31 return hw_aes_enc_last_round(B, K);
32 } else {
33 return hw_aes_dec_last_round(B, K);
34 }
35}
36
37template <bool ENC, typename... Bs>
38BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void hw_aes_round_all(SIMD_4x32 K, Bs&... B) {
39 ((B = hw_aes_round<ENC>(B, K)), ...);
40}
41
42// The round keys are indexed with constants so that GCC keeps them in registers
43template <size_t R, bool ENC, size_t... Is, typename... Bs>
44BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void hw_aes_inner_rounds(const SIMD_4x32 (&K)[R + 1],
45 std::index_sequence<Is...> /*rounds*/,
46 Bs&... B) {
47 constexpr size_t S = HwAesKeyFirst ? 0 : 1;
48 (hw_aes_round_all<ENC>(K[Is + S], B...), ...);
49}
50
51/*
52* Run the full cipher over any number of blocks in parallel. Where the ISA adds
53* the round key before SubBytes the whitening key goes into the first round
54* instruction, otherwise the last round key goes into the final one.
55*/
56template <size_t R, bool ENC, typename... Bs>
57BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void hw_aes_blocks(const SIMD_4x32 (&K)[R + 1], Bs&... B) {
58 constexpr size_t S = HwAesKeyFirst ? 0 : 1;
59
60 if constexpr(!HwAesKeyFirst) {
61 ((B ^= K[0]), ...);
62 }
63
64 hw_aes_inner_rounds<R, ENC>(K, std::make_index_sequence<R - 1>{}, B...);
65
66 ((B = hw_aes_last_round<ENC>(B, K[R - 1 + S])), ...);
67
68 if constexpr(HwAesKeyFirst) {
69 ((B ^= K[R]), ...);
70 }
71}
72
73template <size_t R, size_t... Is>
74BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void hw_aes_load_keys(SIMD_4x32 (&K)[R + 1],
75 const uint32_t keys[],
76 std::index_sequence<Is...> /*keys*/) {
77 ((K[Is] = hw_aes_load_block(reinterpret_cast<const uint8_t*>(&keys[4 * Is]))), ...);
78}
79
80template <size_t R, bool ENC>
81BOTAN_FN_ISA_HWAES void hw_aes_process(const uint8_t in[], uint8_t out[], size_t blocks, const uint32_t keys[]) {
82 SIMD_4x32 K[R + 1];
83 hw_aes_load_keys<R>(K, keys, std::make_index_sequence<R + 1>{});
84
85 while(blocks >= 4) {
87 SIMD_4x32 B1 = hw_aes_load_block(in + 16);
88 SIMD_4x32 B2 = hw_aes_load_block(in + 32);
89 SIMD_4x32 B3 = hw_aes_load_block(in + 48);
90
91 hw_aes_blocks<R, ENC>(K, B0, B1, B2, B3);
92
93 hw_aes_store_block(B0, out);
94 hw_aes_store_block(B1, out + 16);
95 hw_aes_store_block(B2, out + 32);
96 hw_aes_store_block(B3, out + 48);
97
98 in += 4 * 16;
99 out += 4 * 16;
100 blocks -= 4;
101 }
102
103 for(size_t i = 0; i != blocks; ++i) {
104 SIMD_4x32 B = hw_aes_load_block(in + 16 * i);
105 hw_aes_blocks<R, ENC>(K, B);
106 hw_aes_store_block(B, out + 16 * i);
107 }
108}
109
110/*
111* Each word of a round key is the xor of the previous word and the word four
112* positions back. So given the previous round key and t = SubWord(RotWord(w))
113* ^ Rcon broadcast to every word, the new round key is the running xor over
114* the words of prev, with t folded into each.
115*/
116BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 next_round_key(SIMD_4x32 prev, SIMD_4x32 t) {
117 prev ^= prev.shift_elems_left<1>();
118 prev ^= prev.shift_elems_left<2>();
119 return prev ^ t;
120}
121
122// SubWord(RotWord(w3)) ^ rcon broadcast to all four words
123BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 sub_rot_w3(SIMD_4x32 k, SIMD_4x32 rcon) {
124 return hw_aes_sbox_splat(SIMD_4x32::byte_shuffle(k, SIMD_4x32::splat(0x0C0F0E0D)), rcon);
125}
126
127// SubWord(w3) broadcast to all four words
128BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 sub_w3(SIMD_4x32 k) {
130}
131
132// SubWord(RotWord(w1)) ^ rcon broadcast to all four words
133BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 sub_rot_w1(SIMD_4x32 k, SIMD_4x32 rcon) {
134 return hw_aes_sbox_splat(SIMD_4x32::byte_shuffle(k, SIMD_4x32::splat(0x04070605)), rcon);
135}
136
137template <uint8_t RC>
138BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES SIMD_4x32 aes_128_round_key(SIMD_4x32 prev, uint32_t out[]) {
139 const auto K = next_round_key(prev, sub_rot_w3(prev, SIMD_4x32::splat(RC)));
140 K.store_le(out);
141 return K;
142}
143
144/*
145* AES-192 produces six words per step, four in K0 and two in the low half of
146* K1. The high half of K1 is garbage, so it is written before the following
147* four words, which overwrite it.
148*/
149template <uint8_t RC>
150BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void aes_192_round_keys(SIMD_4x32& K0, SIMD_4x32& K1, uint32_t out[]) {
151 K0 = next_round_key(K0, sub_rot_w1(K1, SIMD_4x32::splat(RC)));
152 K0.store_le(out);
153 K1 ^= K1.shift_elems_left<1>();
154 K1 ^= SIMD_4x32::byte_shuffle(K0, SIMD_4x32::splat(0x0F0E0D0C));
155 K1.store_le(out + 4);
156}
157
158// AES-256 alternates between the two key halves; the second uses SubWord alone
159template <uint8_t RC>
160BOTAN_FORCE_INLINE BOTAN_FN_ISA_HWAES void aes_256_round_keys(SIMD_4x32& K0, SIMD_4x32& K1, uint32_t out[]) {
161 K0 = next_round_key(K0, sub_rot_w3(K1, SIMD_4x32::splat(RC)));
162 K0.store_le(out);
163 K1 = next_round_key(K1, sub_w3(K0));
164 K1.store_le(out + 4);
165}
166
167/*
168* The decryption round keys are the encryption round keys in reverse order,
169* with the inner ones converted to the form the decryption instructions take
170*/
171BOTAN_FN_ISA_HWAES void hw_aes_decryption_keys(const secure_vector<uint32_t>& EK, secure_vector<uint32_t>& DK) {
172 const size_t rounds = EK.size() / 4 - 1;
173 DK.resize(EK.size());
174
175 SIMD_4x32::load_le(&EK[4 * rounds]).store_le(DK.data());
176 for(size_t i = 1; i != rounds; ++i) {
177 hw_aes_dec_round_key(SIMD_4x32::load_le(&EK[4 * (rounds - i)])).store_le(&DK[4 * i]);
178 }
179 SIMD_4x32::load_le(EK.data()).store_le(&DK[4 * rounds]);
180}
181
182} // namespace
183
184void BOTAN_FN_ISA_HWAES AES_128::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
185 hw_aes_process<10, true>(in, out, blocks, m_EK.data());
186}
187
188void BOTAN_FN_ISA_HWAES AES_128::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
189 hw_aes_process<10, false>(in, out, blocks, m_DK.data());
190}
191
192void BOTAN_FN_ISA_HWAES AES_192::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
193 hw_aes_process<12, true>(in, out, blocks, m_EK.data());
194}
195
196void BOTAN_FN_ISA_HWAES AES_192::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
197 hw_aes_process<12, false>(in, out, blocks, m_DK.data());
198}
199
200void BOTAN_FN_ISA_HWAES AES_256::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
201 hw_aes_process<14, true>(in, out, blocks, m_EK.data());
202}
203
204void BOTAN_FN_ISA_HWAES AES_256::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
205 hw_aes_process<14, false>(in, out, blocks, m_DK.data());
206}
207
208void BOTAN_FN_ISA_HWAES AES_128::hw_aes_key_schedule(std::span<const uint8_t> key) {
209 m_EK.resize(44);
210
211 SIMD_4x32 K = SIMD_4x32::load_le(key.first<16>());
212 K.store_le(m_EK.data());
213 K = aes_128_round_key<0x01>(K, &m_EK[4]);
214 K = aes_128_round_key<0x02>(K, &m_EK[8]);
215 K = aes_128_round_key<0x04>(K, &m_EK[12]);
216 K = aes_128_round_key<0x08>(K, &m_EK[16]);
217 K = aes_128_round_key<0x10>(K, &m_EK[20]);
218 K = aes_128_round_key<0x20>(K, &m_EK[24]);
219 K = aes_128_round_key<0x40>(K, &m_EK[28]);
220 K = aes_128_round_key<0x80>(K, &m_EK[32]);
221 K = aes_128_round_key<0x1B>(K, &m_EK[36]);
222 aes_128_round_key<0x36>(K, &m_EK[40]);
223
224 hw_aes_decryption_keys(m_EK, m_DK);
225}
226
227void BOTAN_FN_ISA_HWAES AES_192::hw_aes_key_schedule(std::span<const uint8_t> key) {
228 m_EK.resize(52);
229
230 SIMD_4x32 K0 = SIMD_4x32::load_le(key.first<16>());
231 SIMD_4x32 K1 = SIMD_4x32::load_le(key.subspan<8, 16>()).shift_elems_right<2>();
232 K0.store_le(m_EK.data());
233 K1.store_le(&m_EK[4]);
234
235 aes_192_round_keys<0x01>(K0, K1, &m_EK[6]);
236 aes_192_round_keys<0x02>(K0, K1, &m_EK[12]);
237 aes_192_round_keys<0x04>(K0, K1, &m_EK[18]);
238 aes_192_round_keys<0x08>(K0, K1, &m_EK[24]);
239 aes_192_round_keys<0x10>(K0, K1, &m_EK[30]);
240 aes_192_round_keys<0x20>(K0, K1, &m_EK[36]);
241 aes_192_round_keys<0x40>(K0, K1, &m_EK[42]);
242
243 // The final step produces just four words
244 K0 = next_round_key(K0, sub_rot_w1(K1, SIMD_4x32::splat(0x80)));
245 K0.store_le(&m_EK[48]);
246
247 hw_aes_decryption_keys(m_EK, m_DK);
248}
249
250void BOTAN_FN_ISA_HWAES AES_256::hw_aes_key_schedule(std::span<const uint8_t> key) {
251 m_EK.resize(60);
252
253 SIMD_4x32 K0 = SIMD_4x32::load_le(key.first<16>());
254 SIMD_4x32 K1 = SIMD_4x32::load_le(key.subspan<16, 16>());
255 K0.store_le(m_EK.data());
256 K1.store_le(&m_EK[4]);
257
258 aes_256_round_keys<0x01>(K0, K1, &m_EK[8]);
259 aes_256_round_keys<0x02>(K0, K1, &m_EK[16]);
260 aes_256_round_keys<0x04>(K0, K1, &m_EK[24]);
261 aes_256_round_keys<0x08>(K0, K1, &m_EK[32]);
262 aes_256_round_keys<0x10>(K0, K1, &m_EK[40]);
263 aes_256_round_keys<0x20>(K0, K1, &m_EK[48]);
264
265 // The final step produces just one round key
266 K0 = next_round_key(K0, sub_rot_w3(K1, SIMD_4x32::splat(0x40)));
267 K0.store_le(&m_EK[56]);
268
269 hw_aes_decryption_keys(m_EK, m_DK);
270}
271
272} // namespace Botan
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept
Definition simd_4x32.h:210
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:183
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_shuffle(const SIMD_4x32 &tbl, const SIMD_4x32 &idx)
Definition simd_4x32.h:796
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_right() const noexcept
Definition simd_4x32.h:599
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat(uint32_t B) noexcept
Definition simd_4x32.h:123
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_left() const noexcept
Definition simd_4x32.h:562
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
constexpr uint32_t K1
Definition sha1_f.h:16
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_last_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:183
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox_splat(SIMD_4x32 x, SIMD_4x32 k)
Definition simd_hwaes.h:119
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round_key(SIMD_4x32 K)
Definition simd_hwaes.h:233
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_last_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:214
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:167
constexpr bool HwAesKeyFirst
Definition simd_hwaes.h:139
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round(SIMD_4x32 B, SIMD_4x32 K)
Definition simd_hwaes.h:198
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_load_block(const uint8_t in[16])
Definition simd_hwaes.h:145
std::vector< T, secure_allocator< T > > secure_vector
Definition secmem.h:128
void BOTAN_FN_ISA_HWAES hw_aes_store_block(SIMD_4x32 B, uint8_t out[16])
Definition simd_hwaes.h:156