Botan 3.13.0
Crypto and TLS for C&
aes_ni.cpp
Go to the documentation of this file.
1/*
2* AES using AES-NI instructions
3* (C) 2009,2012 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#include <botan/internal/aes.h>
9
10#include <botan/internal/isa_extn.h>
11#include <botan/internal/loadstor.h>
12#include <botan/internal/simd_4x32.h>
13#include <wmmintrin.h>
14
15namespace Botan {
16
17namespace {
18
19// NOLINTBEGIN(portability-simd-intrinsics)
20
21template <uint8_t RC>
22BOTAN_FN_ISA_AESNI inline SIMD_4x32 aes_128_key_expansion(SIMD_4x32 key, SIMD_4x32 key_getting_rcon) {
23 __m128i key_with_rcon = _mm_aeskeygenassist_si128(key_getting_rcon.raw(), RC);
24 key_with_rcon = _mm_shuffle_epi32(key_with_rcon, _MM_SHUFFLE(3, 3, 3, 3));
25 key ^= key.shift_elems_left<1>();
26 key ^= key.shift_elems_left<1>();
27 key ^= key.shift_elems_left<1>();
28 key ^= SIMD_4x32(key_with_rcon);
29 return key;
30}
31
32BOTAN_FN_ISA_AESNI
33void aes_192_key_expansion(
34 SIMD_4x32& K1, SIMD_4x32& K2, SIMD_4x32 key2_with_rcon, secure_vector<uint32_t>& out, size_t offset) {
35 const SIMD_4x32 rcon = SIMD_4x32(_mm_shuffle_epi32(key2_with_rcon.raw(), _MM_SHUFFLE(1, 1, 1, 1)));
36 K1 ^= K1.shift_elems_left<1>();
37 K1 ^= K1.shift_elems_left<1>();
38 K1 ^= K1.shift_elems_left<1>();
39 K1 ^= rcon;
40
41 K1.store_le(&out[offset]);
42
43 if(offset == 48) {
44 return;
45 }
46
47 K2 ^= K2.shift_elems_left<1>();
48 K2 ^= SIMD_4x32(_mm_shuffle_epi32(K1.raw(), _MM_SHUFFLE(3, 3, 3, 3)));
49
50 out[offset + 4] = K2.extract_word<0>();
51 out[offset + 5] = K2.extract_word<1>();
52}
53
54/*
55* The second half of the AES-256 key expansion (other half same as AES-128)
56*/
57BOTAN_FN_ISA_AESNI SIMD_4x32 aes_256_key_expansion(SIMD_4x32 key, SIMD_4x32 key2) {
58 __m128i key_with_rcon = _mm_aeskeygenassist_si128(key2.raw(), 0x00);
59 key_with_rcon = _mm_shuffle_epi32(key_with_rcon, _MM_SHUFFLE(2, 2, 2, 2));
60
61 key ^= key.shift_elems_left<1>();
62 key ^= key.shift_elems_left<1>();
63 key ^= key.shift_elems_left<1>();
64 key ^= SIMD_4x32(key_with_rcon);
65 return key;
66}
67
68BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void keyxor(
69 SIMD_4x32 K, SIMD_4x32& B0, SIMD_4x32& B1, SIMD_4x32& B2, SIMD_4x32& B3) {
70 B0 ^= K;
71 B1 ^= K;
72 B2 ^= K;
73 B3 ^= K;
74}
75
76BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesenc(SIMD_4x32 K, SIMD_4x32& B) {
77 B = SIMD_4x32(_mm_aesenc_si128(B.raw(), K.raw()));
78}
79
80BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesenc(
81 SIMD_4x32 K, SIMD_4x32& B0, SIMD_4x32& B1, SIMD_4x32& B2, SIMD_4x32& B3) {
82 B0 = SIMD_4x32(_mm_aesenc_si128(B0.raw(), K.raw()));
83 B1 = SIMD_4x32(_mm_aesenc_si128(B1.raw(), K.raw()));
84 B2 = SIMD_4x32(_mm_aesenc_si128(B2.raw(), K.raw()));
85 B3 = SIMD_4x32(_mm_aesenc_si128(B3.raw(), K.raw()));
86}
87
88BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesenclast(SIMD_4x32 K, SIMD_4x32& B) {
89 B = SIMD_4x32(_mm_aesenclast_si128(B.raw(), K.raw()));
90}
91
92BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesenclast(
93 SIMD_4x32 K, SIMD_4x32& B0, SIMD_4x32& B1, SIMD_4x32& B2, SIMD_4x32& B3) {
94 B0 = SIMD_4x32(_mm_aesenclast_si128(B0.raw(), K.raw()));
95 B1 = SIMD_4x32(_mm_aesenclast_si128(B1.raw(), K.raw()));
96 B2 = SIMD_4x32(_mm_aesenclast_si128(B2.raw(), K.raw()));
97 B3 = SIMD_4x32(_mm_aesenclast_si128(B3.raw(), K.raw()));
98}
99
100BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesdec(SIMD_4x32 K, SIMD_4x32& B) {
101 B = SIMD_4x32(_mm_aesdec_si128(B.raw(), K.raw()));
102}
103
104BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesdec(
105 SIMD_4x32 K, SIMD_4x32& B0, SIMD_4x32& B1, SIMD_4x32& B2, SIMD_4x32& B3) {
106 B0 = SIMD_4x32(_mm_aesdec_si128(B0.raw(), K.raw()));
107 B1 = SIMD_4x32(_mm_aesdec_si128(B1.raw(), K.raw()));
108 B2 = SIMD_4x32(_mm_aesdec_si128(B2.raw(), K.raw()));
109 B3 = SIMD_4x32(_mm_aesdec_si128(B3.raw(), K.raw()));
110}
111
112BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesdeclast(SIMD_4x32 K, SIMD_4x32& B) {
113 B = SIMD_4x32(_mm_aesdeclast_si128(B.raw(), K.raw()));
114}
115
116BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI void aesdeclast(
117 SIMD_4x32 K, SIMD_4x32& B0, SIMD_4x32& B1, SIMD_4x32& B2, SIMD_4x32& B3) {
118 B0 = SIMD_4x32(_mm_aesdeclast_si128(B0.raw(), K.raw()));
119 B1 = SIMD_4x32(_mm_aesdeclast_si128(B1.raw(), K.raw()));
120 B2 = SIMD_4x32(_mm_aesdeclast_si128(B2.raw(), K.raw()));
121 B3 = SIMD_4x32(_mm_aesdeclast_si128(B3.raw(), K.raw()));
122}
123
124BOTAN_FORCE_INLINE BOTAN_FN_ISA_AESNI SIMD_4x32 aesimc(SIMD_4x32 B) {
125 return SIMD_4x32(_mm_aesimc_si128(B.raw()));
126}
127
128// NOLINTEND(portability-simd-intrinsics)
129
130} // namespace
131
132/*
133* AES-128 Encryption
134*/
135BOTAN_FN_ISA_AESNI void AES_128::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
136 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_EK[4 * 0]);
137 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_EK[4 * 1]);
138 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_EK[4 * 2]);
139 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_EK[4 * 3]);
140 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_EK[4 * 4]);
141 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_EK[4 * 5]);
142 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_EK[4 * 6]);
143 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_EK[4 * 7]);
144 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_EK[4 * 8]);
145 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_EK[4 * 9]);
146 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_EK[4 * 10]);
147
148 while(blocks >= 4) {
149 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
150 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
151 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
152 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
153
154 keyxor(K0, B0, B1, B2, B3);
155 aesenc(K1, B0, B1, B2, B3);
156 aesenc(K2, B0, B1, B2, B3);
157 aesenc(K3, B0, B1, B2, B3);
158 aesenc(K4, B0, B1, B2, B3);
159 aesenc(K5, B0, B1, B2, B3);
160 aesenc(K6, B0, B1, B2, B3);
161 aesenc(K7, B0, B1, B2, B3);
162 aesenc(K8, B0, B1, B2, B3);
163 aesenc(K9, B0, B1, B2, B3);
164 aesenclast(K10, B0, B1, B2, B3);
165
166 B0.store_le(out + 16 * 0);
167 B1.store_le(out + 16 * 1);
168 B2.store_le(out + 16 * 2);
169 B3.store_le(out + 16 * 3);
170
171 blocks -= 4;
172 in += 4 * 16;
173 out += 4 * 16;
174 }
175
176 for(size_t i = 0; i != blocks; ++i) {
177 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
178
179 B0 ^= K0;
180 aesenc(K1, B0);
181 aesenc(K2, B0);
182 aesenc(K3, B0);
183 aesenc(K4, B0);
184 aesenc(K5, B0);
185 aesenc(K6, B0);
186 aesenc(K7, B0);
187 aesenc(K8, B0);
188 aesenc(K9, B0);
189 aesenclast(K10, B0);
190
191 B0.store_le(out + 16 * i);
192 }
193}
194
195/*
196* AES-128 Decryption
197*/
198BOTAN_FN_ISA_AESNI void AES_128::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
199 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_DK[4 * 0]);
200 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_DK[4 * 1]);
201 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_DK[4 * 2]);
202 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_DK[4 * 3]);
203 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_DK[4 * 4]);
204 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_DK[4 * 5]);
205 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_DK[4 * 6]);
206 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_DK[4 * 7]);
207 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_DK[4 * 8]);
208 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_DK[4 * 9]);
209 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_DK[4 * 10]);
210
211 while(blocks >= 4) {
212 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
213 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
214 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
215 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
216
217 keyxor(K0, B0, B1, B2, B3);
218 aesdec(K1, B0, B1, B2, B3);
219 aesdec(K2, B0, B1, B2, B3);
220 aesdec(K3, B0, B1, B2, B3);
221 aesdec(K4, B0, B1, B2, B3);
222 aesdec(K5, B0, B1, B2, B3);
223 aesdec(K6, B0, B1, B2, B3);
224 aesdec(K7, B0, B1, B2, B3);
225 aesdec(K8, B0, B1, B2, B3);
226 aesdec(K9, B0, B1, B2, B3);
227 aesdeclast(K10, B0, B1, B2, B3);
228
229 B0.store_le(out + 16 * 0);
230 B1.store_le(out + 16 * 1);
231 B2.store_le(out + 16 * 2);
232 B3.store_le(out + 16 * 3);
233
234 blocks -= 4;
235 in += 4 * 16;
236 out += 4 * 16;
237 }
238
239 for(size_t i = 0; i != blocks; ++i) {
240 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
241
242 B0 ^= K0;
243 aesdec(K1, B0);
244 aesdec(K2, B0);
245 aesdec(K3, B0);
246 aesdec(K4, B0);
247 aesdec(K5, B0);
248 aesdec(K6, B0);
249 aesdec(K7, B0);
250 aesdec(K8, B0);
251 aesdec(K9, B0);
252 aesdeclast(K10, B0);
253
254 B0.store_le(out + 16 * i);
255 }
256}
257
258/*
259* AES-128 Key Schedule
260*/
261BOTAN_FN_ISA_AESNI void AES_128::aesni_key_schedule(const uint8_t key[], size_t /*length*/) {
262 m_EK.resize(44);
263 m_DK.resize(44);
264
265 const SIMD_4x32 K0 = SIMD_4x32::load_le(key);
266 const SIMD_4x32 K1 = aes_128_key_expansion<0x01>(K0, K0);
267 const SIMD_4x32 K2 = aes_128_key_expansion<0x02>(K1, K1);
268 const SIMD_4x32 K3 = aes_128_key_expansion<0x04>(K2, K2);
269 const SIMD_4x32 K4 = aes_128_key_expansion<0x08>(K3, K3);
270 const SIMD_4x32 K5 = aes_128_key_expansion<0x10>(K4, K4);
271 const SIMD_4x32 K6 = aes_128_key_expansion<0x20>(K5, K5);
272 const SIMD_4x32 K7 = aes_128_key_expansion<0x40>(K6, K6);
273 const SIMD_4x32 K8 = aes_128_key_expansion<0x80>(K7, K7);
274 const SIMD_4x32 K9 = aes_128_key_expansion<0x1B>(K8, K8);
275 const SIMD_4x32 K10 = aes_128_key_expansion<0x36>(K9, K9);
276
277 K0.store_le(&m_EK[4 * 0]);
278 K1.store_le(&m_EK[4 * 1]);
279 K2.store_le(&m_EK[4 * 2]);
280 K3.store_le(&m_EK[4 * 3]);
281 K4.store_le(&m_EK[4 * 4]);
282 K5.store_le(&m_EK[4 * 5]);
283 K6.store_le(&m_EK[4 * 6]);
284 K7.store_le(&m_EK[4 * 7]);
285 K8.store_le(&m_EK[4 * 8]);
286 K9.store_le(&m_EK[4 * 9]);
287 K10.store_le(&m_EK[4 * 10]);
288
289 // Now generate decryption keys
290 K10.store_le(&m_DK[4 * 0]);
291 aesimc(K9).store_le(&m_DK[4 * 1]);
292 aesimc(K8).store_le(&m_DK[4 * 2]);
293 aesimc(K7).store_le(&m_DK[4 * 3]);
294 aesimc(K6).store_le(&m_DK[4 * 4]);
295 aesimc(K5).store_le(&m_DK[4 * 5]);
296 aesimc(K4).store_le(&m_DK[4 * 6]);
297 aesimc(K3).store_le(&m_DK[4 * 7]);
298 aesimc(K2).store_le(&m_DK[4 * 8]);
299 aesimc(K1).store_le(&m_DK[4 * 9]);
300 K0.store_le(&m_DK[4 * 10]);
301}
302
303/*
304* AES-192 Encryption
305*/
306BOTAN_FN_ISA_AESNI void AES_192::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
307 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_EK[4 * 0]);
308 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_EK[4 * 1]);
309 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_EK[4 * 2]);
310 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_EK[4 * 3]);
311 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_EK[4 * 4]);
312 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_EK[4 * 5]);
313 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_EK[4 * 6]);
314 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_EK[4 * 7]);
315 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_EK[4 * 8]);
316 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_EK[4 * 9]);
317 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_EK[4 * 10]);
318 const SIMD_4x32 K11 = SIMD_4x32::load_le(&m_EK[4 * 11]);
319 const SIMD_4x32 K12 = SIMD_4x32::load_le(&m_EK[4 * 12]);
320
321 while(blocks >= 4) {
322 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
323 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
324 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
325 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
326
327 keyxor(K0, B0, B1, B2, B3);
328 aesenc(K1, B0, B1, B2, B3);
329 aesenc(K2, B0, B1, B2, B3);
330 aesenc(K3, B0, B1, B2, B3);
331 aesenc(K4, B0, B1, B2, B3);
332 aesenc(K5, B0, B1, B2, B3);
333 aesenc(K6, B0, B1, B2, B3);
334 aesenc(K7, B0, B1, B2, B3);
335 aesenc(K8, B0, B1, B2, B3);
336 aesenc(K9, B0, B1, B2, B3);
337 aesenc(K10, B0, B1, B2, B3);
338 aesenc(K11, B0, B1, B2, B3);
339 aesenclast(K12, B0, B1, B2, B3);
340
341 B0.store_le(out + 16 * 0);
342 B1.store_le(out + 16 * 1);
343 B2.store_le(out + 16 * 2);
344 B3.store_le(out + 16 * 3);
345
346 blocks -= 4;
347 in += 4 * 16;
348 out += 4 * 16;
349 }
350
351 for(size_t i = 0; i != blocks; ++i) {
352 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
353
354 B0 ^= K0;
355
356 aesenc(K1, B0);
357 aesenc(K2, B0);
358 aesenc(K3, B0);
359 aesenc(K4, B0);
360 aesenc(K5, B0);
361 aesenc(K6, B0);
362 aesenc(K7, B0);
363 aesenc(K8, B0);
364 aesenc(K9, B0);
365 aesenc(K10, B0);
366 aesenc(K11, B0);
367 aesenclast(K12, B0);
368
369 B0.store_le(out + 16 * i);
370 }
371}
372
373/*
374* AES-192 Decryption
375*/
376BOTAN_FN_ISA_AESNI void AES_192::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
377 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_DK[4 * 0]);
378 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_DK[4 * 1]);
379 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_DK[4 * 2]);
380 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_DK[4 * 3]);
381 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_DK[4 * 4]);
382 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_DK[4 * 5]);
383 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_DK[4 * 6]);
384 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_DK[4 * 7]);
385 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_DK[4 * 8]);
386 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_DK[4 * 9]);
387 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_DK[4 * 10]);
388 const SIMD_4x32 K11 = SIMD_4x32::load_le(&m_DK[4 * 11]);
389 const SIMD_4x32 K12 = SIMD_4x32::load_le(&m_DK[4 * 12]);
390
391 while(blocks >= 4) {
392 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
393 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
394 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
395 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
396
397 keyxor(K0, B0, B1, B2, B3);
398 aesdec(K1, B0, B1, B2, B3);
399 aesdec(K2, B0, B1, B2, B3);
400 aesdec(K3, B0, B1, B2, B3);
401 aesdec(K4, B0, B1, B2, B3);
402 aesdec(K5, B0, B1, B2, B3);
403 aesdec(K6, B0, B1, B2, B3);
404 aesdec(K7, B0, B1, B2, B3);
405 aesdec(K8, B0, B1, B2, B3);
406 aesdec(K9, B0, B1, B2, B3);
407 aesdec(K10, B0, B1, B2, B3);
408 aesdec(K11, B0, B1, B2, B3);
409 aesdeclast(K12, B0, B1, B2, B3);
410
411 B0.store_le(out + 16 * 0);
412 B1.store_le(out + 16 * 1);
413 B2.store_le(out + 16 * 2);
414 B3.store_le(out + 16 * 3);
415
416 blocks -= 4;
417 in += 4 * 16;
418 out += 4 * 16;
419 }
420
421 for(size_t i = 0; i != blocks; ++i) {
422 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
423
424 B0 ^= K0;
425
426 aesdec(K1, B0);
427 aesdec(K2, B0);
428 aesdec(K3, B0);
429 aesdec(K4, B0);
430 aesdec(K5, B0);
431 aesdec(K6, B0);
432 aesdec(K7, B0);
433 aesdec(K8, B0);
434 aesdec(K9, B0);
435 aesdec(K10, B0);
436 aesdec(K11, B0);
437 aesdeclast(K12, B0);
438
439 B0.store_le(out + 16 * i);
440 }
441}
442
443/*
444* AES-192 Key Schedule
445*/
446BOTAN_FN_ISA_AESNI void AES_192::aesni_key_schedule(const uint8_t key[], size_t /*length*/) {
447 m_EK.resize(52);
448 m_DK.resize(52);
449
450 SIMD_4x32 K0 = SIMD_4x32::load_le(key);
451 SIMD_4x32 K1 = SIMD_4x32::load_le(key + 8).shift_elems_right<2>();
452
453 load_le(m_EK.data(), key, 6);
454
455 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x01)), m_EK, 6);
456 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x02)), m_EK, 12);
457 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x04)), m_EK, 18);
458 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x08)), m_EK, 24);
459 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x10)), m_EK, 30);
460 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x20)), m_EK, 36);
461 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x40)), m_EK, 42);
462 aes_192_key_expansion(K0, K1, SIMD_4x32(_mm_aeskeygenassist_si128(K1.raw(), 0x80)), m_EK, 48);
463
464 // Now generate decryption keys
465 SIMD_4x32::load_le(&m_EK[4 * 12]).store_le(&m_DK[4 * 0]);
466 aesimc(SIMD_4x32::load_le(&m_EK[4 * 11])).store_le(&m_DK[4 * 1]);
467 aesimc(SIMD_4x32::load_le(&m_EK[4 * 10])).store_le(&m_DK[4 * 2]);
468 aesimc(SIMD_4x32::load_le(&m_EK[4 * 9])).store_le(&m_DK[4 * 3]);
469 aesimc(SIMD_4x32::load_le(&m_EK[4 * 8])).store_le(&m_DK[4 * 4]);
470 aesimc(SIMD_4x32::load_le(&m_EK[4 * 7])).store_le(&m_DK[4 * 5]);
471 aesimc(SIMD_4x32::load_le(&m_EK[4 * 6])).store_le(&m_DK[4 * 6]);
472 aesimc(SIMD_4x32::load_le(&m_EK[4 * 5])).store_le(&m_DK[4 * 7]);
473 aesimc(SIMD_4x32::load_le(&m_EK[4 * 4])).store_le(&m_DK[4 * 8]);
474 aesimc(SIMD_4x32::load_le(&m_EK[4 * 3])).store_le(&m_DK[4 * 9]);
475 aesimc(SIMD_4x32::load_le(&m_EK[4 * 2])).store_le(&m_DK[4 * 10]);
476 aesimc(SIMD_4x32::load_le(&m_EK[4 * 1])).store_le(&m_DK[4 * 11]);
477 SIMD_4x32::load_le(&m_EK[4 * 0]).store_le(&m_DK[4 * 12]);
478}
479
480/*
481* AES-256 Encryption
482*/
483BOTAN_FN_ISA_AESNI void AES_256::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
484 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_EK[4 * 0]);
485 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_EK[4 * 1]);
486 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_EK[4 * 2]);
487 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_EK[4 * 3]);
488 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_EK[4 * 4]);
489 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_EK[4 * 5]);
490 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_EK[4 * 6]);
491 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_EK[4 * 7]);
492 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_EK[4 * 8]);
493 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_EK[4 * 9]);
494 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_EK[4 * 10]);
495 const SIMD_4x32 K11 = SIMD_4x32::load_le(&m_EK[4 * 11]);
496 const SIMD_4x32 K12 = SIMD_4x32::load_le(&m_EK[4 * 12]);
497 const SIMD_4x32 K13 = SIMD_4x32::load_le(&m_EK[4 * 13]);
498 const SIMD_4x32 K14 = SIMD_4x32::load_le(&m_EK[4 * 14]);
499
500 while(blocks >= 4) {
501 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
502 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
503 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
504 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
505
506 keyxor(K0, B0, B1, B2, B3);
507 aesenc(K1, B0, B1, B2, B3);
508 aesenc(K2, B0, B1, B2, B3);
509 aesenc(K3, B0, B1, B2, B3);
510 aesenc(K4, B0, B1, B2, B3);
511 aesenc(K5, B0, B1, B2, B3);
512 aesenc(K6, B0, B1, B2, B3);
513 aesenc(K7, B0, B1, B2, B3);
514 aesenc(K8, B0, B1, B2, B3);
515 aesenc(K9, B0, B1, B2, B3);
516 aesenc(K10, B0, B1, B2, B3);
517 aesenc(K11, B0, B1, B2, B3);
518 aesenc(K12, B0, B1, B2, B3);
519 aesenc(K13, B0, B1, B2, B3);
520 aesenclast(K14, B0, B1, B2, B3);
521
522 B0.store_le(out + 16 * 0);
523 B1.store_le(out + 16 * 1);
524 B2.store_le(out + 16 * 2);
525 B3.store_le(out + 16 * 3);
526
527 blocks -= 4;
528 in += 4 * 16;
529 out += 4 * 16;
530 }
531
532 for(size_t i = 0; i != blocks; ++i) {
533 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
534
535 B0 ^= K0;
536
537 aesenc(K1, B0);
538 aesenc(K2, B0);
539 aesenc(K3, B0);
540 aesenc(K4, B0);
541 aesenc(K5, B0);
542 aesenc(K6, B0);
543 aesenc(K7, B0);
544 aesenc(K8, B0);
545 aesenc(K9, B0);
546 aesenc(K10, B0);
547 aesenc(K11, B0);
548 aesenc(K12, B0);
549 aesenc(K13, B0);
550 aesenclast(K14, B0);
551
552 B0.store_le(out + 16 * i);
553 }
554}
555
556/*
557* AES-256 Decryption
558*/
559BOTAN_FN_ISA_AESNI void AES_256::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
560 const SIMD_4x32 K0 = SIMD_4x32::load_le(&m_DK[4 * 0]);
561 const SIMD_4x32 K1 = SIMD_4x32::load_le(&m_DK[4 * 1]);
562 const SIMD_4x32 K2 = SIMD_4x32::load_le(&m_DK[4 * 2]);
563 const SIMD_4x32 K3 = SIMD_4x32::load_le(&m_DK[4 * 3]);
564 const SIMD_4x32 K4 = SIMD_4x32::load_le(&m_DK[4 * 4]);
565 const SIMD_4x32 K5 = SIMD_4x32::load_le(&m_DK[4 * 5]);
566 const SIMD_4x32 K6 = SIMD_4x32::load_le(&m_DK[4 * 6]);
567 const SIMD_4x32 K7 = SIMD_4x32::load_le(&m_DK[4 * 7]);
568 const SIMD_4x32 K8 = SIMD_4x32::load_le(&m_DK[4 * 8]);
569 const SIMD_4x32 K9 = SIMD_4x32::load_le(&m_DK[4 * 9]);
570 const SIMD_4x32 K10 = SIMD_4x32::load_le(&m_DK[4 * 10]);
571 const SIMD_4x32 K11 = SIMD_4x32::load_le(&m_DK[4 * 11]);
572 const SIMD_4x32 K12 = SIMD_4x32::load_le(&m_DK[4 * 12]);
573 const SIMD_4x32 K13 = SIMD_4x32::load_le(&m_DK[4 * 13]);
574 const SIMD_4x32 K14 = SIMD_4x32::load_le(&m_DK[4 * 14]);
575
576 while(blocks >= 4) {
577 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * 0);
578 SIMD_4x32 B1 = SIMD_4x32::load_le(in + 16 * 1);
579 SIMD_4x32 B2 = SIMD_4x32::load_le(in + 16 * 2);
580 SIMD_4x32 B3 = SIMD_4x32::load_le(in + 16 * 3);
581
582 keyxor(K0, B0, B1, B2, B3);
583 aesdec(K1, B0, B1, B2, B3);
584 aesdec(K2, B0, B1, B2, B3);
585 aesdec(K3, B0, B1, B2, B3);
586 aesdec(K4, B0, B1, B2, B3);
587 aesdec(K5, B0, B1, B2, B3);
588 aesdec(K6, B0, B1, B2, B3);
589 aesdec(K7, B0, B1, B2, B3);
590 aesdec(K8, B0, B1, B2, B3);
591 aesdec(K9, B0, B1, B2, B3);
592 aesdec(K10, B0, B1, B2, B3);
593 aesdec(K11, B0, B1, B2, B3);
594 aesdec(K12, B0, B1, B2, B3);
595 aesdec(K13, B0, B1, B2, B3);
596 aesdeclast(K14, B0, B1, B2, B3);
597
598 B0.store_le(out + 16 * 0);
599 B1.store_le(out + 16 * 1);
600 B2.store_le(out + 16 * 2);
601 B3.store_le(out + 16 * 3);
602
603 blocks -= 4;
604 in += 4 * 16;
605 out += 4 * 16;
606 }
607
608 for(size_t i = 0; i != blocks; ++i) {
609 SIMD_4x32 B0 = SIMD_4x32::load_le(in + 16 * i);
610
611 B0 ^= K0;
612
613 aesdec(K1, B0);
614 aesdec(K2, B0);
615 aesdec(K3, B0);
616 aesdec(K4, B0);
617 aesdec(K5, B0);
618 aesdec(K6, B0);
619 aesdec(K7, B0);
620 aesdec(K8, B0);
621 aesdec(K9, B0);
622 aesdec(K10, B0);
623 aesdec(K11, B0);
624 aesdec(K12, B0);
625 aesdec(K13, B0);
626 aesdeclast(K14, B0);
627
628 B0.store_le(out + 16 * i);
629 }
630}
631
632/*
633* AES-256 Key Schedule
634*/
635BOTAN_FN_ISA_AESNI void AES_256::aesni_key_schedule(const uint8_t key[], size_t /*length*/) {
636 m_EK.resize(60);
637 m_DK.resize(60);
638
639 const SIMD_4x32 K0 = SIMD_4x32::load_le(key);
640 const SIMD_4x32 K1 = SIMD_4x32::load_le(key + 16);
641
642 const SIMD_4x32 K2 = aes_128_key_expansion<0x01>(K0, K1);
643 const SIMD_4x32 K3 = aes_256_key_expansion(K1, K2);
644
645 const SIMD_4x32 K4 = aes_128_key_expansion<0x02>(K2, K3);
646 const SIMD_4x32 K5 = aes_256_key_expansion(K3, K4);
647
648 const SIMD_4x32 K6 = aes_128_key_expansion<0x04>(K4, K5);
649 const SIMD_4x32 K7 = aes_256_key_expansion(K5, K6);
650
651 const SIMD_4x32 K8 = aes_128_key_expansion<0x08>(K6, K7);
652 const SIMD_4x32 K9 = aes_256_key_expansion(K7, K8);
653
654 const SIMD_4x32 K10 = aes_128_key_expansion<0x10>(K8, K9);
655 const SIMD_4x32 K11 = aes_256_key_expansion(K9, K10);
656
657 const SIMD_4x32 K12 = aes_128_key_expansion<0x20>(K10, K11);
658 const SIMD_4x32 K13 = aes_256_key_expansion(K11, K12);
659
660 const SIMD_4x32 K14 = aes_128_key_expansion<0x40>(K12, K13);
661
662 K0.store_le(&m_EK[4 * 0]);
663 K1.store_le(&m_EK[4 * 1]);
664 K2.store_le(&m_EK[4 * 2]);
665 K3.store_le(&m_EK[4 * 3]);
666 K4.store_le(&m_EK[4 * 4]);
667 K5.store_le(&m_EK[4 * 5]);
668 K6.store_le(&m_EK[4 * 6]);
669 K7.store_le(&m_EK[4 * 7]);
670 K8.store_le(&m_EK[4 * 8]);
671 K9.store_le(&m_EK[4 * 9]);
672 K10.store_le(&m_EK[4 * 10]);
673 K11.store_le(&m_EK[4 * 11]);
674 K12.store_le(&m_EK[4 * 12]);
675 K13.store_le(&m_EK[4 * 13]);
676 K14.store_le(&m_EK[4 * 14]);
677
678 K14.store_le(&m_DK[4 * 0]);
679 aesimc(K13).store_le(&m_DK[4 * 1]);
680 aesimc(K12).store_le(&m_DK[4 * 2]);
681 aesimc(K11).store_le(&m_DK[4 * 3]);
682 aesimc(K10).store_le(&m_DK[4 * 4]);
683 aesimc(K9).store_le(&m_DK[4 * 5]);
684 aesimc(K8).store_le(&m_DK[4 * 6]);
685 aesimc(K7).store_le(&m_DK[4 * 7]);
686 aesimc(K6).store_le(&m_DK[4 * 8]);
687 aesimc(K5).store_le(&m_DK[4 * 9]);
688 aesimc(K4).store_le(&m_DK[4 * 10]);
689 aesimc(K3).store_le(&m_DK[4 * 11]);
690 aesimc(K2).store_le(&m_DK[4 * 12]);
691 aesimc(K1).store_le(&m_DK[4 * 13]);
692 K0.store_le(&m_DK[4 * 14]);
693}
694
695} // namespace Botan
Botan::SIMD_4x32
Definition simd_4x32.h:61
Botan::SIMD_4x32::store_le
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept
Definition simd_4x32.h:219
Botan::SIMD_4x32::load_le
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:162
Botan::SIMD_4x32::shift_elems_right
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shift_elems_right() const noexcept
Definition simd_4x32.h:639
BOTAN_FORCE_INLINE
#define BOTAN_FORCE_INLINE
Definition compiler.h:87
Botan::SHA1_F::K1
constexpr uint32_t K1
Definition sha1_f.h:16
Botan::SHA1_F::K4
constexpr uint32_t K4
Definition sha1_f.h:19
Botan::SHA1_F::K3
constexpr uint32_t K3
Definition sha1_f.h:18
Botan::SHA1_F::K2
constexpr uint32_t K2
Definition sha1_f.h:17
Botan::load_le
constexpr auto load_le(ParamTs &&... params)
Definition loadstor.h:495
Botan::secure_vector
std::vector< T, secure_allocator< T > > secure_vector
Definition secmem.h:128
Generated by doxygen 1.16.1