Botan 3.9.0
Crypto and TLS for C&
aes_power8.cpp
Go to the documentation of this file.
1/*
2* AES using POWER8/POWER9 crypto extensions
3*
4* Contributed by Jeffrey Walton
5*
6* Further changes
7* (C) 2018,2019 Jack Lloyd
8*
9* Botan is released under the Simplified BSD License (see license.txt)
10*/
11
12#include <botan/internal/aes.h>
13
14#include <botan/internal/isa_extn.h>
15#include <bit>
16
17#include <altivec.h>
18#undef vector
19#undef bool
20
21namespace Botan {
22
23// NOLINTBEGIN(readability-container-data-pointer)
24
25typedef __vector unsigned long long Altivec64x2;
26typedef __vector unsigned int Altivec32x4;
27typedef __vector unsigned char Altivec8x16;
28
29namespace {
30
31static_assert(std::endian::native == std::endian::big || std::endian::native == std::endian::little);
32
34 if constexpr(std::endian::native == std::endian::little) {
35 const Altivec8x16 mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
36 const Altivec8x16 zero = {0};
37 return vec_perm(src, zero, mask);
38 } else {
39 return src;
40 }
41}
42
43BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE Altivec64x2 load_key(const uint32_t key[]) {
44 return reinterpret_cast<Altivec64x2>(reverse_vec(reinterpret_cast<Altivec8x16>(vec_vsx_ld(0, key))));
45}
46
47BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE Altivec64x2 load_block(const uint8_t src[]) {
48 return reinterpret_cast<Altivec64x2>(reverse_vec(vec_vsx_ld(0, src)));
49}
50
51BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void store_block(Altivec64x2 src, uint8_t dest[]) {
52 vec_vsx_st(reverse_vec(reinterpret_cast<Altivec8x16>(src)), 0, dest);
53}
54
55BOTAN_FORCE_INLINE void store_blocks(Altivec64x2 B0, Altivec64x2 B1, Altivec64x2 B2, Altivec64x2 B3, uint8_t out[]) {
56 store_block(B0, out);
57 store_block(B1, out + 16);
58 store_block(B2, out + 16 * 2);
59 store_block(B3, out + 16 * 3);
60}
61
62BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void xor_blocks(
64 B0 = vec_xor(B0, K);
65 B1 = vec_xor(B1, K);
66 B2 = vec_xor(B2, K);
67 B3 = vec_xor(B3, K);
68}
69
70BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vcipher(Altivec64x2& B, Altivec64x2 K) {
71#if defined(__clang__)
72 B = reinterpret_cast<Altivec64x2>(
73 __builtin_crypto_vcipher(reinterpret_cast<Altivec8x16>(B), reinterpret_cast<Altivec8x16>(K)));
74#else
75 B = __builtin_crypto_vcipher(B, K);
76#endif
77}
78
79BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vcipherlast(Altivec64x2& B, Altivec64x2 K) {
80#if defined(__clang__)
81 B = reinterpret_cast<Altivec64x2>(
82 __builtin_crypto_vcipherlast(reinterpret_cast<Altivec8x16>(B), reinterpret_cast<Altivec8x16>(K)));
83#else
84 B = __builtin_crypto_vcipherlast(B, K);
85#endif
86}
87
88BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vncipher(Altivec64x2& B, Altivec64x2 K) {
89#if defined(__clang__)
90 B = reinterpret_cast<Altivec64x2>(
91 __builtin_crypto_vncipher(reinterpret_cast<Altivec8x16>(B), reinterpret_cast<Altivec8x16>(K)));
92#else
93 B = __builtin_crypto_vncipher(B, K);
94#endif
95}
96
97BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vncipherlast(Altivec64x2& B, Altivec64x2 K) {
98#if defined(__clang__)
99 B = reinterpret_cast<Altivec64x2>(
100 __builtin_crypto_vncipherlast(reinterpret_cast<Altivec8x16>(B), reinterpret_cast<Altivec8x16>(K)));
101#else
102 B = __builtin_crypto_vncipherlast(B, K);
103#endif
104}
105
106BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vcipher(
108 aes_vcipher(B0, K);
109 aes_vcipher(B1, K);
110 aes_vcipher(B2, K);
111 aes_vcipher(B3, K);
112}
113
114BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vcipherlast(
116 aes_vcipherlast(B0, K);
117 aes_vcipherlast(B1, K);
118 aes_vcipherlast(B2, K);
119 aes_vcipherlast(B3, K);
120}
121
122BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vncipher(
124 aes_vncipher(B0, K);
125 aes_vncipher(B1, K);
126 aes_vncipher(B2, K);
127 aes_vncipher(B3, K);
128}
129
130BOTAN_FN_ISA_AES BOTAN_FORCE_INLINE void aes_vncipherlast(
132 aes_vncipherlast(B0, K);
133 aes_vncipherlast(B1, K);
134 aes_vncipherlast(B2, K);
135 aes_vncipherlast(B3, K);
136}
137
138} // namespace
139
140BOTAN_FN_ISA_AES void AES_128::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
141 const Altivec64x2 K0 = load_key(&m_EK[0]);
142 const Altivec64x2 K1 = load_key(&m_EK[4]);
143 const Altivec64x2 K2 = load_key(&m_EK[8]);
144 const Altivec64x2 K3 = load_key(&m_EK[12]);
145 const Altivec64x2 K4 = load_key(&m_EK[16]);
146 const Altivec64x2 K5 = load_key(&m_EK[20]);
147 const Altivec64x2 K6 = load_key(&m_EK[24]);
148 const Altivec64x2 K7 = load_key(&m_EK[28]);
149 const Altivec64x2 K8 = load_key(&m_EK[32]);
150 const Altivec64x2 K9 = load_key(&m_EK[36]);
151 const Altivec64x2 K10 = load_key(&m_EK[40]);
152
153 while(blocks >= 4) {
154 Altivec64x2 B0 = load_block(in);
155 Altivec64x2 B1 = load_block(in + 16);
156 Altivec64x2 B2 = load_block(in + 16 * 2);
157 Altivec64x2 B3 = load_block(in + 16 * 3);
158
159 xor_blocks(B0, B1, B2, B3, K0);
160 aes_vcipher(B0, B1, B2, B3, K1);
161 aes_vcipher(B0, B1, B2, B3, K2);
162 aes_vcipher(B0, B1, B2, B3, K3);
163 aes_vcipher(B0, B1, B2, B3, K4);
164 aes_vcipher(B0, B1, B2, B3, K5);
165 aes_vcipher(B0, B1, B2, B3, K6);
166 aes_vcipher(B0, B1, B2, B3, K7);
167 aes_vcipher(B0, B1, B2, B3, K8);
168 aes_vcipher(B0, B1, B2, B3, K9);
169 aes_vcipherlast(B0, B1, B2, B3, K10);
170
171 store_blocks(B0, B1, B2, B3, out);
172
173 out += 4 * 16;
174 in += 4 * 16;
175 blocks -= 4;
176 }
177
178 for(size_t i = 0; i != blocks; ++i) {
179 Altivec64x2 B = load_block(in);
180
181 B = vec_xor(B, K0);
182 aes_vcipher(B, K1);
183 aes_vcipher(B, K2);
184 aes_vcipher(B, K3);
185 aes_vcipher(B, K4);
186 aes_vcipher(B, K5);
187 aes_vcipher(B, K6);
188 aes_vcipher(B, K7);
189 aes_vcipher(B, K8);
190 aes_vcipher(B, K9);
191 aes_vcipherlast(B, K10);
192
193 store_block(B, out);
194
195 out += 16;
196 in += 16;
197 }
198}
199
200BOTAN_FN_ISA_AES void AES_128::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
201 const Altivec64x2 K0 = load_key(&m_EK[40]);
202 const Altivec64x2 K1 = load_key(&m_EK[36]);
203 const Altivec64x2 K2 = load_key(&m_EK[32]);
204 const Altivec64x2 K3 = load_key(&m_EK[28]);
205 const Altivec64x2 K4 = load_key(&m_EK[24]);
206 const Altivec64x2 K5 = load_key(&m_EK[20]);
207 const Altivec64x2 K6 = load_key(&m_EK[16]);
208 const Altivec64x2 K7 = load_key(&m_EK[12]);
209 const Altivec64x2 K8 = load_key(&m_EK[8]);
210 const Altivec64x2 K9 = load_key(&m_EK[4]);
211 const Altivec64x2 K10 = load_key(&m_EK[0]);
212
213 while(blocks >= 4) {
214 Altivec64x2 B0 = load_block(in);
215 Altivec64x2 B1 = load_block(in + 16);
216 Altivec64x2 B2 = load_block(in + 16 * 2);
217 Altivec64x2 B3 = load_block(in + 16 * 3);
218
219 xor_blocks(B0, B1, B2, B3, K0);
220 aes_vncipher(B0, B1, B2, B3, K1);
221 aes_vncipher(B0, B1, B2, B3, K2);
222 aes_vncipher(B0, B1, B2, B3, K3);
223 aes_vncipher(B0, B1, B2, B3, K4);
224 aes_vncipher(B0, B1, B2, B3, K5);
225 aes_vncipher(B0, B1, B2, B3, K6);
226 aes_vncipher(B0, B1, B2, B3, K7);
227 aes_vncipher(B0, B1, B2, B3, K8);
228 aes_vncipher(B0, B1, B2, B3, K9);
229 aes_vncipherlast(B0, B1, B2, B3, K10);
230
231 store_blocks(B0, B1, B2, B3, out);
232
233 out += 4 * 16;
234 in += 4 * 16;
235 blocks -= 4;
236 }
237
238 for(size_t i = 0; i != blocks; ++i) {
239 Altivec64x2 B = load_block(in);
240
241 B = vec_xor(B, K0);
242 aes_vncipher(B, K1);
243 aes_vncipher(B, K2);
244 aes_vncipher(B, K3);
245 aes_vncipher(B, K4);
246 aes_vncipher(B, K5);
247 aes_vncipher(B, K6);
248 aes_vncipher(B, K7);
249 aes_vncipher(B, K8);
250 aes_vncipher(B, K9);
251 aes_vncipherlast(B, K10);
252
253 store_block(B, out);
254
255 out += 16;
256 in += 16;
257 }
258}
259
260BOTAN_FN_ISA_AES void AES_192::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
261 const Altivec64x2 K0 = load_key(&m_EK[0]);
262 const Altivec64x2 K1 = load_key(&m_EK[4]);
263 const Altivec64x2 K2 = load_key(&m_EK[8]);
264 const Altivec64x2 K3 = load_key(&m_EK[12]);
265 const Altivec64x2 K4 = load_key(&m_EK[16]);
266 const Altivec64x2 K5 = load_key(&m_EK[20]);
267 const Altivec64x2 K6 = load_key(&m_EK[24]);
268 const Altivec64x2 K7 = load_key(&m_EK[28]);
269 const Altivec64x2 K8 = load_key(&m_EK[32]);
270 const Altivec64x2 K9 = load_key(&m_EK[36]);
271 const Altivec64x2 K10 = load_key(&m_EK[40]);
272 const Altivec64x2 K11 = load_key(&m_EK[44]);
273 const Altivec64x2 K12 = load_key(&m_EK[48]);
274
275 while(blocks >= 4) {
276 Altivec64x2 B0 = load_block(in);
277 Altivec64x2 B1 = load_block(in + 16);
278 Altivec64x2 B2 = load_block(in + 16 * 2);
279 Altivec64x2 B3 = load_block(in + 16 * 3);
280
281 xor_blocks(B0, B1, B2, B3, K0);
282 aes_vcipher(B0, B1, B2, B3, K1);
283 aes_vcipher(B0, B1, B2, B3, K2);
284 aes_vcipher(B0, B1, B2, B3, K3);
285 aes_vcipher(B0, B1, B2, B3, K4);
286 aes_vcipher(B0, B1, B2, B3, K5);
287 aes_vcipher(B0, B1, B2, B3, K6);
288 aes_vcipher(B0, B1, B2, B3, K7);
289 aes_vcipher(B0, B1, B2, B3, K8);
290 aes_vcipher(B0, B1, B2, B3, K9);
291 aes_vcipher(B0, B1, B2, B3, K10);
292 aes_vcipher(B0, B1, B2, B3, K11);
293 aes_vcipherlast(B0, B1, B2, B3, K12);
294
295 store_blocks(B0, B1, B2, B3, out);
296
297 out += 4 * 16;
298 in += 4 * 16;
299 blocks -= 4;
300 }
301
302 for(size_t i = 0; i != blocks; ++i) {
303 Altivec64x2 B = load_block(in);
304
305 B = vec_xor(B, K0);
306 aes_vcipher(B, K1);
307 aes_vcipher(B, K2);
308 aes_vcipher(B, K3);
309 aes_vcipher(B, K4);
310 aes_vcipher(B, K5);
311 aes_vcipher(B, K6);
312 aes_vcipher(B, K7);
313 aes_vcipher(B, K8);
314 aes_vcipher(B, K9);
315 aes_vcipher(B, K10);
316 aes_vcipher(B, K11);
317 aes_vcipherlast(B, K12);
318
319 store_block(B, out);
320
321 out += 16;
322 in += 16;
323 }
324}
325
326BOTAN_FN_ISA_AES void AES_192::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
327 const Altivec64x2 K0 = load_key(&m_EK[48]);
328 const Altivec64x2 K1 = load_key(&m_EK[44]);
329 const Altivec64x2 K2 = load_key(&m_EK[40]);
330 const Altivec64x2 K3 = load_key(&m_EK[36]);
331 const Altivec64x2 K4 = load_key(&m_EK[32]);
332 const Altivec64x2 K5 = load_key(&m_EK[28]);
333 const Altivec64x2 K6 = load_key(&m_EK[24]);
334 const Altivec64x2 K7 = load_key(&m_EK[20]);
335 const Altivec64x2 K8 = load_key(&m_EK[16]);
336 const Altivec64x2 K9 = load_key(&m_EK[12]);
337 const Altivec64x2 K10 = load_key(&m_EK[8]);
338 const Altivec64x2 K11 = load_key(&m_EK[4]);
339 const Altivec64x2 K12 = load_key(&m_EK[0]);
340
341 while(blocks >= 4) {
342 Altivec64x2 B0 = load_block(in);
343 Altivec64x2 B1 = load_block(in + 16);
344 Altivec64x2 B2 = load_block(in + 16 * 2);
345 Altivec64x2 B3 = load_block(in + 16 * 3);
346
347 xor_blocks(B0, B1, B2, B3, K0);
348 aes_vncipher(B0, B1, B2, B3, K1);
349 aes_vncipher(B0, B1, B2, B3, K2);
350 aes_vncipher(B0, B1, B2, B3, K3);
351 aes_vncipher(B0, B1, B2, B3, K4);
352 aes_vncipher(B0, B1, B2, B3, K5);
353 aes_vncipher(B0, B1, B2, B3, K6);
354 aes_vncipher(B0, B1, B2, B3, K7);
355 aes_vncipher(B0, B1, B2, B3, K8);
356 aes_vncipher(B0, B1, B2, B3, K9);
357 aes_vncipher(B0, B1, B2, B3, K10);
358 aes_vncipher(B0, B1, B2, B3, K11);
359 aes_vncipherlast(B0, B1, B2, B3, K12);
360
361 store_blocks(B0, B1, B2, B3, out);
362
363 out += 4 * 16;
364 in += 4 * 16;
365 blocks -= 4;
366 }
367
368 for(size_t i = 0; i != blocks; ++i) {
369 Altivec64x2 B = load_block(in);
370
371 B = vec_xor(B, K0);
372 aes_vncipher(B, K1);
373 aes_vncipher(B, K2);
374 aes_vncipher(B, K3);
375 aes_vncipher(B, K4);
376 aes_vncipher(B, K5);
377 aes_vncipher(B, K6);
378 aes_vncipher(B, K7);
379 aes_vncipher(B, K8);
380 aes_vncipher(B, K9);
381 aes_vncipher(B, K10);
382 aes_vncipher(B, K11);
383 aes_vncipherlast(B, K12);
384
385 store_block(B, out);
386
387 out += 16;
388 in += 16;
389 }
390}
391
392BOTAN_FN_ISA_AES void AES_256::hw_aes_encrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
393 const Altivec64x2 K0 = load_key(&m_EK[0]);
394 const Altivec64x2 K1 = load_key(&m_EK[4]);
395 const Altivec64x2 K2 = load_key(&m_EK[8]);
396 const Altivec64x2 K3 = load_key(&m_EK[12]);
397 const Altivec64x2 K4 = load_key(&m_EK[16]);
398 const Altivec64x2 K5 = load_key(&m_EK[20]);
399 const Altivec64x2 K6 = load_key(&m_EK[24]);
400 const Altivec64x2 K7 = load_key(&m_EK[28]);
401 const Altivec64x2 K8 = load_key(&m_EK[32]);
402 const Altivec64x2 K9 = load_key(&m_EK[36]);
403 const Altivec64x2 K10 = load_key(&m_EK[40]);
404 const Altivec64x2 K11 = load_key(&m_EK[44]);
405 const Altivec64x2 K12 = load_key(&m_EK[48]);
406 const Altivec64x2 K13 = load_key(&m_EK[52]);
407 const Altivec64x2 K14 = load_key(&m_EK[56]);
408
409 while(blocks >= 4) {
410 Altivec64x2 B0 = load_block(in);
411 Altivec64x2 B1 = load_block(in + 16);
412 Altivec64x2 B2 = load_block(in + 16 * 2);
413 Altivec64x2 B3 = load_block(in + 16 * 3);
414
415 xor_blocks(B0, B1, B2, B3, K0);
416 aes_vcipher(B0, B1, B2, B3, K1);
417 aes_vcipher(B0, B1, B2, B3, K2);
418 aes_vcipher(B0, B1, B2, B3, K3);
419 aes_vcipher(B0, B1, B2, B3, K4);
420 aes_vcipher(B0, B1, B2, B3, K5);
421 aes_vcipher(B0, B1, B2, B3, K6);
422 aes_vcipher(B0, B1, B2, B3, K7);
423 aes_vcipher(B0, B1, B2, B3, K8);
424 aes_vcipher(B0, B1, B2, B3, K9);
425 aes_vcipher(B0, B1, B2, B3, K10);
426 aes_vcipher(B0, B1, B2, B3, K11);
427 aes_vcipher(B0, B1, B2, B3, K12);
428 aes_vcipher(B0, B1, B2, B3, K13);
429 aes_vcipherlast(B0, B1, B2, B3, K14);
430
431 store_blocks(B0, B1, B2, B3, out);
432
433 out += 4 * 16;
434 in += 4 * 16;
435 blocks -= 4;
436 }
437
438 for(size_t i = 0; i != blocks; ++i) {
439 Altivec64x2 B = load_block(in);
440
441 B = vec_xor(B, K0);
442 aes_vcipher(B, K1);
443 aes_vcipher(B, K2);
444 aes_vcipher(B, K3);
445 aes_vcipher(B, K4);
446 aes_vcipher(B, K5);
447 aes_vcipher(B, K6);
448 aes_vcipher(B, K7);
449 aes_vcipher(B, K8);
450 aes_vcipher(B, K9);
451 aes_vcipher(B, K10);
452 aes_vcipher(B, K11);
453 aes_vcipher(B, K12);
454 aes_vcipher(B, K13);
455 aes_vcipherlast(B, K14);
456
457 store_block(B, out);
458
459 out += 16;
460 in += 16;
461 }
462}
463
464BOTAN_FN_ISA_AES void AES_256::hw_aes_decrypt_n(const uint8_t in[], uint8_t out[], size_t blocks) const {
465 const Altivec64x2 K0 = load_key(&m_EK[56]);
466 const Altivec64x2 K1 = load_key(&m_EK[52]);
467 const Altivec64x2 K2 = load_key(&m_EK[48]);
468 const Altivec64x2 K3 = load_key(&m_EK[44]);
469 const Altivec64x2 K4 = load_key(&m_EK[40]);
470 const Altivec64x2 K5 = load_key(&m_EK[36]);
471 const Altivec64x2 K6 = load_key(&m_EK[32]);
472 const Altivec64x2 K7 = load_key(&m_EK[28]);
473 const Altivec64x2 K8 = load_key(&m_EK[24]);
474 const Altivec64x2 K9 = load_key(&m_EK[20]);
475 const Altivec64x2 K10 = load_key(&m_EK[16]);
476 const Altivec64x2 K11 = load_key(&m_EK[12]);
477 const Altivec64x2 K12 = load_key(&m_EK[8]);
478 const Altivec64x2 K13 = load_key(&m_EK[4]);
479 const Altivec64x2 K14 = load_key(&m_EK[0]);
480
481 while(blocks >= 4) {
482 Altivec64x2 B0 = load_block(in);
483 Altivec64x2 B1 = load_block(in + 16);
484 Altivec64x2 B2 = load_block(in + 16 * 2);
485 Altivec64x2 B3 = load_block(in + 16 * 3);
486
487 xor_blocks(B0, B1, B2, B3, K0);
488 aes_vncipher(B0, B1, B2, B3, K1);
489 aes_vncipher(B0, B1, B2, B3, K2);
490 aes_vncipher(B0, B1, B2, B3, K3);
491 aes_vncipher(B0, B1, B2, B3, K4);
492 aes_vncipher(B0, B1, B2, B3, K5);
493 aes_vncipher(B0, B1, B2, B3, K6);
494 aes_vncipher(B0, B1, B2, B3, K7);
495 aes_vncipher(B0, B1, B2, B3, K8);
496 aes_vncipher(B0, B1, B2, B3, K9);
497 aes_vncipher(B0, B1, B2, B3, K10);
498 aes_vncipher(B0, B1, B2, B3, K11);
499 aes_vncipher(B0, B1, B2, B3, K12);
500 aes_vncipher(B0, B1, B2, B3, K13);
501 aes_vncipherlast(B0, B1, B2, B3, K14);
502
503 store_blocks(B0, B1, B2, B3, out);
504
505 out += 4 * 16;
506 in += 4 * 16;
507 blocks -= 4;
508 }
509
510 for(size_t i = 0; i != blocks; ++i) {
511 Altivec64x2 B = load_block(in);
512
513 B = vec_xor(B, K0);
514 aes_vncipher(B, K1);
515 aes_vncipher(B, K2);
516 aes_vncipher(B, K3);
517 aes_vncipher(B, K4);
518 aes_vncipher(B, K5);
519 aes_vncipher(B, K6);
520 aes_vncipher(B, K7);
521 aes_vncipher(B, K8);
522 aes_vncipher(B, K9);
523 aes_vncipher(B, K10);
524 aes_vncipher(B, K11);
525 aes_vncipher(B, K12);
526 aes_vncipher(B, K13);
527 aes_vncipherlast(B, K14);
528
529 store_block(B, out);
530
531 out += 16;
532 in += 16;
533 }
534}
535
536// NOLINTEND(readability-container-data-pointer)
537
538} // namespace Botan
#define BOTAN_FORCE_INLINE
Definition compiler.h:87
std::unique_ptr< Private_Key > load_key(DataSource &source, const std::function< std::string()> &get_pass)
Definition pkcs8.cpp:314
constexpr uint32_t K1
Definition sha1_f.h:16
constexpr uint32_t K4
Definition sha1_f.h:19
constexpr uint32_t K3
Definition sha1_f.h:18
constexpr uint32_t K2
Definition sha1_f.h:17
__vector unsigned long long Altivec64x2
__vector unsigned char Altivec8x16
__vector unsigned int Altivec32x4