7#ifndef BOTAN_SIMD_HWAES_H_
8#define BOTAN_SIMD_HWAES_H_
10#include <botan/internal/gfni_utils.h>
11#include <botan/internal/isa_extn.h>
12#include <botan/internal/simd_4x32.h>
14#if defined(BOTAN_SIMD_USE_SSSE3)
15 #include <wmmintrin.h>
18#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
24#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
29 #if defined(__clang__)
30using HwAesVec = __vector
unsigned char;
32using HwAesVec = __vector
unsigned long long;
35inline HwAesVec BOTAN_FN_ISA_HWAES hw_aes_vec(SIMD_4x32 x) {
36 return reinterpret_cast<HwAesVec
>(x.raw());
39inline SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_simd(HwAesVec x) {
40 return SIMD_4x32(
reinterpret_cast<__vector
unsigned int>(x));
44inline __vector
unsigned char BOTAN_FN_ISA_HWAES hw_aes_state_order(__vector
unsigned char v) {
45 if constexpr(std::endian::native == std::endian::little) {
46 const __vector
unsigned char rev = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
47 return vec_perm(v, v, rev);
61#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
63 return detail::hw_aes_simd(__builtin_crypto_vsbox(detail::hw_aes_vec(x)));
66 const auto inv_sr =
SIMD_4x32(0x070A0D00, 0x0B0E0104, 0x0F020508, 0x0306090C);
68 #if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
69 const auto enc =
SIMD_4x32(_mm_aesenclast_si128(x.
raw(), _mm_setzero_si128()));
70 #elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
71 const auto enc =
SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(x.
raw()), vdupq_n_u8(0))));
73 #error "hw_aes_sbox not implemented for this architecture"
84#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
90 const auto sr = (std::endian::native == std::endian::little)
91 ?
SIMD_4x32(0x030E0904, 0x07020D08, 0x0B06010C, 0x0F0A0500)
92 :
SIMD_4x32(0x0306090C, 0x070A0D00, 0x0B0E0104, 0x0F020508);
95 const auto sr =
SIMD_4x32(0x0F0A0500, 0x030E0904, 0x07020D08, 0x0B06010C);
98#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
99 const auto dec =
SIMD_4x32(_mm_aesdeclast_si128(x.
raw(), _mm_setzero_si128()));
100#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
101 const auto dec =
SIMD_4x32(vreinterpretq_u32_u8(vaesdq_u8(vreinterpretq_u8_u32(x.
raw()), vdupq_n_u8(0))));
102#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
104 detail::hw_aes_simd(__builtin_crypto_vncipherlast(detail::hw_aes_vec(x), detail::hw_aes_vec(
SIMD_4x32())));
106 #error "hw_aes_inv_sbox not implemented for this architecture"
120#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
122#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
124 return SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(x.
raw()), vdupq_n_u8(0)))) ^ k;
125#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
126 return detail::hw_aes_simd(__builtin_crypto_vcipherlast(detail::hw_aes_vec(x), detail::hw_aes_vec(k)));
128 #error "hw_aes_sbox_splat not implemented for this architecture"
136#if defined(BOTAN_TARGET_ARCH_IS_ARM64)
146#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
147 return SIMD_4x32(
reinterpret_cast<__vector
unsigned int>(detail::hw_aes_state_order(vec_xl(0, in))));
157#if defined(BOTAN_TARGET_ARCH_IS_PPC64)
158 vec_xst(detail::hw_aes_state_order(
reinterpret_cast<__vector
unsigned char>(B.
raw())), 0, out);
168#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
170#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
172 vreinterpretq_u32_u8(vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_u32(B.
raw()), vreinterpretq_u8_u32(K.
raw())))));
173#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
174 return detail::hw_aes_simd(__builtin_crypto_vcipher(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
176 #error "hw_aes_enc_round not implemented for this architecture"
184#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
186#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
187 return SIMD_4x32(vreinterpretq_u32_u8(vaeseq_u8(vreinterpretq_u8_u32(B.
raw()), vreinterpretq_u8_u32(K.
raw()))));
188#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
189 return detail::hw_aes_simd(__builtin_crypto_vcipherlast(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
191 #error "hw_aes_enc_last_round not implemented for this architecture"
199#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
201#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
203 vreinterpretq_u32_u8(vaesimcq_u8(vaesdq_u8(vreinterpretq_u8_u32(B.
raw()), vreinterpretq_u8_u32(K.
raw())))));
204#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
205 return detail::hw_aes_simd(__builtin_crypto_vncipher(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
207 #error "hw_aes_dec_round not implemented for this architecture"
215#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
217#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
218 return SIMD_4x32(vreinterpretq_u32_u8(vaesdq_u8(vreinterpretq_u8_u32(B.
raw()), vreinterpretq_u8_u32(K.
raw()))));
219#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
220 return detail::hw_aes_simd(__builtin_crypto_vncipherlast(detail::hw_aes_vec(B), detail::hw_aes_vec(K)));
222 #error "hw_aes_dec_last_round not implemented for this architecture"
234#if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
236#elif defined(BOTAN_TARGET_ARCH_IS_ARM64)
237 return SIMD_4x32(vreinterpretq_u32_u8(vaesimcq_u8(vreinterpretq_u8_u32(K.
raw()))));
238#elif defined(BOTAN_TARGET_ARCH_IS_PPC64)
241 #error "hw_aes_dec_round_key not implemented for this architecture"
253 for(
size_t i = 0; i != 8; ++i) {
255 for(
size_t j = 0; j != 8; ++j) {
256 if(((M >> (56 - 8 * i + j)) & 1) == 1) {
270 for(
size_t i = 0; i != 8; ++i) {
271 for(
size_t j = 0; j != 8; ++j) {
273 for(
size_t k = 0; k != 8; ++k) {
274 auto a_ik =
static_cast<uint8_t
>((A >> (56 - 8 * i + k)) & 1);
275 auto b_kj =
static_cast<uint8_t
>((B >> (56 - 8 * k + j)) & 1);
279 result |= uint64_t(1) << (56 - 8 * i + j);
318 for(
size_t i = 0; i != 16; ++i) {
323 lo[i / 4] |=
static_cast<uint32_t
>(lo_val) << (8 * (i % 4));
324 hi[i / 4] |=
static_cast<uint32_t
>(hi_val) << (8 * (i % 4));
349 const SIMD_4x32 tbl_lo(lo[0], lo[1], lo[2], lo[3]);
350 const SIMD_4x32 tbl_hi(hi[0], hi[1], hi[2], hi[3]);
void BOTAN_FN_ISA_SIMD_4X32 store_le(uint32_t out[4]) const noexcept
SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 shr() const noexcept
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 byte_shuffle(const SIMD_4x32 &tbl, const SIMD_4x32 &idx)
native_simd_type BOTAN_FN_ISA_SIMD_4X32 raw() const noexcept
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 splat_u8(uint8_t B) noexcept
constexpr uint64_t AES_AFF
consteval uint8_t gf2_mat_vec(uint64_t M, uint8_t x)
consteval uint64_t gf2_mat_mul(uint64_t A, uint64_t B)
constexpr uint64_t AES_AFF_INV
constexpr uint8_t AES_C_INV
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_last_round(SIMD_4x32 B, SIMD_4x32 K)
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox_splat(SIMD_4x32 x, SIMD_4x32 k)
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round_key(SIMD_4x32 K)
consteval uint64_t gfni_matrix(std::string_view s)
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_last_round(SIMD_4x32 B, SIMD_4x32 K)
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_enc_round(SIMD_4x32 B, SIMD_4x32 K)
constexpr bool HwAesKeyFirst
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_dec_round(SIMD_4x32 B, SIMD_4x32 K)
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_load_block(const uint8_t in[16])
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_inv_sbox(SIMD_4x32 x)
void BOTAN_FN_ISA_HWAES hw_aes_store_block(SIMD_4x32 B, uint8_t out[16])
SIMD_4x32 BOTAN_FN_ISA_HWAES hw_aes_sbox(SIMD_4x32 x)