Botan 3.14.1
Crypto and TLS for C&
twofish_avx512.cpp
Go to the documentation of this file.
1/*
2* (C) 2026 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/twofish.h>
8
9#include <botan/mem_ops.h>
10#include <botan/internal/isa_extn.h>
11#include <botan/internal/simd_avx512.h>
12#include <immintrin.h>
13
14namespace Botan {
15
16namespace {
17
18namespace Twofish_AVX512 {
19
20// NOLINTBEGIN(portability-simd-intrinsics)
21
22template <size_t N>
23BOTAN_FN_ISA_AVX512_GFNI BOTAN_FORCE_INLINE __m512i lookup_sbox(const SIMD_16x32 W, const uint8_t* QS) {
24 static_assert(N < 4);
25
26 // Parallel sbox lookup using permutations + blend
27
28 const auto q0 = _mm512_loadu_si512(QS);
29 const auto q1 = _mm512_loadu_si512(QS + 64);
30 const auto q2 = _mm512_loadu_si512(QS + 128);
31 const auto q3 = _mm512_loadu_si512(QS + 192);
32
33 const auto bytemask = _mm512_set1_epi32(0xFF);
34 const auto idx = _mm512_and_si512(_mm512_srli_epi32(W.raw(), N * 8), bytemask);
35
36 // Select on both Q[0-128] and Q[128-256] using the low 7 bits
37 const __m512i lo = _mm512_permutex2var_epi8(q0, idx, q1);
38 const __m512i hi = _mm512_permutex2var_epi8(q2, idx, q3);
39
40 // Then select between those results using the top bit
41 return _mm512_mask_blend_epi8(_mm512_movepi8_mask(idx), lo, hi);
42}
43
44BOTAN_FN_ISA_AVX512_GFNI
45BOTAN_FORCE_INLINE SIMD_16x32 apply_mds(__m512i q, __m512i mds_gfni) {
46 // clang-format off
47 alignas(64) constexpr uint8_t MDS_PRE_SHUFFLE[64] = {
48 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28,
49 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28,
50 32, 36, 40, 44, 48, 52, 56, 60, 32, 36, 40, 44, 48, 52, 56, 60,
51 32, 36, 40, 44, 48, 52, 56, 60, 32, 36, 40, 44, 48, 52, 56, 60,
52 };
53
54 alignas(64) constexpr uint8_t MDS_POST_SHUFFLE[64] = {
55 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
56 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31,
57 32, 40, 48, 56, 33, 41, 49, 57, 34, 42, 50, 58, 35, 43, 51, 59,
58 36, 44, 52, 60, 37, 45, 53, 61, 38, 46, 54, 62, 39, 47, 55, 63,
59 };
60 // clang-format on
61
62 const __m512i pre = _mm512_permutexvar_epi8(_mm512_load_si512(MDS_PRE_SHUFFLE), q);
63 const __m512i transformed = _mm512_gf2p8affine_epi64_epi8(pre, mds_gfni, 0);
64 return SIMD_16x32(_mm512_permutexvar_epi8(_mm512_load_si512(MDS_POST_SHUFFLE), transformed));
65}
66
67BOTAN_FN_ISA_AVX512_GFNI
68BOTAN_FORCE_INLINE SIMD_16x32 g_func(SIMD_16x32 W, const uint8_t* QS) {
69 constexpr uint64_t GFNI_ID = 0x0102040810204080;
70 constexpr uint64_t GFNI_5B = 0x050B162953A24182;
71 constexpr uint64_t GFNI_EF = 0x070F1F3972E3C183;
72
73 const __m512i MDS0 = _mm512_set_epi64(GFNI_EF, GFNI_EF, GFNI_5B, GFNI_ID, GFNI_EF, GFNI_EF, GFNI_5B, GFNI_ID);
74 const __m512i MDS1 = _mm512_set_epi64(GFNI_ID, GFNI_5B, GFNI_EF, GFNI_EF, GFNI_ID, GFNI_5B, GFNI_EF, GFNI_EF);
75 const __m512i MDS2 = _mm512_set_epi64(GFNI_EF, GFNI_ID, GFNI_EF, GFNI_5B, GFNI_EF, GFNI_ID, GFNI_EF, GFNI_5B);
76 const __m512i MDS3 = _mm512_set_epi64(GFNI_5B, GFNI_EF, GFNI_ID, GFNI_5B, GFNI_5B, GFNI_EF, GFNI_ID, GFNI_5B);
77
78 const auto r0 = apply_mds(lookup_sbox<0>(W, QS), MDS0);
79 const auto r1 = apply_mds(lookup_sbox<1>(W, QS + 256), MDS1);
80 const auto r2 = apply_mds(lookup_sbox<2>(W, QS + 512), MDS2);
81 const auto r3 = apply_mds(lookup_sbox<3>(W, QS + 768), MDS3);
82
83 return (r0 ^ r1 ^ r2 ^ r3);
84}
85
86// NOLINTEND(portability-simd-intrinsics)
87
88BOTAN_FN_ISA_AVX512_GFNI
89BOTAN_FORCE_INLINE void twofish_encrypt_round(
90 SIMD_16x32 A, SIMD_16x32 B, SIMD_16x32& C, SIMD_16x32& D, uint32_t rk1, uint32_t rk2, const uint8_t* QS) {
91 SIMD_16x32 X = g_func(A, QS);
92 SIMD_16x32 Y = g_func(B.rotl<8>(), QS);
93
94 X += Y;
95 Y += X;
96
97 X += SIMD_16x32::splat(rk1);
98 Y += SIMD_16x32::splat(rk2);
99
100 C = (C ^ X).rotr<1>();
101 D = D.rotl<1>() ^ Y;
102}
103
104BOTAN_FN_ISA_AVX512_GFNI
105BOTAN_FORCE_INLINE void twofish_decrypt_round(
106 SIMD_16x32 A, SIMD_16x32 B, SIMD_16x32& C, SIMD_16x32& D, uint32_t rk1, uint32_t rk2, const uint8_t* QS) {
107 SIMD_16x32 X = g_func(A, QS);
108 SIMD_16x32 Y = g_func(B.rotl<8>(), QS);
109
110 X += Y;
111 Y += X;
112
113 X += SIMD_16x32::splat(rk1);
114 Y += SIMD_16x32::splat(rk2);
115
116 C = C.rotl<1>() ^ X;
117 D = (D ^ Y).rotr<1>();
118}
119
120BOTAN_FN_ISA_AVX512_GFNI
121void encrypt_16(const uint8_t in[16 * 16], uint8_t out[16 * 16], std::span<const uint32_t> RK, const uint8_t* QS) {
123 SIMD_16x32 B1 = SIMD_16x32::load_le(in + 64);
124 SIMD_16x32 B2 = SIMD_16x32::load_le(in + 128);
125 SIMD_16x32 B3 = SIMD_16x32::load_le(in + 192);
126
127 SIMD_16x32::transpose(B0, B1, B2, B3);
128
129 B0 ^= SIMD_16x32::splat(RK[0]);
130 B1 ^= SIMD_16x32::splat(RK[1]);
131 B2 ^= SIMD_16x32::splat(RK[2]);
132 B3 ^= SIMD_16x32::splat(RK[3]);
133
134 for(size_t k = 8; k != 40; k += 4) {
135 twofish_encrypt_round(B0, B1, B2, B3, RK[k], RK[k + 1], QS);
136 twofish_encrypt_round(B2, B3, B0, B1, RK[k + 2], RK[k + 3], QS);
137 }
138
139 B2 ^= SIMD_16x32::splat(RK[4]);
140 B3 ^= SIMD_16x32::splat(RK[5]);
141 B0 ^= SIMD_16x32::splat(RK[6]);
142 B1 ^= SIMD_16x32::splat(RK[7]);
143
144 SIMD_16x32::transpose(B2, B3, B0, B1);
145
146 B2.store_le(out);
147 B3.store_le(out + 64);
148 B0.store_le(out + 128);
149 B1.store_le(out + 192);
150}
151
152BOTAN_FN_ISA_AVX512_GFNI
153void decrypt_16(const uint8_t in[16 * 16], uint8_t out[16 * 16], std::span<const uint32_t> RK, const uint8_t* QS) {
155 SIMD_16x32 B1 = SIMD_16x32::load_le(in + 64);
156 SIMD_16x32 B2 = SIMD_16x32::load_le(in + 128);
157 SIMD_16x32 B3 = SIMD_16x32::load_le(in + 192);
158
159 SIMD_16x32::transpose(B0, B1, B2, B3);
160
161 B0 ^= SIMD_16x32::splat(RK[4]);
162 B1 ^= SIMD_16x32::splat(RK[5]);
163 B2 ^= SIMD_16x32::splat(RK[6]);
164 B3 ^= SIMD_16x32::splat(RK[7]);
165
166 for(size_t k = 40; k != 8; k -= 4) {
167 twofish_decrypt_round(B0, B1, B2, B3, RK[k - 2], RK[k - 1], QS);
168 twofish_decrypt_round(B2, B3, B0, B1, RK[k - 4], RK[k - 3], QS);
169 }
170
171 B2 ^= SIMD_16x32::splat(RK[0]);
172 B3 ^= SIMD_16x32::splat(RK[1]);
173 B0 ^= SIMD_16x32::splat(RK[2]);
174 B1 ^= SIMD_16x32::splat(RK[3]);
175
176 SIMD_16x32::transpose(B2, B3, B0, B1);
177
178 B2.store_le(out);
179 B3.store_le(out + 64);
180 B0.store_le(out + 128);
181 B1.store_le(out + 192);
182}
183
184} // namespace Twofish_AVX512
185
186} // namespace
187
188void BOTAN_FN_ISA_AVX512_GFNI Twofish::avx512_encrypt(const uint8_t in[], uint8_t out[], size_t blocks) const {
189 using namespace Twofish_AVX512;
190
191 const uint8_t* QS = m_QS.data();
192
193 while(blocks >= 16) {
194 encrypt_16(in, out, m_RK, QS);
195 in += 16 * BLOCK_SIZE;
196 out += 16 * BLOCK_SIZE;
197 blocks -= 16;
198 }
199
200 if(blocks > 0) {
201 uint8_t ibuf[16 * BLOCK_SIZE] = {0};
202 uint8_t obuf[16 * BLOCK_SIZE] = {0};
203 copy_mem(ibuf, in, blocks * BLOCK_SIZE);
204 encrypt_16(ibuf, obuf, m_RK, QS);
205 copy_mem(out, obuf, blocks * BLOCK_SIZE);
206 }
207
209}
210
211void BOTAN_FN_ISA_AVX512_GFNI Twofish::avx512_decrypt(const uint8_t in[], uint8_t out[], size_t blocks) const {
212 using namespace Twofish_AVX512;
213
214 const uint8_t* QS = m_QS.data();
215
216 while(blocks >= 16) {
217 decrypt_16(in, out, m_RK, QS);
218 in += 16 * BLOCK_SIZE;
219 out += 16 * BLOCK_SIZE;
220 blocks -= 16;
221 }
222
223 if(blocks > 0) {
224 uint8_t ibuf[16 * BLOCK_SIZE] = {0};
225 uint8_t obuf[16 * BLOCK_SIZE] = {0};
226 copy_mem(ibuf, in, blocks * BLOCK_SIZE);
227 decrypt_16(ibuf, obuf, m_RK, QS);
228 copy_mem(out, obuf, blocks * BLOCK_SIZE);
229 }
230
232}
233
234} // namespace Botan
static BOTAN_FN_ISA_AVX512 void transpose(SIMD_16x32 &B0, SIMD_16x32 &B1, SIMD_16x32 &B2, SIMD_16x32 &B3)
BOTAN_FN_ISA_AVX512 void store_le(uint8_t out[]) const
Definition simd_avx512.h:80
BOTAN_FN_ISA_AVX512 SIMD_16x32 rotl() const
Definition simd_avx512.h:86
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_le(const uint8_t *in)
Definition simd_avx512.h:63
static BOTAN_FN_ISA_AVX512 void zero_registers()
__m512i BOTAN_FN_ISA_AVX512 raw() const
static BOTAN_FN_ISA_AVX512 SIMD_16x32 splat(uint32_t B)
Definition simd_avx512.h:60
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
constexpr void copy_mem(T *out, const T *in, size_t n)
Definition mem_ops.h:144
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35