Botan 3.14.1
Crypto and TLS for C&
ghash_avx512_clmul.cpp
Go to the documentation of this file.
1/*
2* (C) 2026 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/ghash.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/polyval_fn.h>
11#include <botan/internal/simd_4x32.h>
12#include <immintrin.h>
13
14#if defined(BOTAN_HAS_POLYVAL)
15 #include <botan/internal/polyval.h>
16#endif
17
18namespace Botan {
19
20namespace {
21
22template <bool BSWAP>
23BOTAN_FORCE_INLINE __m512i BOTAN_FN_ISA_AVX512_CLMUL load_blocks(const uint8_t in[]) {
24 const auto B = _mm512_loadu_si512(in);
25
26 if constexpr(BSWAP) {
27 // Byte swap each lane
28 const auto swap = _mm512_set_epi64(0x0001020304050607,
29 0x08090A0B0C0D0E0F,
30 0x0001020304050607,
31 0x08090A0B0C0D0E0F,
32 0x0001020304050607,
33 0x08090A0B0C0D0E0F,
34 0x0001020304050607,
35 0x08090A0B0C0D0E0F);
36 return _mm512_shuffle_epi8(B, swap);
37 } else {
38 return B;
39 }
40}
41
42BOTAN_FORCE_INLINE __m512i BOTAN_FN_ISA_AVX512_CLMUL fold(__m512i H) {
43 return _mm512_xor_si512(H, _mm512_bsrli_epi128(H, 8));
44}
45
46BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_AVX512_CLMUL reduce_xor(__m512i z) {
47 const auto y = _mm256_xor_si256(_mm512_castsi512_si256(z), _mm512_extracti64x4_epi64(z, 0x1));
48 const auto x = _mm_xor_si128(_mm256_castsi256_si128(y), _mm256_extracti32x4_epi32(y, 0x1));
49 return SIMD_4x32(x);
50}
51
52BOTAN_FORCE_INLINE void BOTAN_FN_ISA_AVX512_CLMUL
53ghash_x4_accum(__m512i H, __m512i H_fold, __m512i M, __m512i& lo, __m512i& hi, __m512i& mid) {
54 lo = _mm512_xor_si512(lo, _mm512_clmulepi64_epi128(H, M, 0x00));
55 hi = _mm512_xor_si512(hi, _mm512_clmulepi64_epi128(H, M, 0x11));
56 mid = _mm512_xor_si512(mid, _mm512_clmulepi64_epi128(H_fold, fold(M), 0x00));
57}
58
59BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_AVX512_CLMUL ghash_reduce(__m512i lo, __m512i hi, __m512i mid) {
60 mid = _mm512_ternarylogic_epi64(lo, mid, hi, 0x96); // mid ^= lo ^ hi
61 hi = _mm512_xor_si512(hi, _mm512_bsrli_epi128(mid, 8));
62 lo = _mm512_xor_si512(lo, _mm512_bslli_epi128(mid, 8));
63 return polyval_reduce(reduce_xor(hi), reduce_xor(lo));
64}
65
66BOTAN_FORCE_INLINE __m512i BOTAN_FN_ISA_AVX512_CLMUL insert_a(__m512i M, const SIMD_4x32& a) {
67 return _mm512_xor_epi64(M, _mm512_inserti64x2(_mm512_setzero_si512(), a.raw(), 0));
68}
69
70void BOTAN_FN_ISA_AVX512_CLMUL precompute_avx512(const SIMD_4x32& H1, uint64_t H_pow[16 * 2]) {
71 const SIMD_4x32 H2 = polyval_multiply(H1, H1);
72 const SIMD_4x32 H3 = polyval_multiply(H1, H2);
73 const SIMD_4x32 H4 = polyval_multiply(H2, H2);
74
75 const SIMD_4x32 H5 = polyval_multiply(H4, H1);
76 const SIMD_4x32 H6 = polyval_multiply(H4, H2);
77 const SIMD_4x32 H7 = polyval_multiply(H4, H3);
78 const SIMD_4x32 H8 = polyval_multiply(H4, H4);
79
80 const SIMD_4x32 H9 = polyval_multiply(H8, H1);
81 const SIMD_4x32 H10 = polyval_multiply(H8, H2);
82 const SIMD_4x32 H11 = polyval_multiply(H8, H3);
83 const SIMD_4x32 H12 = polyval_multiply(H8, H4);
84
85 const SIMD_4x32 H13 = polyval_multiply(H8, H5);
86 const SIMD_4x32 H14 = polyval_multiply(H8, H6);
87 const SIMD_4x32 H15 = polyval_multiply(H8, H7);
88 const SIMD_4x32 H16 = polyval_multiply(H8, H8);
89
90 // Store in reversed order in blocks of 4 so that the zmm load
91 // of H powers matches up with the message blocks
92 H4.store_le(H_pow);
93 H3.store_le(H_pow + 2);
94 H2.store_le(H_pow + 4);
95 H1.store_le(H_pow + 6);
96
97 H8.store_le(H_pow + 8);
98 H7.store_le(H_pow + 10);
99 H6.store_le(H_pow + 12);
100 H5.store_le(H_pow + 14);
101
102 H12.store_le(H_pow + 16);
103 H11.store_le(H_pow + 18);
104 H10.store_le(H_pow + 20);
105 H9.store_le(H_pow + 22);
106
107 H16.store_le(H_pow + 24);
108 H15.store_le(H_pow + 26);
109 H14.store_le(H_pow + 28);
110 H13.store_le(H_pow + 30);
111}
112
113template <bool BSWAP>
114void BOTAN_FN_ISA_AVX512_CLMUL
115multiply_avx512(uint8_t x[16], const uint64_t H_pow[16 * 2], const uint8_t input[], size_t blocks) {
117
118 if(blocks >= 16) {
119 const auto H1 = _mm512_loadu_si512(H_pow); // [H4,H3,H2,H1]
120 const auto H2 = _mm512_loadu_si512(H_pow + 8); // [H8,H7,H6,H5]
121 const auto H3 = _mm512_loadu_si512(H_pow + 16); // [H12,H11,H10,H9]
122 const auto H4 = _mm512_loadu_si512(H_pow + 24); // [H16,H15,H14,H13]
123
124 // Precompute H folds (H ^ (H >> 64)) for Karatsuba - loop invariant
125 const auto H1_fold = fold(H1);
126 const auto H2_fold = fold(H2);
127 const auto H3_fold = fold(H3);
128 const auto H4_fold = fold(H4);
129
130 while(blocks >= 16) {
131 __m512i M1 = load_blocks<BSWAP>(input);
132 const auto M2 = load_blocks<BSWAP>(input + 64);
133 const auto M3 = load_blocks<BSWAP>(input + 128);
134 const auto M4 = load_blocks<BSWAP>(input + 192);
135
136 M1 = insert_a(M1, a);
137
138 auto lo = _mm512_setzero_si512();
139 auto hi = _mm512_setzero_si512();
140 auto mid = _mm512_setzero_si512();
141
142 ghash_x4_accum(H4, H4_fold, M1, lo, hi, mid);
143 ghash_x4_accum(H3, H3_fold, M2, lo, hi, mid);
144 ghash_x4_accum(H2, H2_fold, M3, lo, hi, mid);
145 ghash_x4_accum(H1, H1_fold, M4, lo, hi, mid);
146
147 a = ghash_reduce(lo, hi, mid);
148
149 input += 16 * 16;
150 blocks -= 16;
151 }
152 }
153
154 if(blocks >= 8) {
155 const auto H1 = _mm512_loadu_si512(H_pow); // [H4,H3,H2,H1]
156 const auto H2 = _mm512_loadu_si512(H_pow + 8); // [H8,H7,H6,H5]
157
158 const auto H1_fold = fold(H1);
159 const auto H2_fold = fold(H2);
160
161 while(blocks >= 8) {
162 __m512i M1 = load_blocks<BSWAP>(input);
163 const __m512i M2 = load_blocks<BSWAP>(input + 64);
164
165 M1 = insert_a(M1, a);
166
167 auto lo = _mm512_setzero_si512();
168 auto hi = _mm512_setzero_si512();
169 auto mid = _mm512_setzero_si512();
170
171 ghash_x4_accum(H2, H2_fold, M1, lo, hi, mid);
172 ghash_x4_accum(H1, H1_fold, M2, lo, hi, mid);
173
174 a = ghash_reduce(lo, hi, mid);
175
176 input += 8 * 16;
177 blocks -= 8;
178 }
179 }
180
181 if(blocks >= 4) {
182 const auto H1 = _mm512_loadu_si512(H_pow); // [H4,H3,H2,H1]
183 const auto H1_fold = fold(H1);
184
185 while(blocks >= 4) {
186 __m512i M = load_blocks<BSWAP>(input);
187 M = insert_a(M, a);
188
189 const auto lo = _mm512_clmulepi64_epi128(H1, M, 0x00);
190 const auto hi = _mm512_clmulepi64_epi128(H1, M, 0x11);
191 const auto mid = _mm512_clmulepi64_epi128(H1_fold, fold(M), 0x00);
192
193 a = ghash_reduce(lo, hi, mid);
194
195 input += 4 * 16;
196 blocks -= 4;
197 }
198 }
199
200 if(blocks > 0) {
201 // H1 is at offset 6 in the reversed layout [H4,H3,H2,H1,...]
202 const SIMD_4x32 H1 = SIMD_4x32::load_le(H_pow + 6);
203
204 for(size_t i = 0; i != blocks; ++i) {
205 const SIMD_4x32 m = load_block<BSWAP>(input + 16 * i);
206 a ^= m;
207 a = polyval_multiply(H1, a);
208 }
209 }
210
211 store_block<BSWAP>(a, x);
212}
213
214} // namespace
215
216void BOTAN_FN_ISA_AVX512_CLMUL GHASH::ghash_precompute_avx512_clmul(const uint8_t H_bytes[16], uint64_t H_pow[16 * 2]) {
217 precompute_avx512(mulx_polyval(SIMD_4x32::load_le(H_bytes).reverse_all_bytes()), H_pow);
218}
219
220void BOTAN_FN_ISA_AVX512_CLMUL GHASH::ghash_multiply_avx512_clmul(uint8_t x[16],
221 const uint64_t H_pow[16 * 2],
222 const uint8_t input[],
223 size_t blocks) {
224 multiply_avx512<true>(x, H_pow, input, blocks);
225}
226
227#if defined(BOTAN_HAS_POLYVAL)
228
229void BOTAN_FN_ISA_AVX512_CLMUL Polyval::polyval_precompute_avx512_clmul(const uint8_t H[16], uint64_t H_pow[16 * 2]) {
230 precompute_avx512(SIMD_4x32::load_le(H), H_pow);
231}
232
233void BOTAN_FN_ISA_AVX512_CLMUL Polyval::polyval_multiply_avx512_clmul(uint8_t x[16],
234 const uint64_t H_pow[16 * 2],
235 const uint8_t input[],
236 size_t blocks) {
237 multiply_avx512<false>(x, H_pow, input, blocks);
238}
239
240#endif
241
242} // namespace Botan
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:183
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 mulx_polyval(const SIMD_4x32 &h)
Definition polyval_fn.h:104
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 load_block(const uint8_t in[])
Definition polyval_fn.h:87
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_multiply(const SIMD_4x32 &H, const SIMD_4x32 &x)
Definition polyval_fn.h:140
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_reduce(const SIMD_4x32 &hi, const SIMD_4x32 &lo)
Definition polyval_fn.h:119
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 void store_block(const SIMD_4x32 &b, uint8_t out[])
Definition polyval_fn.h:96