Botan 3.14.1
Crypto and TLS for C&
sha2_32_avx2.cpp
Go to the documentation of this file.
1/*
2* (C) 2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/sha2_32.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_32_f.h>
11#include <botan/internal/simd_4x32.h>
12#include <botan/internal/simd_8x32.h>
13#include <botan/internal/stack_scrubbing.h>
14
15#include <immintrin.h>
16
17namespace Botan {
18
19namespace {
20
21BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_4x32 alignr4(const SIMD_4x32& a, const SIMD_4x32& b) {
22 return SIMD_4x32(_mm_alignr_epi8(a.raw(), b.raw(), 4));
23}
24
25template <size_t S>
26BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_4x32 shr64(const SIMD_4x32& a) {
27 return SIMD_4x32(_mm_srli_epi64(a.raw(), S));
28}
29
30template <uint8_t S>
31BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_4x32 shuffle_32(const SIMD_4x32& a) {
32 return SIMD_4x32(_mm_shuffle_epi32(a.raw(), S));
33}
34
35BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_8x32 alignr4(const SIMD_8x32& a, const SIMD_8x32& b) {
36 return SIMD_8x32(_mm256_alignr_epi8(a.raw(), b.raw(), 4));
37}
38
39template <size_t S>
40BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_8x32 shr64(const SIMD_8x32& a) {
41 return SIMD_8x32(_mm256_srli_epi64(a.raw(), S));
42}
43
44template <uint8_t S>
45BOTAN_FN_ISA_AVX2_BMI2 inline SIMD_8x32 shuffle_32(const SIMD_8x32& a) {
46 return SIMD_8x32(_mm256_shuffle_epi32(a.raw(), S));
47}
48
49template <typename SIMD_T>
50BOTAN_FN_ISA_AVX2_BMI2 BOTAN_FORCE_INLINE SIMD_T next_w(SIMD_T x[4]) {
51 constexpr size_t sigma0_0 = 7;
52 constexpr size_t sigma0_1 = 18;
53 constexpr size_t sigma0_2 = 3;
54 constexpr size_t sigma1_0 = 17;
55 constexpr size_t sigma1_1 = 19;
56 constexpr size_t sigma1_2 = 10;
57
58 const SIMD_T lo_mask = SIMD_T(0x03020100, 0x0b0a0908, 0x80808080, 0x80808080);
59 const SIMD_T hi_mask = SIMD_T(0x80808080, 0x80808080, 0x03020100, 0x0b0a0908);
60
61 auto t0 = alignr4(x[1], x[0]);
62 x[0] += alignr4(x[3], x[2]);
63
64 auto t1 = t0.template shl<32 - sigma0_1>();
65 auto t2 = t0.template shr<sigma0_0>();
66 auto t3 = t0.template shr<sigma0_2>();
67 t0 = t3 ^ t2;
68
69 t3 = shuffle_32<0b11111010>(x[3]);
70 t2 = t2.template shr<sigma0_1 - sigma0_0>();
71 t0 ^= t1 ^ t2;
72 t1 = t1.template shl<sigma0_1 - sigma0_0>();
73 t2 = t3.template shr<sigma1_2>();
74 t3 = shr64<sigma1_0>(t3);
75 x[0] += t0 ^ t1;
76
77 t2 ^= t3;
78 t3 = shr64<sigma1_1 - sigma1_0>(t3);
79 x[0] += SIMD_T::byte_shuffle(t2 ^ t3, lo_mask);
80
81 t3 = shuffle_32<0b01010000>(x[0]);
82 t2 = t3.template shr<sigma1_2>();
83 t3 = shr64<sigma1_0>(t3);
84 t2 ^= t3;
85 t3 = shr64<sigma1_1 - sigma1_0>(t3);
86 x[0] += SIMD_T::byte_shuffle(t2 ^ t3, hi_mask);
87
88 const auto tmp = x[0];
89 x[0] = x[1];
90 x[1] = x[2];
91 x[2] = x[3];
92 x[3] = tmp;
93
94 return x[3];
95}
96
97} // namespace
98
99BOTAN_FN_ISA_AVX2_BMI2 BOTAN_SCRUB_STACK_AFTER_RETURN void SHA_256::compress_digest_x86_avx2(
100 digest_type& digest, std::span<const uint8_t> input, size_t blocks) {
101 alignas(64) uint32_t W[16];
102 alignas(64) uint32_t W2[64];
103
104 uint32_t A = digest[0];
105 uint32_t B = digest[1];
106 uint32_t C = digest[2];
107 uint32_t D = digest[3];
108 uint32_t E = digest[4];
109 uint32_t F = digest[5];
110 uint32_t G = digest[6];
111 uint32_t H = digest[7];
112
113 const uint8_t* data = input.data();
114
115 while(blocks >= 2) {
116 SIMD_8x32 WS[4];
117
118 for(size_t i = 0; i < 4; i++) {
119 WS[i] = SIMD_8x32::load_be128(&data[16 * i], &data[64 + 16 * i]);
120 const auto WK = WS[i] + SIMD_8x32::load_le128(&SHA256_K[4 * i]);
121 WK.store_le128(&W[4 * i], &W2[4 * i]);
122 }
123
124 data += 2 * 64;
125 blocks -= 2;
126
127 for(size_t r = 0; r != 48; r += 16) {
128 auto w = next_w(WS) + SIMD_8x32::load_le128(&SHA256_K[r + 16]);
129
130 SHA2_32_F(A, B, C, D, E, F, G, H, W[0]);
131 SHA2_32_F(H, A, B, C, D, E, F, G, W[1]);
132 SHA2_32_F(G, H, A, B, C, D, E, F, W[2]);
133 SHA2_32_F(F, G, H, A, B, C, D, E, W[3]);
134
135 w.store_le128(&W[0], &W2[r + 16]);
136
137 w = next_w(WS) + SIMD_8x32::load_le128(&SHA256_K[r + 20]);
138
139 SHA2_32_F(E, F, G, H, A, B, C, D, W[4]);
140 SHA2_32_F(D, E, F, G, H, A, B, C, W[5]);
141 SHA2_32_F(C, D, E, F, G, H, A, B, W[6]);
142 SHA2_32_F(B, C, D, E, F, G, H, A, W[7]);
143
144 w.store_le128(&W[4], &W2[r + 20]);
145
146 w = next_w(WS) + SIMD_8x32::load_le128(&SHA256_K[r + 24]);
147
148 SHA2_32_F(A, B, C, D, E, F, G, H, W[8]);
149 SHA2_32_F(H, A, B, C, D, E, F, G, W[9]);
150 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
151 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
152
153 w.store_le128(&W[8], &W2[r + 24]);
154
155 w = next_w(WS) + SIMD_8x32::load_le128(&SHA256_K[r + 28]);
156
157 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
158 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
159 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
160 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
161
162 w.store_le128(&W[12], &W2[r + 28]);
163 }
164
165 SHA2_32_F(A, B, C, D, E, F, G, H, W[0]);
166 SHA2_32_F(H, A, B, C, D, E, F, G, W[1]);
167 SHA2_32_F(G, H, A, B, C, D, E, F, W[2]);
168 SHA2_32_F(F, G, H, A, B, C, D, E, W[3]);
169 SHA2_32_F(E, F, G, H, A, B, C, D, W[4]);
170 SHA2_32_F(D, E, F, G, H, A, B, C, W[5]);
171 SHA2_32_F(C, D, E, F, G, H, A, B, W[6]);
172 SHA2_32_F(B, C, D, E, F, G, H, A, W[7]);
173 SHA2_32_F(A, B, C, D, E, F, G, H, W[8]);
174 SHA2_32_F(H, A, B, C, D, E, F, G, W[9]);
175 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
176 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
177 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
178 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
179 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
180 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
181
182 A = (digest[0] += A);
183 B = (digest[1] += B);
184 C = (digest[2] += C);
185 D = (digest[3] += D);
186 E = (digest[4] += E);
187 F = (digest[5] += F);
188 G = (digest[6] += G);
189 H = (digest[7] += H);
190
191 // Now the second block, with already expanded message
192 SHA2_32_F(A, B, C, D, E, F, G, H, W2[0]);
193 SHA2_32_F(H, A, B, C, D, E, F, G, W2[1]);
194 SHA2_32_F(G, H, A, B, C, D, E, F, W2[2]);
195 SHA2_32_F(F, G, H, A, B, C, D, E, W2[3]);
196 SHA2_32_F(E, F, G, H, A, B, C, D, W2[4]);
197 SHA2_32_F(D, E, F, G, H, A, B, C, W2[5]);
198 SHA2_32_F(C, D, E, F, G, H, A, B, W2[6]);
199 SHA2_32_F(B, C, D, E, F, G, H, A, W2[7]);
200 SHA2_32_F(A, B, C, D, E, F, G, H, W2[8]);
201 SHA2_32_F(H, A, B, C, D, E, F, G, W2[9]);
202 SHA2_32_F(G, H, A, B, C, D, E, F, W2[10]);
203 SHA2_32_F(F, G, H, A, B, C, D, E, W2[11]);
204 SHA2_32_F(E, F, G, H, A, B, C, D, W2[12]);
205 SHA2_32_F(D, E, F, G, H, A, B, C, W2[13]);
206 SHA2_32_F(C, D, E, F, G, H, A, B, W2[14]);
207 SHA2_32_F(B, C, D, E, F, G, H, A, W2[15]);
208
209 SHA2_32_F(A, B, C, D, E, F, G, H, W2[16]);
210 SHA2_32_F(H, A, B, C, D, E, F, G, W2[17]);
211 SHA2_32_F(G, H, A, B, C, D, E, F, W2[18]);
212 SHA2_32_F(F, G, H, A, B, C, D, E, W2[19]);
213 SHA2_32_F(E, F, G, H, A, B, C, D, W2[20]);
214 SHA2_32_F(D, E, F, G, H, A, B, C, W2[21]);
215 SHA2_32_F(C, D, E, F, G, H, A, B, W2[22]);
216 SHA2_32_F(B, C, D, E, F, G, H, A, W2[23]);
217 SHA2_32_F(A, B, C, D, E, F, G, H, W2[24]);
218 SHA2_32_F(H, A, B, C, D, E, F, G, W2[25]);
219 SHA2_32_F(G, H, A, B, C, D, E, F, W2[26]);
220 SHA2_32_F(F, G, H, A, B, C, D, E, W2[27]);
221 SHA2_32_F(E, F, G, H, A, B, C, D, W2[28]);
222 SHA2_32_F(D, E, F, G, H, A, B, C, W2[29]);
223 SHA2_32_F(C, D, E, F, G, H, A, B, W2[30]);
224 SHA2_32_F(B, C, D, E, F, G, H, A, W2[31]);
225
226 SHA2_32_F(A, B, C, D, E, F, G, H, W2[32]);
227 SHA2_32_F(H, A, B, C, D, E, F, G, W2[33]);
228 SHA2_32_F(G, H, A, B, C, D, E, F, W2[34]);
229 SHA2_32_F(F, G, H, A, B, C, D, E, W2[35]);
230 SHA2_32_F(E, F, G, H, A, B, C, D, W2[36]);
231 SHA2_32_F(D, E, F, G, H, A, B, C, W2[37]);
232 SHA2_32_F(C, D, E, F, G, H, A, B, W2[38]);
233 SHA2_32_F(B, C, D, E, F, G, H, A, W2[39]);
234 SHA2_32_F(A, B, C, D, E, F, G, H, W2[40]);
235 SHA2_32_F(H, A, B, C, D, E, F, G, W2[41]);
236 SHA2_32_F(G, H, A, B, C, D, E, F, W2[42]);
237 SHA2_32_F(F, G, H, A, B, C, D, E, W2[43]);
238 SHA2_32_F(E, F, G, H, A, B, C, D, W2[44]);
239 SHA2_32_F(D, E, F, G, H, A, B, C, W2[45]);
240 SHA2_32_F(C, D, E, F, G, H, A, B, W2[46]);
241 SHA2_32_F(B, C, D, E, F, G, H, A, W2[47]);
242
243 SHA2_32_F(A, B, C, D, E, F, G, H, W2[48]);
244 SHA2_32_F(H, A, B, C, D, E, F, G, W2[49]);
245 SHA2_32_F(G, H, A, B, C, D, E, F, W2[50]);
246 SHA2_32_F(F, G, H, A, B, C, D, E, W2[51]);
247 SHA2_32_F(E, F, G, H, A, B, C, D, W2[52]);
248 SHA2_32_F(D, E, F, G, H, A, B, C, W2[53]);
249 SHA2_32_F(C, D, E, F, G, H, A, B, W2[54]);
250 SHA2_32_F(B, C, D, E, F, G, H, A, W2[55]);
251 SHA2_32_F(A, B, C, D, E, F, G, H, W2[56]);
252 SHA2_32_F(H, A, B, C, D, E, F, G, W2[57]);
253 SHA2_32_F(G, H, A, B, C, D, E, F, W2[58]);
254 SHA2_32_F(F, G, H, A, B, C, D, E, W2[59]);
255 SHA2_32_F(E, F, G, H, A, B, C, D, W2[60]);
256 SHA2_32_F(D, E, F, G, H, A, B, C, W2[61]);
257 SHA2_32_F(C, D, E, F, G, H, A, B, W2[62]);
258 SHA2_32_F(B, C, D, E, F, G, H, A, W2[63]);
259
260 A = (digest[0] += A);
261 B = (digest[1] += B);
262 C = (digest[2] += C);
263 D = (digest[3] += D);
264 E = (digest[4] += E);
265 F = (digest[5] += F);
266 G = (digest[6] += G);
267 H = (digest[7] += H);
268 }
269
270 while(blocks > 0) {
271 SIMD_4x32 WS[4];
272
273 for(size_t i = 0; i < 4; i++) {
274 WS[i] = SIMD_4x32::load_be(&data[16 * i]);
275 const auto WK = WS[i] + SIMD_4x32::load_le(&SHA256_K[4 * i]);
276 WK.store_le(&W[4 * i]);
277 }
278
279 data += 64;
280 blocks -= 1;
281
282 for(size_t r = 0; r != 48; r += 16) {
283 auto w = next_w(WS) + SIMD_4x32::load_le(&SHA256_K[r + 16]);
284
285 SHA2_32_F(A, B, C, D, E, F, G, H, W[0]);
286 SHA2_32_F(H, A, B, C, D, E, F, G, W[1]);
287 SHA2_32_F(G, H, A, B, C, D, E, F, W[2]);
288 SHA2_32_F(F, G, H, A, B, C, D, E, W[3]);
289
290 w.store_le(&W[0]);
291
292 w = next_w(WS) + SIMD_4x32::load_le(&SHA256_K[r + 20]);
293
294 SHA2_32_F(E, F, G, H, A, B, C, D, W[4]);
295 SHA2_32_F(D, E, F, G, H, A, B, C, W[5]);
296 SHA2_32_F(C, D, E, F, G, H, A, B, W[6]);
297 SHA2_32_F(B, C, D, E, F, G, H, A, W[7]);
298
299 w.store_le(&W[4]);
300
301 w = next_w(WS) + SIMD_4x32::load_le(&SHA256_K[r + 24]);
302
303 SHA2_32_F(A, B, C, D, E, F, G, H, W[8]);
304 SHA2_32_F(H, A, B, C, D, E, F, G, W[9]);
305 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
306 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
307
308 w.store_le(&W[8]);
309
310 w = next_w(WS) + SIMD_4x32::load_le(&SHA256_K[r + 28]);
311
312 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
313 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
314 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
315 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
316
317 w.store_le(&W[12]);
318 }
319
320 SHA2_32_F(A, B, C, D, E, F, G, H, W[0]);
321 SHA2_32_F(H, A, B, C, D, E, F, G, W[1]);
322 SHA2_32_F(G, H, A, B, C, D, E, F, W[2]);
323 SHA2_32_F(F, G, H, A, B, C, D, E, W[3]);
324 SHA2_32_F(E, F, G, H, A, B, C, D, W[4]);
325 SHA2_32_F(D, E, F, G, H, A, B, C, W[5]);
326 SHA2_32_F(C, D, E, F, G, H, A, B, W[6]);
327 SHA2_32_F(B, C, D, E, F, G, H, A, W[7]);
328 SHA2_32_F(A, B, C, D, E, F, G, H, W[8]);
329 SHA2_32_F(H, A, B, C, D, E, F, G, W[9]);
330 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
331 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
332 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
333 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
334 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
335 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
336
337 A = (digest[0] += A);
338 B = (digest[1] += B);
339 C = (digest[2] += C);
340 D = (digest[3] += D);
341 E = (digest[4] += E);
342 F = (digest[5] += F);
343 G = (digest[6] += G);
344 H = (digest[7] += H);
345 }
346}
347
348} // namespace Botan
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(const void *in) noexcept
Definition simd_4x32.h:194
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:183
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le128(const uint8_t *in) noexcept
Definition simd_8x32.h:71
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_be128(const uint8_t in1[], const uint8_t in2[]) noexcept
Definition simd_8x32.h:111
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
BOTAN_FORCE_INLINE void SHA2_32_F(uint32_t A, uint32_t B, uint32_t C, uint32_t &D, uint32_t E, uint32_t F, uint32_t G, uint32_t &H, uint32_t &M1, uint32_t M2, uint32_t M3, uint32_t M4, uint32_t magic)
Definition sha2_32_f.h:33
SIMD_4x32 shl(SIMD_4x32 input)
Definition simd_4x32.h:1004
constexpr uint32_t SHA256_K[64]
Definition sha2_32_f.h:19
#define BOTAN_SCRUB_STACK_AFTER_RETURN