Botan 3.14.1
Crypto and TLS for C&
sha2_64_avx2.cpp
Go to the documentation of this file.
1/*
2* (C) 2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/sha2_64.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_64_f.h>
11#include <botan/internal/simd_2x64.h>
12#include <botan/internal/simd_4x64.h>
13
14namespace Botan {
15
16namespace {
17
18template <typename SIMD_T>
19BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX2_BMI2 SIMD_T sha512_next_w(SIMD_T x[8]) {
20 const auto t0 = SIMD_T::alignr8(x[1], x[0]);
21 const auto t1 = SIMD_T::alignr8(x[5], x[4]);
22
23 const auto s0 = t0.template rotr<1>() ^ t0.template rotr<8>() ^ t0.template shr<7>();
24 const auto s1 = x[7].template rotr<19>() ^ x[7].template rotr<61>() ^ x[7].template shr<6>();
25
26 const auto nx = x[0] + s0 + s1 + t1;
27
28 x[0] = x[1];
29 x[1] = x[2];
30 x[2] = x[3];
31 x[3] = x[4];
32 x[4] = x[5];
33 x[5] = x[6];
34 x[6] = x[7];
35 x[7] = nx;
36
37 return x[7];
38}
39
40} // namespace
41
42BOTAN_FN_ISA_AVX2_BMI2 void SHA_512::compress_digest_x86_avx2(digest_type& digest,
43 std::span<const uint8_t> input,
44 size_t blocks) {
45 alignas(64) uint64_t W[16] = {0};
46 alignas(64) uint64_t W2[80];
47
48 uint64_t A = digest[0];
49 uint64_t B = digest[1];
50 uint64_t C = digest[2];
51 uint64_t D = digest[3];
52 uint64_t E = digest[4];
53 uint64_t F = digest[5];
54 uint64_t G = digest[6];
55 uint64_t H = digest[7];
56
57 const uint8_t* data = input.data();
58
59 while(blocks >= 2) {
60 SIMD_4x64 WS[8];
61
62 for(size_t i = 0; i < 8; i++) {
63 WS[i] = SIMD_4x64::load_be2(&data[16 * i], &data[128 + 16 * i]);
64 auto WK = WS[i] + SIMD_4x64::broadcast_2x64(&SHA512_K[2 * i]);
65 WK.store_le2(&W[2 * i], &W2[2 * i]);
66 }
67
68 data += 2 * 128;
69 blocks -= 2;
70
71 // First 64 rounds of SHA-512
72 for(size_t r = 0; r != 64; r += 16) {
73 auto w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 16]);
74 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
75 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
76 w.store_le2(&W[0], &W2[r + 16]);
77
78 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 18]);
79 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
80 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
81 w.store_le2(&W[2], &W2[r + 18]);
82
83 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 20]);
84 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
85 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
86 w.store_le2(&W[4], &W2[r + 20]);
87
88 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 22]);
89 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
90 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
91 w.store_le2(&W[6], &W2[r + 22]);
92
93 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 24]);
94 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
95 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
96 w.store_le2(&W[8], &W2[r + 24]);
97
98 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 26]);
99 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
100 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
101 w.store_le2(&W[10], &W2[r + 26]);
102
103 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 28]);
104 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
105 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
106 w.store_le2(&W[12], &W2[r + 28]);
107
108 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&SHA512_K[r + 30]);
109 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
110 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
111 w.store_le2(&W[14], &W2[r + 30]);
112 }
113
114 // Final 16 rounds of SHA-512
115 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
116 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
117 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
118 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
119 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
120 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
121 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
122 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
123 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
124 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
125 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
126 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
127 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
128 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
129 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
130 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
131
132 A = (digest[0] += A);
133 B = (digest[1] += B);
134 C = (digest[2] += C);
135 D = (digest[3] += D);
136 E = (digest[4] += E);
137 F = (digest[5] += F);
138 G = (digest[6] += G);
139 H = (digest[7] += H);
140
141 // Second block of SHA-512 compression, with pre-expanded message
142 SHA2_64_F(A, B, C, D, E, F, G, H, W2[0]);
143 SHA2_64_F(H, A, B, C, D, E, F, G, W2[1]);
144 SHA2_64_F(G, H, A, B, C, D, E, F, W2[2]);
145 SHA2_64_F(F, G, H, A, B, C, D, E, W2[3]);
146 SHA2_64_F(E, F, G, H, A, B, C, D, W2[4]);
147 SHA2_64_F(D, E, F, G, H, A, B, C, W2[5]);
148 SHA2_64_F(C, D, E, F, G, H, A, B, W2[6]);
149 SHA2_64_F(B, C, D, E, F, G, H, A, W2[7]);
150 SHA2_64_F(A, B, C, D, E, F, G, H, W2[8]);
151 SHA2_64_F(H, A, B, C, D, E, F, G, W2[9]);
152 SHA2_64_F(G, H, A, B, C, D, E, F, W2[10]);
153 SHA2_64_F(F, G, H, A, B, C, D, E, W2[11]);
154 SHA2_64_F(E, F, G, H, A, B, C, D, W2[12]);
155 SHA2_64_F(D, E, F, G, H, A, B, C, W2[13]);
156 SHA2_64_F(C, D, E, F, G, H, A, B, W2[14]);
157 SHA2_64_F(B, C, D, E, F, G, H, A, W2[15]);
158
159 SHA2_64_F(A, B, C, D, E, F, G, H, W2[16]);
160 SHA2_64_F(H, A, B, C, D, E, F, G, W2[17]);
161 SHA2_64_F(G, H, A, B, C, D, E, F, W2[18]);
162 SHA2_64_F(F, G, H, A, B, C, D, E, W2[19]);
163 SHA2_64_F(E, F, G, H, A, B, C, D, W2[20]);
164 SHA2_64_F(D, E, F, G, H, A, B, C, W2[21]);
165 SHA2_64_F(C, D, E, F, G, H, A, B, W2[22]);
166 SHA2_64_F(B, C, D, E, F, G, H, A, W2[23]);
167 SHA2_64_F(A, B, C, D, E, F, G, H, W2[24]);
168 SHA2_64_F(H, A, B, C, D, E, F, G, W2[25]);
169 SHA2_64_F(G, H, A, B, C, D, E, F, W2[26]);
170 SHA2_64_F(F, G, H, A, B, C, D, E, W2[27]);
171 SHA2_64_F(E, F, G, H, A, B, C, D, W2[28]);
172 SHA2_64_F(D, E, F, G, H, A, B, C, W2[29]);
173 SHA2_64_F(C, D, E, F, G, H, A, B, W2[30]);
174 SHA2_64_F(B, C, D, E, F, G, H, A, W2[31]);
175
176 SHA2_64_F(A, B, C, D, E, F, G, H, W2[32]);
177 SHA2_64_F(H, A, B, C, D, E, F, G, W2[33]);
178 SHA2_64_F(G, H, A, B, C, D, E, F, W2[34]);
179 SHA2_64_F(F, G, H, A, B, C, D, E, W2[35]);
180 SHA2_64_F(E, F, G, H, A, B, C, D, W2[36]);
181 SHA2_64_F(D, E, F, G, H, A, B, C, W2[37]);
182 SHA2_64_F(C, D, E, F, G, H, A, B, W2[38]);
183 SHA2_64_F(B, C, D, E, F, G, H, A, W2[39]);
184 SHA2_64_F(A, B, C, D, E, F, G, H, W2[40]);
185 SHA2_64_F(H, A, B, C, D, E, F, G, W2[41]);
186 SHA2_64_F(G, H, A, B, C, D, E, F, W2[42]);
187 SHA2_64_F(F, G, H, A, B, C, D, E, W2[43]);
188 SHA2_64_F(E, F, G, H, A, B, C, D, W2[44]);
189 SHA2_64_F(D, E, F, G, H, A, B, C, W2[45]);
190 SHA2_64_F(C, D, E, F, G, H, A, B, W2[46]);
191 SHA2_64_F(B, C, D, E, F, G, H, A, W2[47]);
192
193 SHA2_64_F(A, B, C, D, E, F, G, H, W2[48]);
194 SHA2_64_F(H, A, B, C, D, E, F, G, W2[49]);
195 SHA2_64_F(G, H, A, B, C, D, E, F, W2[50]);
196 SHA2_64_F(F, G, H, A, B, C, D, E, W2[51]);
197 SHA2_64_F(E, F, G, H, A, B, C, D, W2[52]);
198 SHA2_64_F(D, E, F, G, H, A, B, C, W2[53]);
199 SHA2_64_F(C, D, E, F, G, H, A, B, W2[54]);
200 SHA2_64_F(B, C, D, E, F, G, H, A, W2[55]);
201 SHA2_64_F(A, B, C, D, E, F, G, H, W2[56]);
202 SHA2_64_F(H, A, B, C, D, E, F, G, W2[57]);
203 SHA2_64_F(G, H, A, B, C, D, E, F, W2[58]);
204 SHA2_64_F(F, G, H, A, B, C, D, E, W2[59]);
205 SHA2_64_F(E, F, G, H, A, B, C, D, W2[60]);
206 SHA2_64_F(D, E, F, G, H, A, B, C, W2[61]);
207 SHA2_64_F(C, D, E, F, G, H, A, B, W2[62]);
208 SHA2_64_F(B, C, D, E, F, G, H, A, W2[63]);
209
210 SHA2_64_F(A, B, C, D, E, F, G, H, W2[64]);
211 SHA2_64_F(H, A, B, C, D, E, F, G, W2[65]);
212 SHA2_64_F(G, H, A, B, C, D, E, F, W2[66]);
213 SHA2_64_F(F, G, H, A, B, C, D, E, W2[67]);
214 SHA2_64_F(E, F, G, H, A, B, C, D, W2[68]);
215 SHA2_64_F(D, E, F, G, H, A, B, C, W2[69]);
216 SHA2_64_F(C, D, E, F, G, H, A, B, W2[70]);
217 SHA2_64_F(B, C, D, E, F, G, H, A, W2[71]);
218 SHA2_64_F(A, B, C, D, E, F, G, H, W2[72]);
219 SHA2_64_F(H, A, B, C, D, E, F, G, W2[73]);
220 SHA2_64_F(G, H, A, B, C, D, E, F, W2[74]);
221 SHA2_64_F(F, G, H, A, B, C, D, E, W2[75]);
222 SHA2_64_F(E, F, G, H, A, B, C, D, W2[76]);
223 SHA2_64_F(D, E, F, G, H, A, B, C, W2[77]);
224 SHA2_64_F(C, D, E, F, G, H, A, B, W2[78]);
225 SHA2_64_F(B, C, D, E, F, G, H, A, W2[79]);
226
227 A = (digest[0] += A);
228 B = (digest[1] += B);
229 C = (digest[2] += C);
230 D = (digest[3] += D);
231 E = (digest[4] += E);
232 F = (digest[5] += F);
233 G = (digest[6] += G);
234 H = (digest[7] += H);
235 }
236
237 while(blocks > 0) {
238 SIMD_2x64 WS[8];
239
240 for(size_t i = 0; i < 8; i++) {
241 WS[i] = SIMD_2x64::load_be(&data[16 * i]);
242 const auto WK = WS[i] + SIMD_2x64::load_le(&SHA512_K[2 * i]);
243 WK.store_le(&W[2 * i]);
244 }
245
246 data += 128;
247 blocks -= 1;
248
249 // First 64 rounds of SHA-512
250 for(size_t r = 0; r != 64; r += 16) {
251 auto w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 16]);
252 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
253 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
254 w.store_le(&W[0]);
255
256 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 18]);
257 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
258 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
259 w.store_le(&W[2]);
260
261 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 20]);
262 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
263 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
264 w.store_le(&W[4]);
265
266 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 22]);
267 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
268 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
269 w.store_le(&W[6]);
270
271 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 24]);
272 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
273 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
274 w.store_le(&W[8]);
275
276 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 26]);
277 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
278 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
279 w.store_le(&W[10]);
280
281 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 28]);
282 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
283 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
284 w.store_le(&W[12]);
285
286 w = sha512_next_w(WS) + SIMD_2x64::load_le(&SHA512_K[r + 30]);
287 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
288 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
289 w.store_le(&W[14]);
290 }
291
292 // Final 16 rounds of SHA-512
293 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
294 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
295 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
296 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
297 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
298 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
299 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
300 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
301 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
302 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
303 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
304 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
305 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
306 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
307 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
308 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
309
310 A = (digest[0] += A);
311 B = (digest[1] += B);
312 C = (digest[2] += C);
313 D = (digest[3] += D);
314 E = (digest[4] += E);
315 F = (digest[5] += F);
316 G = (digest[6] += G);
317 H = (digest[7] += H);
318 }
319}
320
321} // namespace Botan
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
Definition simd_2x64.h:132
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
Definition simd_2x64.h:124
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t *in)
Definition simd_4x64.h:61
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void *lo, const void *hi)
Definition simd_4x64.h:41
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
constexpr uint64_t SHA512_K[80]
Definition sha2_64_f.h:19
BOTAN_FORCE_INLINE void SHA2_64_F(uint64_t A, uint64_t B, uint64_t C, uint64_t &D, uint64_t E, uint64_t F, uint64_t G, uint64_t &H, uint64_t &M1, uint64_t M2, uint64_t M3, uint64_t M4, uint64_t magic)
Definition sha2_64_f.h:41
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35