Botan 3.11.0
Crypto and TLS for C&
sha2_64_avx2.cpp
Go to the documentation of this file.
1/*
2* (C) 2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/sha2_64.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_64_f.h>
11#include <botan/internal/simd_2x64.h>
12#include <botan/internal/simd_4x64.h>
13
14namespace Botan {
15
16namespace {
17
18template <typename SIMD_T>
19BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX2_BMI2 SIMD_T sha512_next_w(SIMD_T x[8]) {
20 auto t0 = SIMD_T::alignr8(x[1], x[0]);
21 auto t1 = SIMD_T::alignr8(x[5], x[4]);
22
23 auto s0 = t0.template rotr<1>() ^ t0.template rotr<8>() ^ t0.template shr<7>();
24 auto s1 = x[7].template rotr<19>() ^ x[7].template rotr<61>() ^ x[7].template shr<6>();
25
26 auto nx = x[0] + s0 + s1 + t1;
27
28 x[0] = x[1];
29 x[1] = x[2];
30 x[2] = x[3];
31 x[3] = x[4];
32 x[4] = x[5];
33 x[5] = x[6];
34 x[6] = x[7];
35 x[7] = nx;
36
37 return x[7];
38}
39
40} // namespace
41
42BOTAN_FN_ISA_AVX2_BMI2 void SHA_512::compress_digest_x86_avx2(digest_type& digest,
43 std::span<const uint8_t> input,
44 size_t blocks) {
45 // clang-format off
46 alignas(64) const uint64_t K[80] = {
47 0x428A2F98D728AE22, 0x7137449123EF65CD, 0xB5C0FBCFEC4D3B2F, 0xE9B5DBA58189DBBC,
48 0x3956C25BF348B538, 0x59F111F1B605D019, 0x923F82A4AF194F9B, 0xAB1C5ED5DA6D8118,
49 0xD807AA98A3030242, 0x12835B0145706FBE, 0x243185BE4EE4B28C, 0x550C7DC3D5FFB4E2,
50 0x72BE5D74F27B896F, 0x80DEB1FE3B1696B1, 0x9BDC06A725C71235, 0xC19BF174CF692694,
51 0xE49B69C19EF14AD2, 0xEFBE4786384F25E3, 0x0FC19DC68B8CD5B5, 0x240CA1CC77AC9C65,
52 0x2DE92C6F592B0275, 0x4A7484AA6EA6E483, 0x5CB0A9DCBD41FBD4, 0x76F988DA831153B5,
53 0x983E5152EE66DFAB, 0xA831C66D2DB43210, 0xB00327C898FB213F, 0xBF597FC7BEEF0EE4,
54 0xC6E00BF33DA88FC2, 0xD5A79147930AA725, 0x06CA6351E003826F, 0x142929670A0E6E70,
55 0x27B70A8546D22FFC, 0x2E1B21385C26C926, 0x4D2C6DFC5AC42AED, 0x53380D139D95B3DF,
56 0x650A73548BAF63DE, 0x766A0ABB3C77B2A8, 0x81C2C92E47EDAEE6, 0x92722C851482353B,
57 0xA2BFE8A14CF10364, 0xA81A664BBC423001, 0xC24B8B70D0F89791, 0xC76C51A30654BE30,
58 0xD192E819D6EF5218, 0xD69906245565A910, 0xF40E35855771202A, 0x106AA07032BBD1B8,
59 0x19A4C116B8D2D0C8, 0x1E376C085141AB53, 0x2748774CDF8EEB99, 0x34B0BCB5E19B48A8,
60 0x391C0CB3C5C95A63, 0x4ED8AA4AE3418ACB, 0x5B9CCA4F7763E373, 0x682E6FF3D6B2B8A3,
61 0x748F82EE5DEFB2FC, 0x78A5636F43172F60, 0x84C87814A1F0AB72, 0x8CC702081A6439EC,
62 0x90BEFFFA23631E28, 0xA4506CEBDE82BDE9, 0xBEF9A3F7B2C67915, 0xC67178F2E372532B,
63 0xCA273ECEEA26619C, 0xD186B8C721C0C207, 0xEADA7DD6CDE0EB1E, 0xF57D4F7FEE6ED178,
64 0x06F067AA72176FBA, 0x0A637DC5A2C898A6, 0x113F9804BEF90DAE, 0x1B710B35131C471B,
65 0x28DB77F523047D84, 0x32CAAB7B40C72493, 0x3C9EBE0A15C9BEBC, 0x431D67C49C100D4C,
66 0x4CC5D4BECB3E42B6, 0x597F299CFC657E2A, 0x5FCB6FAB3AD6FAEC, 0x6C44198C4A475817,
67 };
68 // clang-format on
69
70 alignas(64) uint64_t W[16] = {0};
71 alignas(64) uint64_t W2[80];
72
73 uint64_t A = digest[0];
74 uint64_t B = digest[1];
75 uint64_t C = digest[2];
76 uint64_t D = digest[3];
77 uint64_t E = digest[4];
78 uint64_t F = digest[5];
79 uint64_t G = digest[6];
80 uint64_t H = digest[7];
81
82 const uint8_t* data = input.data();
83
84 while(blocks >= 2) {
85 SIMD_4x64 WS[8];
86
87 for(size_t i = 0; i < 8; i++) {
88 WS[i] = SIMD_4x64::load_be2(&data[16 * i], &data[128 + 16 * i]);
89 auto WK = WS[i] + SIMD_4x64::broadcast_2x64(&K[2 * i]);
90 WK.store_le2(&W[2 * i], &W2[2 * i]);
91 }
92
93 data += 2 * 128;
94 blocks -= 2;
95
96 // First 64 rounds of SHA-512
97 for(size_t r = 0; r != 64; r += 16) {
98 auto w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 16]);
99 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
100 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
101 w.store_le2(&W[0], &W2[r + 16]);
102
103 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 18]);
104 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
105 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
106 w.store_le2(&W[2], &W2[r + 18]);
107
108 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 20]);
109 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
110 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
111 w.store_le2(&W[4], &W2[r + 20]);
112
113 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 22]);
114 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
115 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
116 w.store_le2(&W[6], &W2[r + 22]);
117
118 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 24]);
119 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
120 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
121 w.store_le2(&W[8], &W2[r + 24]);
122
123 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 26]);
124 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
125 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
126 w.store_le2(&W[10], &W2[r + 26]);
127
128 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 28]);
129 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
130 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
131 w.store_le2(&W[12], &W2[r + 28]);
132
133 w = sha512_next_w(WS) + SIMD_4x64::broadcast_2x64(&K[r + 30]);
134 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
135 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
136 w.store_le2(&W[14], &W2[r + 30]);
137 }
138
139 // Final 16 rounds of SHA-512
140 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
141 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
142 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
143 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
144 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
145 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
146 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
147 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
148 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
149 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
150 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
151 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
152 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
153 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
154 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
155 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
156
157 A = (digest[0] += A);
158 B = (digest[1] += B);
159 C = (digest[2] += C);
160 D = (digest[3] += D);
161 E = (digest[4] += E);
162 F = (digest[5] += F);
163 G = (digest[6] += G);
164 H = (digest[7] += H);
165
166 // Second block of SHA-512 compression, with pre-expanded message
167 SHA2_64_F(A, B, C, D, E, F, G, H, W2[0]);
168 SHA2_64_F(H, A, B, C, D, E, F, G, W2[1]);
169 SHA2_64_F(G, H, A, B, C, D, E, F, W2[2]);
170 SHA2_64_F(F, G, H, A, B, C, D, E, W2[3]);
171 SHA2_64_F(E, F, G, H, A, B, C, D, W2[4]);
172 SHA2_64_F(D, E, F, G, H, A, B, C, W2[5]);
173 SHA2_64_F(C, D, E, F, G, H, A, B, W2[6]);
174 SHA2_64_F(B, C, D, E, F, G, H, A, W2[7]);
175 SHA2_64_F(A, B, C, D, E, F, G, H, W2[8]);
176 SHA2_64_F(H, A, B, C, D, E, F, G, W2[9]);
177 SHA2_64_F(G, H, A, B, C, D, E, F, W2[10]);
178 SHA2_64_F(F, G, H, A, B, C, D, E, W2[11]);
179 SHA2_64_F(E, F, G, H, A, B, C, D, W2[12]);
180 SHA2_64_F(D, E, F, G, H, A, B, C, W2[13]);
181 SHA2_64_F(C, D, E, F, G, H, A, B, W2[14]);
182 SHA2_64_F(B, C, D, E, F, G, H, A, W2[15]);
183
184 SHA2_64_F(A, B, C, D, E, F, G, H, W2[16]);
185 SHA2_64_F(H, A, B, C, D, E, F, G, W2[17]);
186 SHA2_64_F(G, H, A, B, C, D, E, F, W2[18]);
187 SHA2_64_F(F, G, H, A, B, C, D, E, W2[19]);
188 SHA2_64_F(E, F, G, H, A, B, C, D, W2[20]);
189 SHA2_64_F(D, E, F, G, H, A, B, C, W2[21]);
190 SHA2_64_F(C, D, E, F, G, H, A, B, W2[22]);
191 SHA2_64_F(B, C, D, E, F, G, H, A, W2[23]);
192 SHA2_64_F(A, B, C, D, E, F, G, H, W2[24]);
193 SHA2_64_F(H, A, B, C, D, E, F, G, W2[25]);
194 SHA2_64_F(G, H, A, B, C, D, E, F, W2[26]);
195 SHA2_64_F(F, G, H, A, B, C, D, E, W2[27]);
196 SHA2_64_F(E, F, G, H, A, B, C, D, W2[28]);
197 SHA2_64_F(D, E, F, G, H, A, B, C, W2[29]);
198 SHA2_64_F(C, D, E, F, G, H, A, B, W2[30]);
199 SHA2_64_F(B, C, D, E, F, G, H, A, W2[31]);
200
201 SHA2_64_F(A, B, C, D, E, F, G, H, W2[32]);
202 SHA2_64_F(H, A, B, C, D, E, F, G, W2[33]);
203 SHA2_64_F(G, H, A, B, C, D, E, F, W2[34]);
204 SHA2_64_F(F, G, H, A, B, C, D, E, W2[35]);
205 SHA2_64_F(E, F, G, H, A, B, C, D, W2[36]);
206 SHA2_64_F(D, E, F, G, H, A, B, C, W2[37]);
207 SHA2_64_F(C, D, E, F, G, H, A, B, W2[38]);
208 SHA2_64_F(B, C, D, E, F, G, H, A, W2[39]);
209 SHA2_64_F(A, B, C, D, E, F, G, H, W2[40]);
210 SHA2_64_F(H, A, B, C, D, E, F, G, W2[41]);
211 SHA2_64_F(G, H, A, B, C, D, E, F, W2[42]);
212 SHA2_64_F(F, G, H, A, B, C, D, E, W2[43]);
213 SHA2_64_F(E, F, G, H, A, B, C, D, W2[44]);
214 SHA2_64_F(D, E, F, G, H, A, B, C, W2[45]);
215 SHA2_64_F(C, D, E, F, G, H, A, B, W2[46]);
216 SHA2_64_F(B, C, D, E, F, G, H, A, W2[47]);
217
218 SHA2_64_F(A, B, C, D, E, F, G, H, W2[48]);
219 SHA2_64_F(H, A, B, C, D, E, F, G, W2[49]);
220 SHA2_64_F(G, H, A, B, C, D, E, F, W2[50]);
221 SHA2_64_F(F, G, H, A, B, C, D, E, W2[51]);
222 SHA2_64_F(E, F, G, H, A, B, C, D, W2[52]);
223 SHA2_64_F(D, E, F, G, H, A, B, C, W2[53]);
224 SHA2_64_F(C, D, E, F, G, H, A, B, W2[54]);
225 SHA2_64_F(B, C, D, E, F, G, H, A, W2[55]);
226 SHA2_64_F(A, B, C, D, E, F, G, H, W2[56]);
227 SHA2_64_F(H, A, B, C, D, E, F, G, W2[57]);
228 SHA2_64_F(G, H, A, B, C, D, E, F, W2[58]);
229 SHA2_64_F(F, G, H, A, B, C, D, E, W2[59]);
230 SHA2_64_F(E, F, G, H, A, B, C, D, W2[60]);
231 SHA2_64_F(D, E, F, G, H, A, B, C, W2[61]);
232 SHA2_64_F(C, D, E, F, G, H, A, B, W2[62]);
233 SHA2_64_F(B, C, D, E, F, G, H, A, W2[63]);
234
235 SHA2_64_F(A, B, C, D, E, F, G, H, W2[64]);
236 SHA2_64_F(H, A, B, C, D, E, F, G, W2[65]);
237 SHA2_64_F(G, H, A, B, C, D, E, F, W2[66]);
238 SHA2_64_F(F, G, H, A, B, C, D, E, W2[67]);
239 SHA2_64_F(E, F, G, H, A, B, C, D, W2[68]);
240 SHA2_64_F(D, E, F, G, H, A, B, C, W2[69]);
241 SHA2_64_F(C, D, E, F, G, H, A, B, W2[70]);
242 SHA2_64_F(B, C, D, E, F, G, H, A, W2[71]);
243 SHA2_64_F(A, B, C, D, E, F, G, H, W2[72]);
244 SHA2_64_F(H, A, B, C, D, E, F, G, W2[73]);
245 SHA2_64_F(G, H, A, B, C, D, E, F, W2[74]);
246 SHA2_64_F(F, G, H, A, B, C, D, E, W2[75]);
247 SHA2_64_F(E, F, G, H, A, B, C, D, W2[76]);
248 SHA2_64_F(D, E, F, G, H, A, B, C, W2[77]);
249 SHA2_64_F(C, D, E, F, G, H, A, B, W2[78]);
250 SHA2_64_F(B, C, D, E, F, G, H, A, W2[79]);
251
252 A = (digest[0] += A);
253 B = (digest[1] += B);
254 C = (digest[2] += C);
255 D = (digest[3] += D);
256 E = (digest[4] += E);
257 F = (digest[5] += F);
258 G = (digest[6] += G);
259 H = (digest[7] += H);
260 }
261
262 while(blocks > 0) {
263 SIMD_2x64 WS[8];
264
265 for(size_t i = 0; i < 8; i++) {
266 WS[i] = SIMD_2x64::load_be(&data[16 * i]);
267 auto WK = WS[i] + SIMD_2x64::load_le(&K[2 * i]);
268 WK.store_le(&W[2 * i]);
269 }
270
271 data += 128;
272 blocks -= 1;
273
274 // First 64 rounds of SHA-512
275 for(size_t r = 0; r != 64; r += 16) {
276 auto w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 16]);
277 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
278 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
279 w.store_le(&W[0]);
280
281 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 18]);
282 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
283 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
284 w.store_le(&W[2]);
285
286 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 20]);
287 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
288 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
289 w.store_le(&W[4]);
290
291 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 22]);
292 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
293 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
294 w.store_le(&W[6]);
295
296 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 24]);
297 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
298 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
299 w.store_le(&W[8]);
300
301 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 26]);
302 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
303 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
304 w.store_le(&W[10]);
305
306 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 28]);
307 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
308 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
309 w.store_le(&W[12]);
310
311 w = sha512_next_w(WS) + SIMD_2x64::load_le(&K[r + 30]);
312 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
313 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
314 w.store_le(&W[14]);
315 }
316
317 // Final 16 rounds of SHA-512
318 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
319 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
320 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
321 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
322 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
323 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
324 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
325 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
326 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
327 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
328 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
329 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
330 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
331 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
332 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
333 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
334
335 A = (digest[0] += A);
336 B = (digest[1] += B);
337 C = (digest[2] += C);
338 D = (digest[3] += D);
339 E = (digest[4] += E);
340 F = (digest[5] += F);
341 G = (digest[6] += G);
342 H = (digest[7] += H);
343 }
344}
345
346} // namespace Botan
Botan::SIMD_2x64::load_be
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
Definition simd_2x64.h:82
Botan::SIMD_2x64::load_le
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
Definition simd_2x64.h:74
Botan::SIMD_4x64::broadcast_2x64
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t *in)
Definition simd_4x64.h:52
Botan::SIMD_4x64::load_be2
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void *lo, const void *hi)
Definition simd_4x64.h:42
BOTAN_FORCE_INLINE
#define BOTAN_FORCE_INLINE
Definition compiler.h:87
Botan::SHA2_64_F
BOTAN_FORCE_INLINE void SHA2_64_F(uint64_t A, uint64_t B, uint64_t C, uint64_t &D, uint64_t E, uint64_t F, uint64_t G, uint64_t &H, uint64_t &M1, uint64_t M2, uint64_t M3, uint64_t M4, uint64_t magic)
Definition sha2_64_f.h:19
Botan::rotr
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35
Generated by doxygen 1.15.0