Botan 3.11.0
Crypto and TLS for C&
sha2_64_avx512.cpp
Go to the documentation of this file.
1/*
2* (C) 2025 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/sha2_64.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_64_f.h>
11#include <botan/internal/simd_2x64.h>
12#include <botan/internal/simd_8x64.h>
13
14namespace Botan {
15
16namespace {
17
18template <typename SIMD_T>
19BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX512_BMI2 SIMD_T sha512_next_w_avx512(SIMD_T x[8]) {
20 auto t0 = SIMD_T::alignr8(x[1], x[0]);
21 auto t1 = SIMD_T::alignr8(x[5], x[4]);
22
23 auto s0 = t0.template rotr<1>() ^ t0.template rotr<8>() ^ t0.template shr<7>();
24 auto s1 = x[7].template rotr<19>() ^ x[7].template rotr<61>() ^ x[7].template shr<6>();
25
26 auto nx = x[0] + s0 + s1 + t1;
27
28 x[0] = x[1];
29 x[1] = x[2];
30 x[2] = x[3];
31 x[3] = x[4];
32 x[4] = x[5];
33 x[5] = x[6];
34 x[6] = x[7];
35 x[7] = nx;
36
37 return x[7];
38}
39
40} // namespace
41
42BOTAN_FN_ISA_AVX512_BMI2 void SHA_512::compress_digest_x86_avx512(digest_type& digest,
43 std::span<const uint8_t> input,
44 size_t blocks) {
45 // clang-format off
46 alignas(64) const uint64_t K[80] = {
47 0x428A2F98D728AE22, 0x7137449123EF65CD, 0xB5C0FBCFEC4D3B2F, 0xE9B5DBA58189DBBC,
48 0x3956C25BF348B538, 0x59F111F1B605D019, 0x923F82A4AF194F9B, 0xAB1C5ED5DA6D8118,
49 0xD807AA98A3030242, 0x12835B0145706FBE, 0x243185BE4EE4B28C, 0x550C7DC3D5FFB4E2,
50 0x72BE5D74F27B896F, 0x80DEB1FE3B1696B1, 0x9BDC06A725C71235, 0xC19BF174CF692694,
51 0xE49B69C19EF14AD2, 0xEFBE4786384F25E3, 0x0FC19DC68B8CD5B5, 0x240CA1CC77AC9C65,
52 0x2DE92C6F592B0275, 0x4A7484AA6EA6E483, 0x5CB0A9DCBD41FBD4, 0x76F988DA831153B5,
53 0x983E5152EE66DFAB, 0xA831C66D2DB43210, 0xB00327C898FB213F, 0xBF597FC7BEEF0EE4,
54 0xC6E00BF33DA88FC2, 0xD5A79147930AA725, 0x06CA6351E003826F, 0x142929670A0E6E70,
55 0x27B70A8546D22FFC, 0x2E1B21385C26C926, 0x4D2C6DFC5AC42AED, 0x53380D139D95B3DF,
56 0x650A73548BAF63DE, 0x766A0ABB3C77B2A8, 0x81C2C92E47EDAEE6, 0x92722C851482353B,
57 0xA2BFE8A14CF10364, 0xA81A664BBC423001, 0xC24B8B70D0F89791, 0xC76C51A30654BE30,
58 0xD192E819D6EF5218, 0xD69906245565A910, 0xF40E35855771202A, 0x106AA07032BBD1B8,
59 0x19A4C116B8D2D0C8, 0x1E376C085141AB53, 0x2748774CDF8EEB99, 0x34B0BCB5E19B48A8,
60 0x391C0CB3C5C95A63, 0x4ED8AA4AE3418ACB, 0x5B9CCA4F7763E373, 0x682E6FF3D6B2B8A3,
61 0x748F82EE5DEFB2FC, 0x78A5636F43172F60, 0x84C87814A1F0AB72, 0x8CC702081A6439EC,
62 0x90BEFFFA23631E28, 0xA4506CEBDE82BDE9, 0xBEF9A3F7B2C67915, 0xC67178F2E372532B,
63 0xCA273ECEEA26619C, 0xD186B8C721C0C207, 0xEADA7DD6CDE0EB1E, 0xF57D4F7FEE6ED178,
64 0x06F067AA72176FBA, 0x0A637DC5A2C898A6, 0x113F9804BEF90DAE, 0x1B710B35131C471B,
65 0x28DB77F523047D84, 0x32CAAB7B40C72493, 0x3C9EBE0A15C9BEBC, 0x431D67C49C100D4C,
66 0x4CC5D4BECB3E42B6, 0x597F299CFC657E2A, 0x5FCB6FAB3AD6FAEC, 0x6C44198C4A475817,
67 };
68
69 // clang-format on
70
71 alignas(64) uint64_t W[16] = {0};
72 alignas(64) uint64_t WN[3][80];
73
74 uint64_t A = digest[0];
75 uint64_t B = digest[1];
76 uint64_t C = digest[2];
77 uint64_t D = digest[3];
78 uint64_t E = digest[4];
79 uint64_t F = digest[5];
80 uint64_t G = digest[6];
81 uint64_t H = digest[7];
82
83 const uint8_t* data = input.data();
84
85 while(blocks >= 4) {
86 SIMD_8x64 WS[8];
87
88 for(size_t i = 0; i < 8; i++) {
89 WS[i] = SIMD_8x64::load_be4(
90 &data[16 * i], &data[1 * 128 + 16 * i], &data[2 * 128 + 16 * i], &data[3 * 128 + 16 * i]);
91 auto WK = WS[i] + SIMD_8x64::broadcast_2x64(&K[2 * i]);
92 WK.store_le4(&W[2 * i], &WN[0][2 * i], &WN[1][2 * i], &WN[2][2 * i]);
93 }
94
95 data += 4 * 128;
96 blocks -= 4;
97
98 // First 64 rounds of SHA-512
99 for(size_t r = 0; r != 64; r += 16) {
100 auto w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 16]);
101 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
102 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
103 w.store_le4(&W[0], &WN[0][r + 16], &WN[1][r + 16], &WN[2][r + 16]);
104
105 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 18]);
106 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
107 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
108 w.store_le4(&W[2], &WN[0][r + 18], &WN[1][r + 18], &WN[2][r + 18]);
109
110 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 20]);
111 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
112 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
113 w.store_le4(&W[4], &WN[0][r + 20], &WN[1][r + 20], &WN[2][r + 20]);
114
115 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 22]);
116 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
117 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
118 w.store_le4(&W[6], &WN[0][r + 22], &WN[1][r + 22], &WN[2][r + 22]);
119
120 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 24]);
121 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
122 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
123 w.store_le4(&W[8], &WN[0][r + 24], &WN[1][r + 24], &WN[2][r + 24]);
124
125 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 26]);
126 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
127 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
128 w.store_le4(&W[10], &WN[0][r + 26], &WN[1][r + 26], &WN[2][r + 26]);
129
130 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 28]);
131 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
132 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
133 w.store_le4(&W[12], &WN[0][r + 28], &WN[1][r + 28], &WN[2][r + 28]);
134
135 w = sha512_next_w_avx512(WS) + SIMD_8x64::broadcast_2x64(&K[r + 30]);
136 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
137 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
138 w.store_le4(&W[14], &WN[0][r + 30], &WN[1][r + 30], &WN[2][r + 30]);
139 }
140
141 // Final 16 rounds of SHA-512
142 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
143 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
144 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
145 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
146 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
147 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
148 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
149 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
150 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
151 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
152 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
153 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
154 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
155 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
156 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
157 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
158
159 A = (digest[0] += A);
160 B = (digest[1] += B);
161 C = (digest[2] += C);
162 D = (digest[3] += D);
163 E = (digest[4] += E);
164 F = (digest[5] += F);
165 G = (digest[6] += G);
166 H = (digest[7] += H);
167
168 // Block 2,3,4 of SHA-512 compression, with pre-expanded messages
169 for(size_t b = 0; b != 3; ++b) { // NOLINT(*-loop-convert)
170 for(size_t r = 0; r != 80; r += 16) {
171 SHA2_64_F(A, B, C, D, E, F, G, H, WN[b][r + 0]);
172 SHA2_64_F(H, A, B, C, D, E, F, G, WN[b][r + 1]);
173 SHA2_64_F(G, H, A, B, C, D, E, F, WN[b][r + 2]);
174 SHA2_64_F(F, G, H, A, B, C, D, E, WN[b][r + 3]);
175 SHA2_64_F(E, F, G, H, A, B, C, D, WN[b][r + 4]);
176 SHA2_64_F(D, E, F, G, H, A, B, C, WN[b][r + 5]);
177 SHA2_64_F(C, D, E, F, G, H, A, B, WN[b][r + 6]);
178 SHA2_64_F(B, C, D, E, F, G, H, A, WN[b][r + 7]);
179 SHA2_64_F(A, B, C, D, E, F, G, H, WN[b][r + 8]);
180 SHA2_64_F(H, A, B, C, D, E, F, G, WN[b][r + 9]);
181 SHA2_64_F(G, H, A, B, C, D, E, F, WN[b][r + 10]);
182 SHA2_64_F(F, G, H, A, B, C, D, E, WN[b][r + 11]);
183 SHA2_64_F(E, F, G, H, A, B, C, D, WN[b][r + 12]);
184 SHA2_64_F(D, E, F, G, H, A, B, C, WN[b][r + 13]);
185 SHA2_64_F(C, D, E, F, G, H, A, B, WN[b][r + 14]);
186 SHA2_64_F(B, C, D, E, F, G, H, A, WN[b][r + 15]);
187 }
188
189 A = (digest[0] += A);
190 B = (digest[1] += B);
191 C = (digest[2] += C);
192 D = (digest[3] += D);
193 E = (digest[4] += E);
194 F = (digest[5] += F);
195 G = (digest[6] += G);
196 H = (digest[7] += H);
197 }
198 }
199
200 while(blocks > 0) {
201 SIMD_2x64 WS[8];
202
203 for(size_t i = 0; i < 8; i++) {
204 WS[i] = SIMD_2x64::load_be(&data[16 * i]);
205 auto WK = WS[i] + SIMD_2x64::load_le(&K[2 * i]);
206 WK.store_le(&W[2 * i]);
207 }
208
209 data += 128;
210 blocks -= 1;
211
212 // First 64 rounds of SHA-512
213 for(size_t r = 0; r != 64; r += 16) {
214 auto w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 16]);
215 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
216 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
217 w.store_le(&W[0]);
218
219 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 18]);
220 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
221 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
222 w.store_le(&W[2]);
223
224 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 20]);
225 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
226 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
227 w.store_le(&W[4]);
228
229 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 22]);
230 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
231 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
232 w.store_le(&W[6]);
233
234 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 24]);
235 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
236 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
237 w.store_le(&W[8]);
238
239 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 26]);
240 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
241 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
242 w.store_le(&W[10]);
243
244 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 28]);
245 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
246 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
247 w.store_le(&W[12]);
248
249 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&K[r + 30]);
250 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
251 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
252 w.store_le(&W[14]);
253 }
254
255 // Final 16 rounds of SHA-512
256 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
257 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
258 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
259 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
260 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
261 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
262 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
263 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
264 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
265 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
266 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
267 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
268 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
269 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
270 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
271 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
272
273 A = (digest[0] += A);
274 B = (digest[1] += B);
275 C = (digest[2] += C);
276 D = (digest[3] += D);
277 E = (digest[4] += E);
278 F = (digest[5] += F);
279 G = (digest[6] += G);
280 H = (digest[7] += H);
281 }
282}
283
284} // namespace Botan
Botan::SIMD_2x64::load_be
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
Definition simd_2x64.h:82
Botan::SIMD_2x64::load_le
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
Definition simd_2x64.h:74
Botan::SIMD_8x64::load_be4
static BOTAN_FN_ISA_SIMD_8X64 SIMD_8x64 load_be4(const void *in0, const void *in1, const void *in2, const void *in3)
Definition simd_8x64.h:46
Botan::SIMD_8x64::broadcast_2x64
static BOTAN_FN_ISA_AVX512 SIMD_8x64 broadcast_2x64(const uint64_t *in)
Definition simd_8x64.h:58
BOTAN_FORCE_INLINE
#define BOTAN_FORCE_INLINE
Definition compiler.h:87
Botan::SHA2_64_F
BOTAN_FORCE_INLINE void SHA2_64_F(uint64_t A, uint64_t B, uint64_t C, uint64_t &D, uint64_t E, uint64_t F, uint64_t G, uint64_t &H, uint64_t &M1, uint64_t M2, uint64_t M3, uint64_t M4, uint64_t magic)
Definition sha2_64_f.h:19
Botan::rotr
BOTAN_FORCE_INLINE constexpr T rotr(T input)
Definition rotate.h:35
Generated by doxygen 1.15.0