7#include <botan/internal/sha2_64.h>
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_64_f.h>
11#include <botan/internal/simd_2x64.h>
12#include <botan/internal/simd_4x64.h>
18template <
typename SIMD_T>
20 const auto t0 = SIMD_T::alignr8(x[1], x[0]);
21 const auto t1 = SIMD_T::alignr8(x[5], x[4]);
23 const auto s0 = t0.template
rotr<1>() ^ t0.template
rotr<8>() ^ t0.template shr<7>();
24 const auto s1 = x[7].template
rotr<19>() ^ x[7].template
rotr<61>() ^ x[7].template shr<6>();
26 const auto nx = x[0] + s0 + s1 + t1;
42BOTAN_FN_ISA_AVX2_BMI2
void SHA_512::compress_digest_x86_avx2(digest_type& digest,
43 std::span<const uint8_t> input,
45 alignas(64) uint64_t W[16] = {0};
46 alignas(64) uint64_t W2[80];
48 uint64_t A = digest[0];
49 uint64_t B = digest[1];
50 uint64_t C = digest[2];
51 uint64_t D = digest[3];
52 uint64_t E = digest[4];
53 uint64_t F = digest[5];
54 uint64_t G = digest[6];
55 uint64_t H = digest[7];
57 const uint8_t* data = input.data();
62 for(
size_t i = 0; i < 8; i++) {
65 WK.store_le2(&W[2 * i], &W2[2 * i]);
72 for(
size_t r = 0; r != 64; r += 16) {
76 w.store_le2(&W[0], &W2[r + 16]);
81 w.store_le2(&W[2], &W2[r + 18]);
86 w.store_le2(&W[4], &W2[r + 20]);
91 w.store_le2(&W[6], &W2[r + 22]);
96 w.store_le2(&W[8], &W2[r + 24]);
100 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
101 w.store_le2(&W[10], &W2[r + 26]);
104 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
105 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
106 w.store_le2(&W[12], &W2[r + 28]);
109 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
110 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
111 w.store_le2(&W[14], &W2[r + 30]);
125 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
126 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
127 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
128 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
129 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
130 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
132 A = (digest[0] += A);
133 B = (digest[1] += B);
134 C = (digest[2] += C);
135 D = (digest[3] += D);
136 E = (digest[4] += E);
137 F = (digest[5] += F);
138 G = (digest[6] += G);
139 H = (digest[7] += H);
142 SHA2_64_F(A, B, C, D, E, F, G, H, W2[0]);
143 SHA2_64_F(H, A, B, C, D, E, F, G, W2[1]);
144 SHA2_64_F(G, H, A, B, C, D, E, F, W2[2]);
145 SHA2_64_F(F, G, H, A, B, C, D, E, W2[3]);
146 SHA2_64_F(E, F, G, H, A, B, C, D, W2[4]);
147 SHA2_64_F(D, E, F, G, H, A, B, C, W2[5]);
148 SHA2_64_F(C, D, E, F, G, H, A, B, W2[6]);
149 SHA2_64_F(B, C, D, E, F, G, H, A, W2[7]);
150 SHA2_64_F(A, B, C, D, E, F, G, H, W2[8]);
151 SHA2_64_F(H, A, B, C, D, E, F, G, W2[9]);
152 SHA2_64_F(G, H, A, B, C, D, E, F, W2[10]);
153 SHA2_64_F(F, G, H, A, B, C, D, E, W2[11]);
154 SHA2_64_F(E, F, G, H, A, B, C, D, W2[12]);
155 SHA2_64_F(D, E, F, G, H, A, B, C, W2[13]);
156 SHA2_64_F(C, D, E, F, G, H, A, B, W2[14]);
157 SHA2_64_F(B, C, D, E, F, G, H, A, W2[15]);
159 SHA2_64_F(A, B, C, D, E, F, G, H, W2[16]);
160 SHA2_64_F(H, A, B, C, D, E, F, G, W2[17]);
161 SHA2_64_F(G, H, A, B, C, D, E, F, W2[18]);
162 SHA2_64_F(F, G, H, A, B, C, D, E, W2[19]);
163 SHA2_64_F(E, F, G, H, A, B, C, D, W2[20]);
164 SHA2_64_F(D, E, F, G, H, A, B, C, W2[21]);
165 SHA2_64_F(C, D, E, F, G, H, A, B, W2[22]);
166 SHA2_64_F(B, C, D, E, F, G, H, A, W2[23]);
167 SHA2_64_F(A, B, C, D, E, F, G, H, W2[24]);
168 SHA2_64_F(H, A, B, C, D, E, F, G, W2[25]);
169 SHA2_64_F(G, H, A, B, C, D, E, F, W2[26]);
170 SHA2_64_F(F, G, H, A, B, C, D, E, W2[27]);
171 SHA2_64_F(E, F, G, H, A, B, C, D, W2[28]);
172 SHA2_64_F(D, E, F, G, H, A, B, C, W2[29]);
173 SHA2_64_F(C, D, E, F, G, H, A, B, W2[30]);
174 SHA2_64_F(B, C, D, E, F, G, H, A, W2[31]);
176 SHA2_64_F(A, B, C, D, E, F, G, H, W2[32]);
177 SHA2_64_F(H, A, B, C, D, E, F, G, W2[33]);
178 SHA2_64_F(G, H, A, B, C, D, E, F, W2[34]);
179 SHA2_64_F(F, G, H, A, B, C, D, E, W2[35]);
180 SHA2_64_F(E, F, G, H, A, B, C, D, W2[36]);
181 SHA2_64_F(D, E, F, G, H, A, B, C, W2[37]);
182 SHA2_64_F(C, D, E, F, G, H, A, B, W2[38]);
183 SHA2_64_F(B, C, D, E, F, G, H, A, W2[39]);
184 SHA2_64_F(A, B, C, D, E, F, G, H, W2[40]);
185 SHA2_64_F(H, A, B, C, D, E, F, G, W2[41]);
186 SHA2_64_F(G, H, A, B, C, D, E, F, W2[42]);
187 SHA2_64_F(F, G, H, A, B, C, D, E, W2[43]);
188 SHA2_64_F(E, F, G, H, A, B, C, D, W2[44]);
189 SHA2_64_F(D, E, F, G, H, A, B, C, W2[45]);
190 SHA2_64_F(C, D, E, F, G, H, A, B, W2[46]);
191 SHA2_64_F(B, C, D, E, F, G, H, A, W2[47]);
193 SHA2_64_F(A, B, C, D, E, F, G, H, W2[48]);
194 SHA2_64_F(H, A, B, C, D, E, F, G, W2[49]);
195 SHA2_64_F(G, H, A, B, C, D, E, F, W2[50]);
196 SHA2_64_F(F, G, H, A, B, C, D, E, W2[51]);
197 SHA2_64_F(E, F, G, H, A, B, C, D, W2[52]);
198 SHA2_64_F(D, E, F, G, H, A, B, C, W2[53]);
199 SHA2_64_F(C, D, E, F, G, H, A, B, W2[54]);
200 SHA2_64_F(B, C, D, E, F, G, H, A, W2[55]);
201 SHA2_64_F(A, B, C, D, E, F, G, H, W2[56]);
202 SHA2_64_F(H, A, B, C, D, E, F, G, W2[57]);
203 SHA2_64_F(G, H, A, B, C, D, E, F, W2[58]);
204 SHA2_64_F(F, G, H, A, B, C, D, E, W2[59]);
205 SHA2_64_F(E, F, G, H, A, B, C, D, W2[60]);
206 SHA2_64_F(D, E, F, G, H, A, B, C, W2[61]);
207 SHA2_64_F(C, D, E, F, G, H, A, B, W2[62]);
208 SHA2_64_F(B, C, D, E, F, G, H, A, W2[63]);
210 SHA2_64_F(A, B, C, D, E, F, G, H, W2[64]);
211 SHA2_64_F(H, A, B, C, D, E, F, G, W2[65]);
212 SHA2_64_F(G, H, A, B, C, D, E, F, W2[66]);
213 SHA2_64_F(F, G, H, A, B, C, D, E, W2[67]);
214 SHA2_64_F(E, F, G, H, A, B, C, D, W2[68]);
215 SHA2_64_F(D, E, F, G, H, A, B, C, W2[69]);
216 SHA2_64_F(C, D, E, F, G, H, A, B, W2[70]);
217 SHA2_64_F(B, C, D, E, F, G, H, A, W2[71]);
218 SHA2_64_F(A, B, C, D, E, F, G, H, W2[72]);
219 SHA2_64_F(H, A, B, C, D, E, F, G, W2[73]);
220 SHA2_64_F(G, H, A, B, C, D, E, F, W2[74]);
221 SHA2_64_F(F, G, H, A, B, C, D, E, W2[75]);
222 SHA2_64_F(E, F, G, H, A, B, C, D, W2[76]);
223 SHA2_64_F(D, E, F, G, H, A, B, C, W2[77]);
224 SHA2_64_F(C, D, E, F, G, H, A, B, W2[78]);
225 SHA2_64_F(B, C, D, E, F, G, H, A, W2[79]);
227 A = (digest[0] += A);
228 B = (digest[1] += B);
229 C = (digest[2] += C);
230 D = (digest[3] += D);
231 E = (digest[4] += E);
232 F = (digest[5] += F);
233 G = (digest[6] += G);
234 H = (digest[7] += H);
240 for(
size_t i = 0; i < 8; i++) {
243 WK.store_le(&W[2 * i]);
250 for(
size_t r = 0; r != 64; r += 16) {
277 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
278 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
282 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
283 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
287 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
288 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
303 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
304 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
305 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
306 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
307 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
308 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
310 A = (digest[0] += A);
311 B = (digest[1] += B);
312 C = (digest[2] += C);
313 D = (digest[3] += D);
314 E = (digest[4] += E);
315 F = (digest[5] += F);
316 G = (digest[6] += G);
317 H = (digest[7] += H);
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 broadcast_2x64(const uint64_t *in)
static BOTAN_FN_ISA_SIMD_4X64 SIMD_4x64 load_be2(const void *lo, const void *hi)
#define BOTAN_FORCE_INLINE
constexpr uint64_t SHA512_K[80]
BOTAN_FORCE_INLINE void SHA2_64_F(uint64_t A, uint64_t B, uint64_t C, uint64_t &D, uint64_t E, uint64_t F, uint64_t G, uint64_t &H, uint64_t &M1, uint64_t M2, uint64_t M3, uint64_t M4, uint64_t magic)
BOTAN_FORCE_INLINE constexpr T rotr(T input)