7#include <botan/internal/sha2_32.h>
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_32_f.h>
11#include <botan/internal/simd_4x32.h>
12#include <botan/internal/simd_8x32.h>
13#include <botan/internal/stack_scrubbing.h>
22 return SIMD_4x32(_mm_alignr_epi8(a.raw(), b.raw(), 4));
27 return SIMD_4x32(_mm_srli_epi64(a.raw(), S));
32 return SIMD_4x32(_mm_shuffle_epi32(a.raw(), S));
36 return SIMD_8x32(_mm256_alignr_epi8(a.raw(), b.raw(), 4));
41 return SIMD_8x32(_mm256_srli_epi64(a.raw(), S));
46 return SIMD_8x32(_mm256_shuffle_epi32(a.raw(), S));
49template <
typename SIMD_T>
51 constexpr size_t sigma0_0 = 7;
52 constexpr size_t sigma0_1 = 18;
53 constexpr size_t sigma0_2 = 3;
54 constexpr size_t sigma1_0 = 17;
55 constexpr size_t sigma1_1 = 19;
56 constexpr size_t sigma1_2 = 10;
58 const SIMD_T lo_mask = SIMD_T(0x03020100, 0x0b0a0908, 0x80808080, 0x80808080);
59 const SIMD_T hi_mask = SIMD_T(0x80808080, 0x80808080, 0x03020100, 0x0b0a0908);
61 auto t0 = alignr4(x[1], x[0]);
62 x[0] += alignr4(x[3], x[2]);
64 auto t1 = t0.template
shl<32 - sigma0_1>();
65 auto t2 = t0.template shr<sigma0_0>();
66 auto t3 = t0.template shr<sigma0_2>();
69 t3 = shuffle_32<0b11111010>(x[3]);
70 t2 = t2.template shr<sigma0_1 - sigma0_0>();
72 t1 = t1.template
shl<sigma0_1 - sigma0_0>();
73 t2 = t3.template shr<sigma1_2>();
74 t3 = shr64<sigma1_0>(t3);
78 t3 = shr64<sigma1_1 - sigma1_0>(t3);
79 x[0] += SIMD_T::byte_shuffle(t2 ^ t3, lo_mask);
81 t3 = shuffle_32<0b01010000>(x[0]);
82 t2 = t3.template shr<sigma1_2>();
83 t3 = shr64<sigma1_0>(t3);
85 t3 = shr64<sigma1_1 - sigma1_0>(t3);
86 x[0] += SIMD_T::byte_shuffle(t2 ^ t3, hi_mask);
88 const auto tmp = x[0];
100 digest_type& digest, std::span<const uint8_t> input,
size_t blocks) {
101 alignas(64) uint32_t W[16];
102 alignas(64) uint32_t W2[64];
104 uint32_t A = digest[0];
105 uint32_t B = digest[1];
106 uint32_t C = digest[2];
107 uint32_t D = digest[3];
108 uint32_t E = digest[4];
109 uint32_t F = digest[5];
110 uint32_t G = digest[6];
111 uint32_t H = digest[7];
113 const uint8_t* data = input.data();
118 for(
size_t i = 0; i < 4; i++) {
121 WK.store_le128(&W[4 * i], &W2[4 * i]);
127 for(
size_t r = 0; r != 48; r += 16) {
135 w.store_le128(&W[0], &W2[r + 16]);
144 w.store_le128(&W[4], &W2[r + 20]);
150 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
151 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
153 w.store_le128(&W[8], &W2[r + 24]);
157 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
158 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
159 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
160 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
162 w.store_le128(&W[12], &W2[r + 28]);
175 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
176 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
177 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
178 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
179 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
180 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
182 A = (digest[0] += A);
183 B = (digest[1] += B);
184 C = (digest[2] += C);
185 D = (digest[3] += D);
186 E = (digest[4] += E);
187 F = (digest[5] += F);
188 G = (digest[6] += G);
189 H = (digest[7] += H);
192 SHA2_32_F(A, B, C, D, E, F, G, H, W2[0]);
193 SHA2_32_F(H, A, B, C, D, E, F, G, W2[1]);
194 SHA2_32_F(G, H, A, B, C, D, E, F, W2[2]);
195 SHA2_32_F(F, G, H, A, B, C, D, E, W2[3]);
196 SHA2_32_F(E, F, G, H, A, B, C, D, W2[4]);
197 SHA2_32_F(D, E, F, G, H, A, B, C, W2[5]);
198 SHA2_32_F(C, D, E, F, G, H, A, B, W2[6]);
199 SHA2_32_F(B, C, D, E, F, G, H, A, W2[7]);
200 SHA2_32_F(A, B, C, D, E, F, G, H, W2[8]);
201 SHA2_32_F(H, A, B, C, D, E, F, G, W2[9]);
202 SHA2_32_F(G, H, A, B, C, D, E, F, W2[10]);
203 SHA2_32_F(F, G, H, A, B, C, D, E, W2[11]);
204 SHA2_32_F(E, F, G, H, A, B, C, D, W2[12]);
205 SHA2_32_F(D, E, F, G, H, A, B, C, W2[13]);
206 SHA2_32_F(C, D, E, F, G, H, A, B, W2[14]);
207 SHA2_32_F(B, C, D, E, F, G, H, A, W2[15]);
209 SHA2_32_F(A, B, C, D, E, F, G, H, W2[16]);
210 SHA2_32_F(H, A, B, C, D, E, F, G, W2[17]);
211 SHA2_32_F(G, H, A, B, C, D, E, F, W2[18]);
212 SHA2_32_F(F, G, H, A, B, C, D, E, W2[19]);
213 SHA2_32_F(E, F, G, H, A, B, C, D, W2[20]);
214 SHA2_32_F(D, E, F, G, H, A, B, C, W2[21]);
215 SHA2_32_F(C, D, E, F, G, H, A, B, W2[22]);
216 SHA2_32_F(B, C, D, E, F, G, H, A, W2[23]);
217 SHA2_32_F(A, B, C, D, E, F, G, H, W2[24]);
218 SHA2_32_F(H, A, B, C, D, E, F, G, W2[25]);
219 SHA2_32_F(G, H, A, B, C, D, E, F, W2[26]);
220 SHA2_32_F(F, G, H, A, B, C, D, E, W2[27]);
221 SHA2_32_F(E, F, G, H, A, B, C, D, W2[28]);
222 SHA2_32_F(D, E, F, G, H, A, B, C, W2[29]);
223 SHA2_32_F(C, D, E, F, G, H, A, B, W2[30]);
224 SHA2_32_F(B, C, D, E, F, G, H, A, W2[31]);
226 SHA2_32_F(A, B, C, D, E, F, G, H, W2[32]);
227 SHA2_32_F(H, A, B, C, D, E, F, G, W2[33]);
228 SHA2_32_F(G, H, A, B, C, D, E, F, W2[34]);
229 SHA2_32_F(F, G, H, A, B, C, D, E, W2[35]);
230 SHA2_32_F(E, F, G, H, A, B, C, D, W2[36]);
231 SHA2_32_F(D, E, F, G, H, A, B, C, W2[37]);
232 SHA2_32_F(C, D, E, F, G, H, A, B, W2[38]);
233 SHA2_32_F(B, C, D, E, F, G, H, A, W2[39]);
234 SHA2_32_F(A, B, C, D, E, F, G, H, W2[40]);
235 SHA2_32_F(H, A, B, C, D, E, F, G, W2[41]);
236 SHA2_32_F(G, H, A, B, C, D, E, F, W2[42]);
237 SHA2_32_F(F, G, H, A, B, C, D, E, W2[43]);
238 SHA2_32_F(E, F, G, H, A, B, C, D, W2[44]);
239 SHA2_32_F(D, E, F, G, H, A, B, C, W2[45]);
240 SHA2_32_F(C, D, E, F, G, H, A, B, W2[46]);
241 SHA2_32_F(B, C, D, E, F, G, H, A, W2[47]);
243 SHA2_32_F(A, B, C, D, E, F, G, H, W2[48]);
244 SHA2_32_F(H, A, B, C, D, E, F, G, W2[49]);
245 SHA2_32_F(G, H, A, B, C, D, E, F, W2[50]);
246 SHA2_32_F(F, G, H, A, B, C, D, E, W2[51]);
247 SHA2_32_F(E, F, G, H, A, B, C, D, W2[52]);
248 SHA2_32_F(D, E, F, G, H, A, B, C, W2[53]);
249 SHA2_32_F(C, D, E, F, G, H, A, B, W2[54]);
250 SHA2_32_F(B, C, D, E, F, G, H, A, W2[55]);
251 SHA2_32_F(A, B, C, D, E, F, G, H, W2[56]);
252 SHA2_32_F(H, A, B, C, D, E, F, G, W2[57]);
253 SHA2_32_F(G, H, A, B, C, D, E, F, W2[58]);
254 SHA2_32_F(F, G, H, A, B, C, D, E, W2[59]);
255 SHA2_32_F(E, F, G, H, A, B, C, D, W2[60]);
256 SHA2_32_F(D, E, F, G, H, A, B, C, W2[61]);
257 SHA2_32_F(C, D, E, F, G, H, A, B, W2[62]);
258 SHA2_32_F(B, C, D, E, F, G, H, A, W2[63]);
260 A = (digest[0] += A);
261 B = (digest[1] += B);
262 C = (digest[2] += C);
263 D = (digest[3] += D);
264 E = (digest[4] += E);
265 F = (digest[5] += F);
266 G = (digest[6] += G);
267 H = (digest[7] += H);
273 for(
size_t i = 0; i < 4; i++) {
276 WK.store_le(&W[4 * i]);
282 for(
size_t r = 0; r != 48; r += 16) {
305 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
306 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
312 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
313 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
314 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
315 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
330 SHA2_32_F(G, H, A, B, C, D, E, F, W[10]);
331 SHA2_32_F(F, G, H, A, B, C, D, E, W[11]);
332 SHA2_32_F(E, F, G, H, A, B, C, D, W[12]);
333 SHA2_32_F(D, E, F, G, H, A, B, C, W[13]);
334 SHA2_32_F(C, D, E, F, G, H, A, B, W[14]);
335 SHA2_32_F(B, C, D, E, F, G, H, A, W[15]);
337 A = (digest[0] += A);
338 B = (digest[1] += B);
339 C = (digest[2] += C);
340 D = (digest[3] += D);
341 E = (digest[4] += E);
342 F = (digest[5] += F);
343 G = (digest[6] += G);
344 H = (digest[7] += H);
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_be(const void *in) noexcept
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_le128(const uint8_t *in) noexcept
static BOTAN_FN_ISA_SIMD_8X32 SIMD_8x32 load_be128(const uint8_t in1[], const uint8_t in2[]) noexcept
#define BOTAN_FORCE_INLINE
BOTAN_FORCE_INLINE void SHA2_32_F(uint32_t A, uint32_t B, uint32_t C, uint32_t &D, uint32_t E, uint32_t F, uint32_t G, uint32_t &H, uint32_t &M1, uint32_t M2, uint32_t M3, uint32_t M4, uint32_t magic)
SIMD_4x32 shl(SIMD_4x32 input)
constexpr uint32_t SHA256_K[64]
#define BOTAN_SCRUB_STACK_AFTER_RETURN