Botan 3.14.1
Crypto and TLS for C&
hash_engine_sha2_32_avx512.cpp
Go to the documentation of this file.
1/*
2* (C) 2026 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/hash_engine_sha2_32.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_32_f.h>
11#include <botan/internal/simd_avx512.h>
12
13namespace Botan {
14
15namespace {
16
17BOTAN_FN_ISA_AVX512 BOTAN_FORCE_INLINE void rnd(const SIMD_16x32& A,
18 const SIMD_16x32& B,
19 const SIMD_16x32& C,
20 SIMD_16x32& D,
21 const SIMD_16x32& E,
22 const SIMD_16x32& F,
23 const SIMD_16x32& G,
24 SIMD_16x32& H,
25 const SIMD_16x32& W,
26 uint32_t K) {
27 const auto T1 = H + E.sigma1() + SIMD_16x32::choose(E, F, G) + SIMD_16x32::splat(K) + W;
28 const auto T2 = A.sigma0() + SIMD_16x32::majority(A, B, C);
29 D += T1;
30 H = T1 + T2;
31}
32
33template <size_t I>
34BOTAN_FN_ISA_AVX512 BOTAN_FORCE_INLINE SIMD_16x32 next_w(SIMD_16x32 W[16]) {
35 const auto& w15 = W[(I + 1) % 16];
36 const auto& w2 = W[(I + 14) % 16];
37 const auto s0 = w15.rotr<7>() ^ w15.rotr<18>() ^ w15.shr<3>();
38 const auto s1 = w2.rotr<17>() ^ w2.rotr<19>() ^ w2.shr<10>();
39 W[I] += s0 + s1 + W[(I + 9) % 16];
40 return W[I];
41}
42
43/**
44* Store each 128-bit quarter of v separately, quarter q to out + q * stride
45*/
46BOTAN_FN_ISA_AVX512 BOTAN_FORCE_INLINE void store_quarters(const SIMD_16x32& v, uint8_t* out, size_t stride) {
47 _mm_storeu_si128(reinterpret_cast<__m128i*>(out), _mm512_castsi512_si128(v.raw()));
48 _mm_storeu_si128(reinterpret_cast<__m128i*>(out + stride), _mm512_extracti32x4_epi32(v.raw(), 1));
49 _mm_storeu_si128(reinterpret_cast<__m128i*>(out + 2 * stride), _mm512_extracti32x4_epi32(v.raw(), 2));
50 _mm_storeu_si128(reinterpret_cast<__m128i*>(out + 3 * stride), _mm512_extracti32x4_epi32(v.raw(), 3));
51}
52
53} // namespace
54
55BOTAN_FN_ISA_AVX512 void sha2_32_mb_compress_x16(uint8_t* states, const uint8_t* const* blocks, size_t nblocks) {
56 SIMD_16x32 S[8];
57 for(size_t j = 0; j != 8; ++j) {
58 S[j] = SIMD_16x32::load_le(states + 64 * j);
59 }
60
61 for(size_t n = 0; n != nblocks; ++n) {
62 // Initialized directly from the loads, since default construction
63 // would zero the whole array first
64 SIMD_16x32 W[16] = {SIMD_16x32::load_be<64>(blocks, 0, n),
65 SIMD_16x32::load_be<64>(blocks, 1, n),
66 SIMD_16x32::load_be<64>(blocks, 2, n),
67 SIMD_16x32::load_be<64>(blocks, 3, n),
68 SIMD_16x32::load_be<64>(blocks, 4, n),
69 SIMD_16x32::load_be<64>(blocks, 5, n),
70 SIMD_16x32::load_be<64>(blocks, 6, n),
71 SIMD_16x32::load_be<64>(blocks, 7, n),
72 SIMD_16x32::load_be<64>(blocks, 8, n),
73 SIMD_16x32::load_be<64>(blocks, 9, n),
74 SIMD_16x32::load_be<64>(blocks, 10, n),
75 SIMD_16x32::load_be<64>(blocks, 11, n),
76 SIMD_16x32::load_be<64>(blocks, 12, n),
77 SIMD_16x32::load_be<64>(blocks, 13, n),
78 SIMD_16x32::load_be<64>(blocks, 14, n),
79 SIMD_16x32::load_be<64>(blocks, 15, n)};
81 W[0], W[1], W[2], W[3], W[4], W[5], W[6], W[7], W[8], W[9], W[10], W[11], W[12], W[13], W[14], W[15]);
82
83 SIMD_16x32 A = S[0];
84 SIMD_16x32 B = S[1];
85 SIMD_16x32 C = S[2];
86 SIMD_16x32 D = S[3];
87 SIMD_16x32 E = S[4];
88 SIMD_16x32 F = S[5];
89 SIMD_16x32 G = S[6];
90 SIMD_16x32 H = S[7];
91
92 // Every index into W is a constant so that it can stay in registers
93 rnd(A, B, C, D, E, F, G, H, W[0], SHA256_K[0]);
94 rnd(H, A, B, C, D, E, F, G, W[1], SHA256_K[1]);
95 rnd(G, H, A, B, C, D, E, F, W[2], SHA256_K[2]);
96 rnd(F, G, H, A, B, C, D, E, W[3], SHA256_K[3]);
97 rnd(E, F, G, H, A, B, C, D, W[4], SHA256_K[4]);
98 rnd(D, E, F, G, H, A, B, C, W[5], SHA256_K[5]);
99 rnd(C, D, E, F, G, H, A, B, W[6], SHA256_K[6]);
100 rnd(B, C, D, E, F, G, H, A, W[7], SHA256_K[7]);
101 rnd(A, B, C, D, E, F, G, H, W[8], SHA256_K[8]);
102 rnd(H, A, B, C, D, E, F, G, W[9], SHA256_K[9]);
103 rnd(G, H, A, B, C, D, E, F, W[10], SHA256_K[10]);
104 rnd(F, G, H, A, B, C, D, E, W[11], SHA256_K[11]);
105 rnd(E, F, G, H, A, B, C, D, W[12], SHA256_K[12]);
106 rnd(D, E, F, G, H, A, B, C, W[13], SHA256_K[13]);
107 rnd(C, D, E, F, G, H, A, B, W[14], SHA256_K[14]);
108 rnd(B, C, D, E, F, G, H, A, W[15], SHA256_K[15]);
109
110 for(size_t t = 16; t != 64; t += 16) {
111 rnd(A, B, C, D, E, F, G, H, next_w<0>(W), SHA256_K[t + 0]);
112 rnd(H, A, B, C, D, E, F, G, next_w<1>(W), SHA256_K[t + 1]);
113 rnd(G, H, A, B, C, D, E, F, next_w<2>(W), SHA256_K[t + 2]);
114 rnd(F, G, H, A, B, C, D, E, next_w<3>(W), SHA256_K[t + 3]);
115 rnd(E, F, G, H, A, B, C, D, next_w<4>(W), SHA256_K[t + 4]);
116 rnd(D, E, F, G, H, A, B, C, next_w<5>(W), SHA256_K[t + 5]);
117 rnd(C, D, E, F, G, H, A, B, next_w<6>(W), SHA256_K[t + 6]);
118 rnd(B, C, D, E, F, G, H, A, next_w<7>(W), SHA256_K[t + 7]);
119 rnd(A, B, C, D, E, F, G, H, next_w<8>(W), SHA256_K[t + 8]);
120 rnd(H, A, B, C, D, E, F, G, next_w<9>(W), SHA256_K[t + 9]);
121 rnd(G, H, A, B, C, D, E, F, next_w<10>(W), SHA256_K[t + 10]);
122 rnd(F, G, H, A, B, C, D, E, next_w<11>(W), SHA256_K[t + 11]);
123 rnd(E, F, G, H, A, B, C, D, next_w<12>(W), SHA256_K[t + 12]);
124 rnd(D, E, F, G, H, A, B, C, next_w<13>(W), SHA256_K[t + 13]);
125 rnd(C, D, E, F, G, H, A, B, next_w<14>(W), SHA256_K[t + 14]);
126 rnd(B, C, D, E, F, G, H, A, next_w<15>(W), SHA256_K[t + 15]);
127 }
128
129 S[0] += A;
130 S[1] += B;
131 S[2] += C;
132 S[3] += D;
133 S[4] += E;
134 S[5] += F;
135 S[6] += G;
136 S[7] += H;
137 }
138
139 for(size_t j = 0; j != 8; ++j) {
140 S[j].store_le(states + 64 * j);
141 }
142}
143
144BOTAN_FN_ISA_AVX512 void sha2_32_mb_extract_x16(const uint8_t* states, uint8_t* digests) {
145 SIMD_16x32 S[8];
146 for(size_t j = 0; j != 8; ++j) {
147 S[j] = SIMD_16x32::load_le(states + 64 * j).bswap();
148 }
149
150 // Each S[j] holds word j of all 16 lanes. The 4-way transpose works
151 // within each 128-bit quarter, so afterwards quarter q of S[k] holds
152 // words 0..3 of lane 4*q+k, and quarter q of S[4+k] its words 4..7.
153 SIMD_16x32::transpose(S[0], S[1], S[2], S[3]);
154 SIMD_16x32::transpose(S[4], S[5], S[6], S[7]);
155
156 for(size_t k = 0; k != 4; ++k) {
157 store_quarters(S[k], digests + 32 * k, 128);
158 store_quarters(S[4 + k], digests + 32 * k + 16, 128);
159 }
160}
161
162} // namespace Botan
static BOTAN_FN_ISA_AVX512 void transpose(SIMD_16x32 &B0, SIMD_16x32 &B1, SIMD_16x32 &B2, SIMD_16x32 &B3)
BOTAN_FN_ISA_AVX512 void store_le(uint8_t out[]) const
Definition simd_avx512.h:80
static BOTAN_FN_ISA_AVX512 SIMD_16x32 choose(const SIMD_16x32 &mask, const SIMD_16x32 &a, const SIMD_16x32 &b)
SIMD_16x32 BOTAN_FN_ISA_AVX512 sigma0() const
Definition simd_avx512.h:97
BOTAN_FN_ISA_AVX512 SIMD_16x32 bswap() const
BOTAN_FN_ISA_AVX512 SIMD_16x32 shr() const
static BOTAN_FN_ISA_AVX512 SIMD_16x32 majority(const SIMD_16x32 &x, const SIMD_16x32 &y, const SIMD_16x32 &z)
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_le(const uint8_t *in)
Definition simd_avx512.h:63
BOTAN_FN_ISA_AVX512 SIMD_16x32 rotr() const
Definition simd_avx512.h:93
static BOTAN_FN_ISA_AVX512 SIMD_16x32 splat(uint32_t B)
Definition simd_avx512.h:60
static BOTAN_FN_ISA_AVX512 SIMD_16x32 load_be(const uint8_t *in)
Definition simd_avx512.h:68
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
BOTAN_FN_ISA_AVX512 void sha2_32_mb_extract_x16(const uint8_t *states, uint8_t *digests)
BOTAN_FN_ISA_AVX512 void sha2_32_mb_compress_x16(uint8_t *states, const uint8_t *const *blocks, size_t nblocks)
constexpr uint32_t SHA256_K[64]
Definition sha2_32_f.h:19