Botan 3.14.1
Crypto and TLS for C&
sha2_64_avx512.cpp
Go to the documentation of this file.
1/*
2* (C) 2025,2026 Jack Lloyd
3*
4* Botan is released under the Simplified BSD License (see license.txt)
5*/
6
7#include <botan/internal/sha2_64.h>
8
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/sha2_64_f.h>
11#include <botan/internal/simd_2x64.h>
12#include <botan/internal/simd_8x64.h>
13
14namespace Botan {
15
16namespace SHA512_AVX512 {
17
18namespace {
19
20template <size_t R1, size_t R2, size_t S1>
21BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX512_BMI2 SIMD_2x64 avx512_sigma(SIMD_2x64 v) {
22 const auto vr1 = _mm_ror_epi64(v.raw(), R1);
23 const auto vr2 = _mm_ror_epi64(v.raw(), R2);
24 const auto vs1 = _mm_srli_epi64(v.raw(), S1);
25 return SIMD_2x64(_mm_ternarylogic_epi64(vr1, vr2, vs1, 0x96));
26}
27
28BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX512_BMI2 SIMD_2x64 sha512_next_w_avx512(SIMD_2x64 x[8]) {
29 const auto t0 = SIMD_2x64::alignr8(x[1], x[0]);
30 const auto t1 = SIMD_2x64::alignr8(x[5], x[4]);
31
32 const auto s0 = avx512_sigma<1, 8, 7>(t0);
33 const auto s1 = avx512_sigma<19, 61, 6>(x[7]);
34
35 auto nx = x[0] + s0 + s1 + t1;
36
37 x[0] = x[1];
38 x[1] = x[2];
39 x[2] = x[3];
40 x[3] = x[4];
41 x[4] = x[5];
42 x[5] = x[6];
43 x[6] = x[7];
44 x[7] = nx;
45
46 return nx;
47}
48
49template <size_t R1, size_t R2, size_t R3>
50BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX512_BMI2 SIMD_2x64 rho(SIMD_2x64 v) {
51 const auto vr1 = _mm_ror_epi64(v.raw(), R1);
52 const auto vr2 = _mm_ror_epi64(v.raw(), R2);
53 const auto vr3 = _mm_ror_epi64(v.raw(), R3);
54 return SIMD_2x64(_mm_ternarylogic_epi64(vr1, vr2, vr3, 0x96));
55}
56
57BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX512_BMI2 void SHA2_64_F(SIMD_2x64 A,
58 SIMD_2x64 B,
59 SIMD_2x64 C,
60 SIMD_2x64& D,
61 SIMD_2x64 E,
62 SIMD_2x64 F,
63 SIMD_2x64 G,
64 SIMD_2x64& H,
65 uint64_t M) {
66 constexpr uint8_t ch = 0xca;
67 constexpr uint8_t maj = 0xe8;
68
69 H += rho<14, 18, 41>(E) + SIMD_2x64(_mm_ternarylogic_epi64(E.raw(), F.raw(), G.raw(), ch)) + SIMD_2x64::splat(M);
70 D += H;
71 H += rho<28, 34, 39>(A) + SIMD_2x64(_mm_ternarylogic_epi64(A.raw(), B.raw(), C.raw(), maj));
72}
73
74} // namespace
75
76} // namespace SHA512_AVX512
77
78BOTAN_FN_ISA_AVX512_BMI2 void SHA_512::compress_digest_x86_avx512(digest_type& digest,
79 std::span<const uint8_t> input,
80 size_t blocks) {
81 using namespace SHA512_AVX512;
82
83 alignas(64) uint64_t W[16] = {0};
84
85 auto digest0 = SIMD_2x64::splat(digest[0]);
86 auto digest1 = SIMD_2x64::splat(digest[1]);
87 auto digest2 = SIMD_2x64::splat(digest[2]);
88 auto digest3 = SIMD_2x64::splat(digest[3]);
89 auto digest4 = SIMD_2x64::splat(digest[4]);
90 auto digest5 = SIMD_2x64::splat(digest[5]);
91 auto digest6 = SIMD_2x64::splat(digest[6]);
92 auto digest7 = SIMD_2x64::splat(digest[7]);
93
94 auto A = digest0;
95 auto B = digest1;
96 auto C = digest2;
97 auto D = digest3;
98 auto E = digest4;
99 auto F = digest5;
100 auto G = digest6;
101 auto H = digest7;
102
103 const uint8_t* data = input.data();
104
105 while(blocks > 0) {
106 SIMD_2x64 WS[8];
107
108 for(size_t i = 0; i < 8; i++) {
109 WS[i] = SIMD_2x64::load_be(&data[16 * i]);
110 const auto WK = WS[i] + SIMD_2x64::load_le(&SHA512_K[2 * i]);
111 WK.store_le(&W[2 * i]);
112 }
113
114 data += 128;
115 blocks -= 1;
116
117 // First 64 rounds of SHA-512
118 for(size_t r = 0; r != 64; r += 16) {
119 auto w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 16]);
120 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
121 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
122 w.store_le(&W[0]);
123
124 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 18]);
125 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
126 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
127 w.store_le(&W[2]);
128
129 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 20]);
130 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
131 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
132 w.store_le(&W[4]);
133
134 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 22]);
135 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
136 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
137 w.store_le(&W[6]);
138
139 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 24]);
140 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
141 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
142 w.store_le(&W[8]);
143
144 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 26]);
145 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
146 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
147 w.store_le(&W[10]);
148
149 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 28]);
150 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
151 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
152 w.store_le(&W[12]);
153
154 w = sha512_next_w_avx512(WS) + SIMD_2x64::load_le(&SHA512_K[r + 30]);
155 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
156 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
157 w.store_le(&W[14]);
158 }
159
160 // Final 16 rounds of SHA-512
161 SHA2_64_F(A, B, C, D, E, F, G, H, W[0]);
162 SHA2_64_F(H, A, B, C, D, E, F, G, W[1]);
163 SHA2_64_F(G, H, A, B, C, D, E, F, W[2]);
164 SHA2_64_F(F, G, H, A, B, C, D, E, W[3]);
165 SHA2_64_F(E, F, G, H, A, B, C, D, W[4]);
166 SHA2_64_F(D, E, F, G, H, A, B, C, W[5]);
167 SHA2_64_F(C, D, E, F, G, H, A, B, W[6]);
168 SHA2_64_F(B, C, D, E, F, G, H, A, W[7]);
169 SHA2_64_F(A, B, C, D, E, F, G, H, W[8]);
170 SHA2_64_F(H, A, B, C, D, E, F, G, W[9]);
171 SHA2_64_F(G, H, A, B, C, D, E, F, W[10]);
172 SHA2_64_F(F, G, H, A, B, C, D, E, W[11]);
173 SHA2_64_F(E, F, G, H, A, B, C, D, W[12]);
174 SHA2_64_F(D, E, F, G, H, A, B, C, W[13]);
175 SHA2_64_F(C, D, E, F, G, H, A, B, W[14]);
176 SHA2_64_F(B, C, D, E, F, G, H, A, W[15]);
177
178 digest0 += A;
179 digest1 += B;
180 digest2 += C;
181 digest3 += D;
182 digest4 += E;
183 digest5 += F;
184 digest6 += G;
185 digest7 += H;
186
187 A = digest0;
188 B = digest1;
189 C = digest2;
190 D = digest3;
191 E = digest4;
192 F = digest5;
193 G = digest6;
194 H = digest7;
195 }
196
197 // Could be optimized a bit by interleaving the registers, reducing store pressure
198 // but probably not worth bothering with
199 _mm_mask_storeu_epi64(&digest[0], 0b01, digest0.raw()); // NOLINT(*-container-data-pointer)
200 _mm_mask_storeu_epi64(&digest[1], 0b01, digest1.raw());
201 _mm_mask_storeu_epi64(&digest[2], 0b01, digest2.raw());
202 _mm_mask_storeu_epi64(&digest[3], 0b01, digest3.raw());
203 _mm_mask_storeu_epi64(&digest[4], 0b01, digest4.raw());
204 _mm_mask_storeu_epi64(&digest[5], 0b01, digest5.raw());
205 _mm_mask_storeu_epi64(&digest[6], 0b01, digest6.raw());
206 _mm_mask_storeu_epi64(&digest[7], 0b01, digest7.raw());
207}
208
209} // namespace Botan
native_simd_type BOTAN_FN_ISA_SIMD_2X64 raw() const noexcept
Definition simd_2x64.h:505
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_be(const void *in)
Definition simd_2x64.h:132
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 load_le(const void *in)
Definition simd_2x64.h:124
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 splat(uint64_t v)
Definition simd_2x64.h:72
static SIMD_2x64 BOTAN_FN_ISA_SIMD_2X64 alignr8(const SIMD_2x64 &a, const SIMD_2x64 &b)
Definition simd_2x64.h:401
#define BOTAN_FORCE_INLINE
Definition compiler.h:86
constexpr uint64_t SHA512_K[80]
Definition sha2_64_f.h:19
BOTAN_FORCE_INLINE void SHA2_64_F(uint64_t A, uint64_t B, uint64_t C, uint64_t &D, uint64_t E, uint64_t F, uint64_t G, uint64_t &H, uint64_t &M1, uint64_t M2, uint64_t M3, uint64_t M4, uint64_t magic)
Definition sha2_64_f.h:41
void R2(uint32_t A, uint32_t &B, uint32_t C, uint32_t &D, uint32_t E, uint32_t &F, uint32_t G, uint32_t &H, uint32_t TJ, uint32_t Wi, uint32_t Wj)
Definition sm3_fn.h:43
BOTAN_FORCE_INLINE constexpr T rho(T x)
Definition rotate.h:53
void R1(uint32_t A, uint32_t &B, uint32_t C, uint32_t &D, uint32_t E, uint32_t &F, uint32_t G, uint32_t &H, uint32_t TJ, uint32_t Wi, uint32_t Wj)
Definition sm3_fn.h:21