7#include <botan/internal/ghash.h>
9#include <botan/internal/isa_extn.h>
10#include <botan/internal/polyval_fn.h>
11#include <botan/internal/simd_4x32.h>
14#if defined(BOTAN_HAS_POLYVAL)
15 #include <botan/internal/polyval.h>
24 const auto B = _mm512_loadu_si512(in);
28 const auto swap = _mm512_set_epi64(0x0001020304050607,
36 return _mm512_shuffle_epi8(B, swap);
43 return _mm512_xor_si512(H, _mm512_bsrli_epi128(H, 8));
47 const auto y = _mm256_xor_si256(_mm512_castsi512_si256(z), _mm512_extracti64x4_epi64(z, 0x1));
48 const auto x = _mm_xor_si128(_mm256_castsi256_si128(y), _mm256_extracti32x4_epi32(y, 0x1));
53ghash_x4_accum(__m512i H, __m512i H_fold, __m512i M, __m512i& lo, __m512i& hi, __m512i& mid) {
54 lo = _mm512_xor_si512(lo, _mm512_clmulepi64_epi128(H, M, 0x00));
55 hi = _mm512_xor_si512(hi, _mm512_clmulepi64_epi128(H, M, 0x11));
56 mid = _mm512_xor_si512(mid, _mm512_clmulepi64_epi128(H_fold, fold(M), 0x00));
60 mid = _mm512_ternarylogic_epi64(lo, mid, hi, 0x96);
61 hi = _mm512_xor_si512(hi, _mm512_bsrli_epi128(mid, 8));
62 lo = _mm512_xor_si512(lo, _mm512_bslli_epi128(mid, 8));
67 return _mm512_xor_epi64(M, _mm512_inserti64x2(_mm512_setzero_si512(), a.raw(), 0));
70void BOTAN_FN_ISA_AVX512_CLMUL precompute_avx512(
const SIMD_4x32& H1, uint64_t H_pow[16 * 2]) {
93 H3.store_le(H_pow + 2);
94 H2.store_le(H_pow + 4);
95 H1.store_le(H_pow + 6);
97 H8.store_le(H_pow + 8);
98 H7.store_le(H_pow + 10);
99 H6.store_le(H_pow + 12);
100 H5.store_le(H_pow + 14);
102 H12.store_le(H_pow + 16);
103 H11.store_le(H_pow + 18);
104 H10.store_le(H_pow + 20);
105 H9.store_le(H_pow + 22);
107 H16.store_le(H_pow + 24);
108 H15.store_le(H_pow + 26);
109 H14.store_le(H_pow + 28);
110 H13.store_le(H_pow + 30);
114void BOTAN_FN_ISA_AVX512_CLMUL
115multiply_avx512(uint8_t x[16],
const uint64_t H_pow[16 * 2],
const uint8_t input[],
size_t blocks) {
119 const auto H1 = _mm512_loadu_si512(H_pow);
120 const auto H2 = _mm512_loadu_si512(H_pow + 8);
121 const auto H3 = _mm512_loadu_si512(H_pow + 16);
122 const auto H4 = _mm512_loadu_si512(H_pow + 24);
125 const auto H1_fold = fold(H1);
126 const auto H2_fold = fold(H2);
127 const auto H3_fold = fold(H3);
128 const auto H4_fold = fold(H4);
130 while(blocks >= 16) {
131 __m512i M1 = load_blocks<BSWAP>(input);
132 const auto M2 = load_blocks<BSWAP>(input + 64);
133 const auto M3 = load_blocks<BSWAP>(input + 128);
134 const auto M4 = load_blocks<BSWAP>(input + 192);
136 M1 = insert_a(M1, a);
138 auto lo = _mm512_setzero_si512();
139 auto hi = _mm512_setzero_si512();
140 auto mid = _mm512_setzero_si512();
142 ghash_x4_accum(H4, H4_fold, M1, lo, hi, mid);
143 ghash_x4_accum(H3, H3_fold, M2, lo, hi, mid);
144 ghash_x4_accum(H2, H2_fold, M3, lo, hi, mid);
145 ghash_x4_accum(H1, H1_fold, M4, lo, hi, mid);
147 a = ghash_reduce(lo, hi, mid);
155 const auto H1 = _mm512_loadu_si512(H_pow);
156 const auto H2 = _mm512_loadu_si512(H_pow + 8);
158 const auto H1_fold = fold(H1);
159 const auto H2_fold = fold(H2);
162 __m512i M1 = load_blocks<BSWAP>(input);
163 const __m512i M2 = load_blocks<BSWAP>(input + 64);
165 M1 = insert_a(M1, a);
167 auto lo = _mm512_setzero_si512();
168 auto hi = _mm512_setzero_si512();
169 auto mid = _mm512_setzero_si512();
171 ghash_x4_accum(H2, H2_fold, M1, lo, hi, mid);
172 ghash_x4_accum(H1, H1_fold, M2, lo, hi, mid);
174 a = ghash_reduce(lo, hi, mid);
182 const auto H1 = _mm512_loadu_si512(H_pow);
183 const auto H1_fold = fold(H1);
186 __m512i M = load_blocks<BSWAP>(input);
189 const auto lo = _mm512_clmulepi64_epi128(H1, M, 0x00);
190 const auto hi = _mm512_clmulepi64_epi128(H1, M, 0x11);
191 const auto mid = _mm512_clmulepi64_epi128(H1_fold, fold(M), 0x00);
193 a = ghash_reduce(lo, hi, mid);
204 for(
size_t i = 0; i != blocks; ++i) {
216void BOTAN_FN_ISA_AVX512_CLMUL GHASH::ghash_precompute_avx512_clmul(
const uint8_t H_bytes[16], uint64_t H_pow[16 * 2]) {
220void BOTAN_FN_ISA_AVX512_CLMUL GHASH::ghash_multiply_avx512_clmul(uint8_t x[16],
221 const uint64_t H_pow[16 * 2],
222 const uint8_t input[],
224 multiply_avx512<true>(x, H_pow, input, blocks);
227#if defined(BOTAN_HAS_POLYVAL)
229void BOTAN_FN_ISA_AVX512_CLMUL Polyval::polyval_precompute_avx512_clmul(
const uint8_t H[16], uint64_t H_pow[16 * 2]) {
233void BOTAN_FN_ISA_AVX512_CLMUL Polyval::polyval_multiply_avx512_clmul(uint8_t x[16],
234 const uint64_t H_pow[16 * 2],
235 const uint8_t input[],
237 multiply_avx512<false>(x, H_pow, input, blocks);
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
#define BOTAN_FORCE_INLINE
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 mulx_polyval(const SIMD_4x32 &h)
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 load_block(const uint8_t in[])
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_multiply(const SIMD_4x32 &H, const SIMD_4x32 &x)
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_reduce(const SIMD_4x32 &hi, const SIMD_4x32 &lo)
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 void store_block(const SIMD_4x32 &b, uint8_t out[])