Botan 3.13.0
Crypto and TLS for C&
ghash_cpu.cpp
Go to the documentation of this file.
1/*
2* Hook for CLMUL/PMULL/VPMSUM
3* (C) 2013,2017,2019,2020 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#include <botan/internal/ghash.h>
9
10#include <botan/internal/isa_extn.h>
11#include <botan/internal/polyval_fn.h>
12#include <botan/internal/simd_4x32.h>
13
14#if defined(BOTAN_HAS_POLYVAL)
15 #include <botan/internal/polyval.h>
16#endif
17
18namespace Botan {
19
20namespace {
21
22inline SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_multiply_x4(const SIMD_4x32& H1,
23 const SIMD_4x32& H2,
24 const SIMD_4x32& H3,
25 const SIMD_4x32& H4,
26 const SIMD_4x32& X1,
27 const SIMD_4x32& X2,
28 const SIMD_4x32& X3,
29 const SIMD_4x32& X4) {
30 const SIMD_4x32 lo = (clmul<0x00>(H1, X1) ^ clmul<0x00>(H2, X2)) ^ (clmul<0x00>(H3, X3) ^ clmul<0x00>(H4, X4));
31 const SIMD_4x32 hi = (clmul<0x11>(H1, X1) ^ clmul<0x11>(H2, X2)) ^ (clmul<0x11>(H3, X3) ^ clmul<0x11>(H4, X4));
32
33 SIMD_4x32 mid;
34
35 mid ^= clmul<0x00>(H1 ^ H1.shift_elems_right<2>(), X1 ^ X1.shift_elems_right<2>());
36 mid ^= clmul<0x00>(H2 ^ H2.shift_elems_right<2>(), X2 ^ X2.shift_elems_right<2>());
37 mid ^= clmul<0x00>(H3 ^ H3.shift_elems_right<2>(), X3 ^ X3.shift_elems_right<2>());
38 mid ^= clmul<0x00>(H4 ^ H4.shift_elems_right<2>(), X4 ^ X4.shift_elems_right<2>());
39 mid ^= lo;
40 mid ^= hi;
41
42 return polyval_reduce(hi ^ mid.shift_elems_right<2>(), lo ^ mid.shift_elems_left<2>());
43}
44
45inline SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_multiply_x8(const SIMD_4x32& H1,
46 const SIMD_4x32& H2,
47 const SIMD_4x32& H3,
48 const SIMD_4x32& H4,
49 const SIMD_4x32& H5,
50 const SIMD_4x32& H6,
51 const SIMD_4x32& H7,
52 const SIMD_4x32& H8,
53 const SIMD_4x32& X1,
54 const SIMD_4x32& X2,
55 const SIMD_4x32& X3,
56 const SIMD_4x32& X4,
57 const SIMD_4x32& X5,
58 const SIMD_4x32& X6,
59 const SIMD_4x32& X7,
60 const SIMD_4x32& X8) {
61 const SIMD_4x32 lo = clmul<0x00>(H1, X1) ^ clmul<0x00>(H2, X2) ^ clmul<0x00>(H3, X3) ^ clmul<0x00>(H4, X4) ^
62 clmul<0x00>(H5, X5) ^ clmul<0x00>(H6, X6) ^ clmul<0x00>(H7, X7) ^ clmul<0x00>(H8, X8);
63
64 const SIMD_4x32 hi = clmul<0x11>(H1, X1) ^ clmul<0x11>(H2, X2) ^ clmul<0x11>(H3, X3) ^ clmul<0x11>(H4, X4) ^
65 clmul<0x11>(H5, X5) ^ clmul<0x11>(H6, X6) ^ clmul<0x11>(H7, X7) ^ clmul<0x11>(H8, X8);
66
67 SIMD_4x32 mid;
68
69 mid ^= clmul<0x00>(H1 ^ H1.shift_elems_right<2>(), X1 ^ X1.shift_elems_right<2>());
70 mid ^= clmul<0x00>(H2 ^ H2.shift_elems_right<2>(), X2 ^ X2.shift_elems_right<2>());
71 mid ^= clmul<0x00>(H3 ^ H3.shift_elems_right<2>(), X3 ^ X3.shift_elems_right<2>());
72 mid ^= clmul<0x00>(H4 ^ H4.shift_elems_right<2>(), X4 ^ X4.shift_elems_right<2>());
73 mid ^= clmul<0x00>(H5 ^ H5.shift_elems_right<2>(), X5 ^ X5.shift_elems_right<2>());
74 mid ^= clmul<0x00>(H6 ^ H6.shift_elems_right<2>(), X6 ^ X6.shift_elems_right<2>());
75 mid ^= clmul<0x00>(H7 ^ H7.shift_elems_right<2>(), X7 ^ X7.shift_elems_right<2>());
76 mid ^= clmul<0x00>(H8 ^ H8.shift_elems_right<2>(), X8 ^ X8.shift_elems_right<2>());
77 mid ^= lo;
78 mid ^= hi;
79
80 return polyval_reduce(hi ^ mid.shift_elems_right<2>(), lo ^ mid.shift_elems_left<2>());
81}
82
83void BOTAN_FN_ISA_CLMUL precompute_clmul(const SIMD_4x32& H1, secure_vector<uint64_t>& H_pow) {
84 const SIMD_4x32 H2 = polyval_multiply(H1, H1);
85 const SIMD_4x32 H3 = polyval_multiply(H1, H2);
86 const SIMD_4x32 H4 = polyval_multiply(H2, H2);
87
88 H_pow.reserve(2 * 8);
89 H_pow.resize(2 * 4);
90 H1.store_le(&H_pow[0]); // NOLINT(*-container-data-pointer)
91 H2.store_le(&H_pow[2]);
92 H3.store_le(&H_pow[4]);
93 H4.store_le(&H_pow[6]);
94}
95
96template <bool BSWAP>
97void BOTAN_FN_ISA_CLMUL
98multiply_clmul(uint8_t x[16], secure_vector<uint64_t>& H_pow, const uint8_t input[], size_t blocks) {
99 BOTAN_ASSERT_NOMSG(H_pow.size() == 2 * 4 || H_pow.size() == 2 * 8);
100
101 const SIMD_4x32 H1 = SIMD_4x32::load_le(&H_pow[0]); // NOLINT(*-container-data-pointer)
102
103 SIMD_4x32 a = load_block<BSWAP>(x);
104
105 if(blocks >= 8) {
106 const SIMD_4x32 H2 = SIMD_4x32::load_le(&H_pow[2]);
107 const SIMD_4x32 H3 = SIMD_4x32::load_le(&H_pow[4]);
108 const SIMD_4x32 H4 = SIMD_4x32::load_le(&H_pow[6]);
109
110 if(H_pow.size() < 2 * 8) {
111 H_pow.resize(2 * 8);
112 const SIMD_4x32 H5 = polyval_multiply(H4, H1);
113 const SIMD_4x32 H6 = polyval_multiply(H4, H2);
114 const SIMD_4x32 H7 = polyval_multiply(H4, H3);
115 const SIMD_4x32 H8 = polyval_multiply(H4, H4);
116 H5.store_le(&H_pow[8]);
117 H6.store_le(&H_pow[10]);
118 H7.store_le(&H_pow[12]);
119 H8.store_le(&H_pow[14]);
120 }
121
122 const SIMD_4x32 H5 = SIMD_4x32::load_le(&H_pow[8]);
123 const SIMD_4x32 H6 = SIMD_4x32::load_le(&H_pow[10]);
124 const SIMD_4x32 H7 = SIMD_4x32::load_le(&H_pow[12]);
125 const SIMD_4x32 H8 = SIMD_4x32::load_le(&H_pow[14]);
126
127 while(blocks >= 8) {
128 const SIMD_4x32 m0 = load_block<BSWAP>(input);
129 const SIMD_4x32 m1 = load_block<BSWAP>(input + 16 * 1);
130 const SIMD_4x32 m2 = load_block<BSWAP>(input + 16 * 2);
131 const SIMD_4x32 m3 = load_block<BSWAP>(input + 16 * 3);
132 const SIMD_4x32 m4 = load_block<BSWAP>(input + 16 * 4);
133 const SIMD_4x32 m5 = load_block<BSWAP>(input + 16 * 5);
134 const SIMD_4x32 m6 = load_block<BSWAP>(input + 16 * 6);
135 const SIMD_4x32 m7 = load_block<BSWAP>(input + 16 * 7);
136
137 a = polyval_multiply_x8(H1, H2, H3, H4, H5, H6, H7, H8, m7, m6, m5, m4, m3, m2, m1, m0 ^ a);
138
139 input += 8 * 16;
140 blocks -= 8;
141 }
142 }
143
144 if(blocks >= 4) {
145 const SIMD_4x32 H2 = SIMD_4x32::load_le(&H_pow[2]);
146 const SIMD_4x32 H3 = SIMD_4x32::load_le(&H_pow[4]);
147 const SIMD_4x32 H4 = SIMD_4x32::load_le(&H_pow[6]);
148
149 while(blocks >= 4) {
150 const SIMD_4x32 m0 = load_block<BSWAP>(input);
151 const SIMD_4x32 m1 = load_block<BSWAP>(input + 16 * 1);
152 const SIMD_4x32 m2 = load_block<BSWAP>(input + 16 * 2);
153 const SIMD_4x32 m3 = load_block<BSWAP>(input + 16 * 3);
154
155 a ^= m0;
156 a = polyval_multiply_x4(H1, H2, H3, H4, m3, m2, m1, a);
157
158 input += 4 * 16;
159 blocks -= 4;
160 }
161 }
162
163 for(size_t i = 0; i != blocks; ++i) {
164 const SIMD_4x32 m = load_block<BSWAP>(input + 16 * i);
165
166 a ^= m;
167 a = polyval_multiply(H1, a);
168 }
169
170 store_block<BSWAP>(a, x);
171}
172
173} // namespace
174
175void BOTAN_FN_ISA_CLMUL GHASH::ghash_precompute_cpu(const uint8_t H_bytes[16], secure_vector<uint64_t>& H_pow) {
176 precompute_clmul(mulx_polyval(reverse_vector(SIMD_4x32::load_le(H_bytes))), H_pow);
177}
178
179void BOTAN_FN_ISA_CLMUL GHASH::ghash_multiply_cpu(uint8_t x[16],
180 secure_vector<uint64_t>& H_pow,
181 const uint8_t input[],
182 size_t blocks) {
183 multiply_clmul<true>(x, H_pow, input, blocks);
184}
185
186#if defined(BOTAN_HAS_POLYVAL)
187
188void BOTAN_FN_ISA_CLMUL Polyval::polyval_precompute_cpu(const uint8_t H[16], secure_vector<uint64_t>& H_pow) {
189 precompute_clmul(SIMD_4x32::load_le(H), H_pow);
190}
191
192void BOTAN_FN_ISA_CLMUL Polyval::polyval_multiply_cpu(uint8_t x[16],
193 secure_vector<uint64_t>& H_pow,
194 const uint8_t input[],
195 size_t blocks) {
196 multiply_clmul<false>(x, H_pow, input, blocks);
197}
198
199#endif
200
201} // namespace Botan
BOTAN_ASSERT_NOMSG
#define BOTAN_ASSERT_NOMSG(expr)
Definition assert.h:75
Botan::SIMD_4x32
Definition simd_4x32.h:61
Botan::SIMD_4x32::load_le
static SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 load_le(const void *in) noexcept
Definition simd_4x32.h:162
Botan::mulx_polyval
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_SIMD_4X32 mulx_polyval(const SIMD_4x32 &h)
Definition polyval_fn.h:114
Botan::load_block
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 load_block(const uint8_t in[])
Definition polyval_fn.h:97
Botan::polyval_multiply
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_multiply(const SIMD_4x32 &H, const SIMD_4x32 &x)
Definition polyval_fn.h:150
Botan::polyval_reduce
BOTAN_FORCE_INLINE SIMD_4x32 BOTAN_FN_ISA_CLMUL polyval_reduce(const SIMD_4x32 &hi, const SIMD_4x32 &lo)
Definition polyval_fn.h:129
Botan::clmul
BOTAN_FORCE_INLINE BOTAN_FN_ISA_CLMUL SIMD_4x32 clmul(const SIMD_4x32 &H, const SIMD_4x32 &x)
Definition polyval_fn.h:31
Botan::reverse_vector
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 reverse_vector(const SIMD_4x32 &in)
Definition polyval_fn.h:16
Botan::secure_vector
std::vector< T, secure_allocator< T > > secure_vector
Definition secmem.h:128
Botan::store_block
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 void store_block(const SIMD_4x32 &b, uint8_t out[])
Definition polyval_fn.h:106
Generated by doxygen 1.16.1