Botan 3.9.0
Crypto and TLS for C&
ghash_cpu.cpp
Go to the documentation of this file.
1/*
2* Hook for CLMUL/PMULL/VPMSUM
3* (C) 2013,2017,2019,2020 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#include <botan/internal/ghash.h>
9
10#include <botan/internal/isa_extn.h>
11#include <botan/internal/simd_4x32.h>
12#include <botan/internal/target_info.h>
13#include <bit>
14
15#if defined(BOTAN_SIMD_USE_SSSE3)
16 #include <immintrin.h>
17 #include <wmmintrin.h>
18#endif
19
20namespace Botan {
21
22namespace {
23
24BOTAN_FORCE_INLINE BOTAN_FN_ISA_SIMD_4X32 SIMD_4x32 reverse_vector(const SIMD_4x32& in) {
25#if defined(BOTAN_SIMD_USE_SSSE3)
26 const __m128i BSWAP_MASK = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
27 return SIMD_4x32(_mm_shuffle_epi8(in.raw(), BSWAP_MASK));
28#elif defined(BOTAN_SIMD_USE_NEON)
29 const uint8_t maskb[16] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
30 const uint8x16_t mask = vld1q_u8(maskb);
31 return SIMD_4x32(vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(in.raw()), mask)));
32#elif defined(BOTAN_SIMD_USE_ALTIVEC)
33 const __vector unsigned char mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
34 return SIMD_4x32(vec_perm(in.raw(), in.raw(), mask));
35#endif
36}
37
38template <int M>
39BOTAN_FORCE_INLINE BOTAN_FN_ISA_CLMUL SIMD_4x32 clmul(const SIMD_4x32& H, const SIMD_4x32& x) {
40 static_assert(M == 0x00 || M == 0x01 || M == 0x10 || M == 0x11, "Valid clmul mode");
41
42#if defined(BOTAN_SIMD_USE_SSSE3)
43 return SIMD_4x32(_mm_clmulepi64_si128(x.raw(), H.raw(), M));
44#elif defined(BOTAN_SIMD_USE_NEON)
45 const uint64_t a = vgetq_lane_u64(vreinterpretq_u64_u32(x.raw()), M & 0x01);
46 const uint64_t b = vgetq_lane_u64(vreinterpretq_u64_u32(H.raw()), (M & 0x10) >> 4);
47
48 #if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
49 __n64 a1 = {a}, b1 = {b};
50 return SIMD_4x32(vmull_p64(a1, b1));
51 #else
52 return SIMD_4x32(reinterpret_cast<uint32x4_t>(vmull_p64(a, b)));
53 #endif
54
55#elif defined(BOTAN_SIMD_USE_ALTIVEC)
56 const SIMD_4x32 mask_lo = SIMD_4x32(0, 0, 0xFFFFFFFF, 0xFFFFFFFF);
57 constexpr uint8_t flip = (std::endian::native == std::endian::big) ? 0x11 : 0x00;
58
59 SIMD_4x32 i1 = x;
60 SIMD_4x32 i2 = H;
61
62 if constexpr(std::endian::native == std::endian::big) {
63 i1 = reverse_vector(i1).bswap();
64 i2 = reverse_vector(i2).bswap();
65 }
66
67 if constexpr(M == (0x11 ^ flip)) {
68 i1 &= mask_lo;
69 i2 &= mask_lo;
70 } else if constexpr(M == (0x10 ^ flip)) {
71 i1 = i1.shift_elems_left<2>();
72 } else if constexpr(M == (0x01 ^ flip)) {
73 i2 = i2.shift_elems_left<2>();
74 } else if constexpr(M == (0x00 ^ flip)) {
75 i1 = mask_lo.andc(i1);
76 i2 = mask_lo.andc(i2);
77 }
78
79 auto i1v = reinterpret_cast<__vector unsigned long long>(i1.raw());
80 auto i2v = reinterpret_cast<__vector unsigned long long>(i2.raw());
81
82 #if BOTAN_COMPILER_HAS_BUILTIN(__builtin_crypto_vpmsumd)
83 auto rv = __builtin_crypto_vpmsumd(i1v, i2v);
84 #else
85 auto rv = __builtin_altivec_crypto_vpmsumd(i1v, i2v);
86 #endif
87
88 auto z = SIMD_4x32(reinterpret_cast<__vector unsigned int>(rv));
89
90 if constexpr(std::endian::native == std::endian::big) {
91 z = reverse_vector(z).bswap();
92 }
93
94 return z;
95#endif
96}
97
98inline SIMD_4x32 BOTAN_FN_ISA_CLMUL gcm_reduce(const SIMD_4x32& B0, const SIMD_4x32& B1) {
99 SIMD_4x32 X0 = B1.shr<31>();
100 SIMD_4x32 X1 = B1.shl<1>();
101 SIMD_4x32 X2 = B0.shr<31>();
102 SIMD_4x32 X3 = B0.shl<1>();
103
104 X3 |= X0.shift_elems_right<3>();
105 X3 |= X2.shift_elems_left<1>();
106 X1 |= X0.shift_elems_left<1>();
107
108 X0 = X1.shl<31>() ^ X1.shl<30>() ^ X1.shl<25>();
109
110 X1 ^= X0.shift_elems_left<3>();
111
112 X0 = X1 ^ X3 ^ X0.shift_elems_right<1>();
113 X0 ^= X1.shr<7>() ^ X1.shr<2>() ^ X1.shr<1>();
114 return X0;
115}
116
117inline SIMD_4x32 BOTAN_FN_ISA_CLMUL gcm_multiply(const SIMD_4x32& H, const SIMD_4x32& x) {
118 SIMD_4x32 T0 = clmul<0x11>(H, x);
119 SIMD_4x32 T1 = clmul<0x10>(H, x);
120 SIMD_4x32 T2 = clmul<0x01>(H, x);
121 SIMD_4x32 T3 = clmul<0x00>(H, x);
122
123 T1 ^= T2;
124 T0 ^= T1.shift_elems_right<2>();
125 T3 ^= T1.shift_elems_left<2>();
126
127 return gcm_reduce(T0, T3);
128}
129
130inline SIMD_4x32 BOTAN_FN_ISA_CLMUL gcm_multiply_x4(const SIMD_4x32& H1,
131 const SIMD_4x32& H2,
132 const SIMD_4x32& H3,
133 const SIMD_4x32& H4,
134 const SIMD_4x32& X1,
135 const SIMD_4x32& X2,
136 const SIMD_4x32& X3,
137 const SIMD_4x32& X4) {
138 /*
139 * Mutiply with delayed reduction, algorithm by Krzysztof Jankowski
140 * and Pierre Laurent of Intel
141 */
142
143 const SIMD_4x32 lo = (clmul<0x00>(H1, X1) ^ clmul<0x00>(H2, X2)) ^ (clmul<0x00>(H3, X3) ^ clmul<0x00>(H4, X4));
144
145 const SIMD_4x32 hi = (clmul<0x11>(H1, X1) ^ clmul<0x11>(H2, X2)) ^ (clmul<0x11>(H3, X3) ^ clmul<0x11>(H4, X4));
146
147 SIMD_4x32 T;
148
149 T ^= clmul<0x00>(H1 ^ H1.shift_elems_right<2>(), X1 ^ X1.shift_elems_right<2>());
150 T ^= clmul<0x00>(H2 ^ H2.shift_elems_right<2>(), X2 ^ X2.shift_elems_right<2>());
151 T ^= clmul<0x00>(H3 ^ H3.shift_elems_right<2>(), X3 ^ X3.shift_elems_right<2>());
152 T ^= clmul<0x00>(H4 ^ H4.shift_elems_right<2>(), X4 ^ X4.shift_elems_right<2>());
153 T ^= lo;
154 T ^= hi;
155
156 return gcm_reduce(hi ^ T.shift_elems_right<2>(), lo ^ T.shift_elems_left<2>());
157}
158
159} // namespace
160
161void BOTAN_FN_ISA_CLMUL GHASH::ghash_precompute_cpu(const uint8_t H_bytes[16], uint64_t H_pow[4 * 2]) {
162 const SIMD_4x32 H1 = reverse_vector(SIMD_4x32::load_le(H_bytes));
163 const SIMD_4x32 H2 = gcm_multiply(H1, H1);
164 const SIMD_4x32 H3 = gcm_multiply(H1, H2);
165 const SIMD_4x32 H4 = gcm_multiply(H2, H2);
166
167 H1.store_le(H_pow);
168 H2.store_le(H_pow + 2);
169 H3.store_le(H_pow + 4);
170 H4.store_le(H_pow + 6);
171}
172
173void BOTAN_FN_ISA_CLMUL GHASH::ghash_multiply_cpu(uint8_t x[16],
174 const uint64_t H_pow[8],
175 const uint8_t input[],
176 size_t blocks) {
177 /*
178 * Algorithms 1 and 5 from Intel's CLMUL guide
179 */
180 const SIMD_4x32 H1 = SIMD_4x32::load_le(H_pow);
181
182 SIMD_4x32 a = reverse_vector(SIMD_4x32::load_le(x));
183
184 if(blocks >= 4) {
185 const SIMD_4x32 H2 = SIMD_4x32::load_le(H_pow + 2);
186 const SIMD_4x32 H3 = SIMD_4x32::load_le(H_pow + 4);
187 const SIMD_4x32 H4 = SIMD_4x32::load_le(H_pow + 6);
188
189 while(blocks >= 4) {
190 const SIMD_4x32 m0 = reverse_vector(SIMD_4x32::load_le(input));
191 const SIMD_4x32 m1 = reverse_vector(SIMD_4x32::load_le(input + 16 * 1));
192 const SIMD_4x32 m2 = reverse_vector(SIMD_4x32::load_le(input + 16 * 2));
193 const SIMD_4x32 m3 = reverse_vector(SIMD_4x32::load_le(input + 16 * 3));
194
195 a ^= m0;
196 a = gcm_multiply_x4(H1, H2, H3, H4, m3, m2, m1, a);
197
198 input += 4 * 16;
199 blocks -= 4;
200 }
201 }
202
203 for(size_t i = 0; i != blocks; ++i) {
204 const SIMD_4x32 m = reverse_vector(SIMD_4x32::load_le(input + 16 * i));
205
206 a ^= m;
207 a = gcm_multiply(H1, a);
208 }
209
210 a = reverse_vector(a);
211 a.store_le(x);
212}
213
214} // namespace Botan
static SIMD_4x32 load_le(const void *in) noexcept
Definition simd_4x32.h:149
SIMD_4x32 shift_elems_left() const noexcept
Definition simd_4x32.h:552
SIMD_4x32 shr() const noexcept
Definition simd_4x32.h:479
SIMD_4x32 shl() const noexcept
Definition simd_4x32.h:461
SIMD_4x32 shift_elems_right() const noexcept
Definition simd_4x32.h:575
#define BOTAN_FORCE_INLINE
Definition compiler.h:87