Botan 3.14.1
Crypto and TLS for C&
cpuid_x86.cpp
Go to the documentation of this file.
1/*
2* Runtime CPU detection for x86
3* (C) 2009,2010,2013,2017,2023,2024 Jack Lloyd
4*
5* Botan is released under the Simplified BSD License (see license.txt)
6*/
7
8#include <botan/internal/cpuid.h>
9
10#include <botan/compiler.h>
11#include <botan/mem_ops.h>
12#include <botan/internal/target_info.h>
13
14#include <immintrin.h>
15
16#if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
17 #include <intrin.h>
18#endif
19
20namespace Botan {
21
22namespace {
23
24void invoke_cpuid(uint32_t type, uint32_t out[4]) {
25 clear_mem(out, 4);
26
27#if defined(BOTAN_USE_GCC_INLINE_ASM)
28 // NOLINTNEXTLINE(*-no-assembler)
29 asm volatile("cpuid\n\t" : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) : "0"(type));
30
31#elif defined(BOTAN_BUILD_COMPILER_IS_MSVC)
32 __cpuid((int*)out, type);
33
34#else
35 BOTAN_UNUSED(type);
36 #warning "No way of calling x86 cpuid instruction for this compiler"
37#endif
38}
39
40void invoke_cpuid_sublevel(uint32_t type, uint32_t level, uint32_t out[4]) {
41 clear_mem(out, 4);
42
43#if defined(BOTAN_USE_GCC_INLINE_ASM)
44 // NOLINTNEXTLINE(*-no-assembler)
45 asm volatile("cpuid\n\t" : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) : "0"(type), "2"(level));
46
47#elif defined(BOTAN_BUILD_COMPILER_IS_MSVC)
48 __cpuidex((int*)out, type, level);
49
50#else
51 BOTAN_UNUSED(type, level);
52 #warning "No way of calling x86 cpuid instruction for this compiler"
53#endif
54}
55
56BOTAN_FUNC_ISA("xsave") uint64_t xgetbv() {
57 return _xgetbv(0);
58}
59
60} // namespace
61
62uint32_t CPUID::CPUID_Data::detect_cpu_features(uint32_t allowed) {
63 enum class x86_CPUID_1_bits : uint64_t {
64 RDTSC = (1ULL << 4),
65 SSE2 = (1ULL << 26),
66 CLMUL = (1ULL << 33),
67 SSSE3 = (1ULL << 41),
68 SSE41 = (1ULL << 51),
69 POPCNT = (1ULL << 55),
70 AESNI = (1ULL << 57),
71 // AVX + OSXSAVE
72 OSXSAVE = (1ULL << 59) | (1ULL << 60),
73 RDRAND = (1ULL << 62)
74 };
75
76 enum class x86_CPUID_7_bits : uint64_t {
77 BMI1 = (1ULL << 3),
78 AVX2 = (1ULL << 5),
79 BMI2 = (1ULL << 8),
80 BMI_1_AND_2 = BMI1 | BMI2,
81 AVX512_F = (1ULL << 16),
82 AVX512_DQ = (1ULL << 17),
83 RDSEED = (1ULL << 18),
84 ADX = (1ULL << 19),
85 AVX512_IFMA = (1ULL << 21),
86 SHA = (1ULL << 29),
87 AVX512_BW = (1ULL << 30),
88 AVX512_VL = (1ULL << 31),
89 AVX512_VBMI = (1ULL << 33),
90 AVX512_VBMI2 = (1ULL << 38),
91 GFNI = (1ULL << 40),
92 AVX512_VAES = (1ULL << 41),
93 AVX512_VCLMUL = (1ULL << 42),
94 AVX512_VBITALG = (1ULL << 44),
95
96 /*
97 We only enable AVX512 support if all of the below flags are available
98
99 This is more than we strictly need for most uses, however it also has
100 the effect of preventing execution of AVX512 codepaths on cores that
101 have serious downclocking problems when AVX512 code executes,
102 especially Intel Skylake.
103
104 VBMI2/VBITALG are the key flags here as they restrict us to Intel Ice
105 Lake/Rocket Lake, or AMD Zen4, all of which do not have penalties for
106 executing AVX512.
107
108 There is nothing stopping some future processor from supporting the
109 above flags and having AVX512 penalties, but maybe you should not have
110 bought such a processor.
111 */
112 AVX512_PROFILE =
113 AVX512_F | AVX512_DQ | AVX512_IFMA | AVX512_BW | AVX512_VL | AVX512_VBMI | AVX512_VBMI2 | AVX512_VBITALG,
114 };
115
116 // NOLINTNEXTLINE(performance-enum-size)
117 enum class x86_CPUID_7_1_bits : uint64_t {
118 SHA512 = (1 << 0),
119 SM3 = (1 << 1),
120 SM4 = (1 << 2),
121 };
122
123 uint32_t feat = 0;
124 uint32_t cpuid[4] = {0};
125 bool has_sse41 = false;
126 bool has_os_ymm_support = false;
127 bool has_os_zmm_support = false;
128
129 // CPUID 0: vendor identification, max sublevel
130 invoke_cpuid(0, cpuid);
131
132 const uint32_t max_supported_sublevel = cpuid[0];
133
134 if(max_supported_sublevel >= 1) {
135 // CPUID 1: feature bits
136 invoke_cpuid(1, cpuid);
137 const uint64_t flags0 = (static_cast<uint64_t>(cpuid[2]) << 32) | cpuid[3];
138
139 feat |= if_set(flags0, x86_CPUID_1_bits::RDTSC, CPUFeature::Bit::RDTSC, allowed);
140
141 feat |= if_set(flags0, x86_CPUID_1_bits::RDRAND, CPUFeature::Bit::RDRAND, allowed);
142
143 feat |= if_set(flags0, x86_CPUID_1_bits::POPCNT, CPUFeature::Bit::POPCNT, allowed);
144
145 feat |= if_set(flags0, x86_CPUID_1_bits::SSE2, CPUFeature::Bit::SSE2, allowed);
146
147 if(is_set(feat, CPUFeature::Bit::SSE2)) {
148 feat |= if_set(flags0, x86_CPUID_1_bits::SSSE3, CPUFeature::Bit::SSSE3, allowed);
149
150 if(is_set(feat, CPUFeature::Bit::SSSE3)) {
151 feat |= if_set(flags0, x86_CPUID_1_bits::CLMUL, CPUFeature::Bit::CLMUL, allowed);
152 feat |= if_set(flags0, x86_CPUID_1_bits::AESNI, CPUFeature::Bit::AESNI, allowed);
153
154 // SSE4.1 is not tracked as a feature, but is required by the SHA-NI code
155 const uint64_t sse41 = static_cast<uint64_t>(x86_CPUID_1_bits::SSE41);
156 has_sse41 = (flags0 & sse41) == sse41;
157 }
158
159 const uint64_t osxsave64 = static_cast<uint64_t>(x86_CPUID_1_bits::OSXSAVE);
160 if((flags0 & osxsave64) == osxsave64) {
161 const uint64_t xcr_flags = xgetbv();
162 if((xcr_flags & 0x6) == 0x6) {
163 has_os_ymm_support = true;
164 has_os_zmm_support = (xcr_flags & 0xE0) == 0xE0;
165 }
166 }
167 }
168 }
169
170 if(max_supported_sublevel >= 7) {
171 clear_mem(cpuid, 4);
172 invoke_cpuid_sublevel(7, 0, cpuid);
173
174 const uint64_t flags7 = (static_cast<uint64_t>(cpuid[2]) << 32) | cpuid[1];
175
176 clear_mem(cpuid, 4);
177 invoke_cpuid_sublevel(7, 1, cpuid);
178 const uint32_t flags7_1 = cpuid[0];
179
180 feat |= if_set(flags7, x86_CPUID_7_bits::RDSEED, CPUFeature::Bit::RDSEED, allowed);
181 feat |= if_set(flags7, x86_CPUID_7_bits::ADX, CPUFeature::Bit::ADX, allowed);
182
183 /*
184 We only set the BMI bit if both BMI1 and BMI2 are supported, since
185 typically we want to use both extensions in the same code.
186 */
187 feat |= if_set(flags7, x86_CPUID_7_bits::BMI_1_AND_2, CPUFeature::Bit::BMI, allowed);
188
189 if(is_set(feat, CPUFeature::Bit::SSSE3)) {
190 // The SHA-NI code uses SSE4.1 instructions in addition to SSSE3
191 if(has_sse41) {
192 feat |= if_set(flags7, x86_CPUID_7_bits::SHA, CPUFeature::Bit::SHA, allowed);
193 }
194
195 // We only consider AVX2 if SSSE3 is supported
196 if(has_os_ymm_support) {
197 feat |= if_set(flags7, x86_CPUID_7_bits::AVX2, CPUFeature::Bit::AVX2, allowed);
198
199 if(is_set(feat, CPUFeature::Bit::AVX2)) {
200 feat |= if_set(flags7, x86_CPUID_7_bits::GFNI, CPUFeature::Bit::GFNI, allowed);
201 if(is_set(feat, CPUFeature::Bit::AESNI)) {
202 feat |= if_set(flags7, x86_CPUID_7_bits::AVX512_VAES, CPUFeature::Bit::AVX2_AES, allowed);
203 }
204 feat |= if_set(flags7_1, x86_CPUID_7_1_bits::SHA512, CPUFeature::Bit::SHA512, allowed);
205 // The SM3/SM4 instructions are VEX encoded and the code using them also uses AVX2
206 feat |= if_set(flags7_1, x86_CPUID_7_1_bits::SM3, CPUFeature::Bit::SM3, allowed);
207 feat |= if_set(flags7_1, x86_CPUID_7_1_bits::SM4, CPUFeature::Bit::SM4, allowed);
208
209 // The VPCLMULQDQ code also uses the legacy 128-bit PCLMULQDQ instruction
210 if(is_set(feat, CPUFeature::Bit::CLMUL)) {
211 feat |= if_set(flags7, x86_CPUID_7_bits::AVX512_VCLMUL, CPUFeature::Bit::AVX2_CLMUL, allowed);
212 }
213
214 // Likewise we only consider AVX-512 if AVX2 is supported
215 if(has_os_zmm_support) {
216 feat |= if_set(flags7, x86_CPUID_7_bits::AVX512_PROFILE, CPUFeature::Bit::AVX512, allowed);
217
218 if(is_set(feat, CPUFeature::Bit::AVX512)) {
219 if(is_set(feat, CPUFeature::Bit::AESNI)) {
220 feat |= if_set(flags7, x86_CPUID_7_bits::AVX512_VAES, CPUFeature::Bit::AVX512_AES, allowed);
221 }
222 if(is_set(feat, CPUFeature::Bit::CLMUL)) {
223 feat |= if_set(flags7, x86_CPUID_7_bits::AVX512_VCLMUL, CPUFeature::Bit::AVX512_CLMUL, allowed);
224 }
225 }
226 }
227 }
228 }
229 }
230 }
231
232/*
233 * If we don't have access to CPUID, we can still safely assume that
234 * any x86-64 processor has SSE2 and RDTSC
235 */
236#if defined(BOTAN_TARGET_ARCH_IS_X86_64)
237 if(feat == 0) {
238 feat |= CPUFeature::Bit::SSE2 & allowed;
239 feat |= CPUFeature::Bit::RDTSC & allowed;
240 }
241#endif
242
243 return feat;
244}
245
246} // namespace Botan
#define BOTAN_UNUSED
Definition assert.h:151
static uint32_t if_set(uint64_t cpuid, T flag, CPUID::Feature bit, uint32_t allowed)
Definition cpuid.h:116
#define BOTAN_FUNC_ISA(isa)
Definition compiler.h:55
constexpr void clear_mem(T *ptr, size_t n)
Definition mem_ops.h:118