9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
12#include <botan/compiler.h>
13#include <botan/types.h>
14#include <botan/internal/target_info.h>
17#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
18 #include <botan/internal/donna128.h>
23#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_X86_64)
24 #define BOTAN_MP_USE_X86_64_ASM
37static constexpr bool use_dword_for_word_add =
false;
39static constexpr bool use_dword_for_word_add =
true;
46concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
54 static const constexpr size_t bytes = 4;
55 static const constexpr size_t bits = 32;
56 static const constexpr uint32_t
max = 0xFFFFFFFF;
57 static const constexpr uint32_t
top_bit = 0x80000000;
66 static const constexpr size_t bytes = 8;
67 static const constexpr size_t bits = 64;
68 static const constexpr uint64_t
max = 0xFFFFFFFFFFFFFFFF;
69 static const constexpr uint64_t
top_bit = 0x8000000000000000;
71#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
72 typedef uint128_t
dword;
85#if defined(BOTAN_MP_USE_X86_64_ASM)
86 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
92 : [a] "=a"(a), [b]
"=rm"(b), [
carry]
"=&d"(*c)
93 :
"0"(a),
"1"(b), [c]
"g"(*c)
101 const dword s = dword(a) * b + *c;
103 return static_cast<W
>(s);
111#if defined(BOTAN_MP_USE_X86_64_ASM)
112 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
122 : [a] "=a"(a), [b]
"=rm"(b), [
carry]
"=&d"(*d)
123 :
"0"(a),
"1"(b), [c]
"g"(c), [d]
"g"(*d)
131 const dword s = dword(a) * b + c + *d;
133 return static_cast<W
>(s);
136#if defined(BOTAN_MP_USE_X86_64_ASM)
138 #define ASM(x) x "\n\t"
140 #define DO_8_TIMES(MACRO, ARG) \
150 #define ADDSUB2_OP(OPERATION, INDEX) \
151 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
152 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
154 #define ADDSUB3_OP(OPERATION, INDEX) \
155 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
156 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
157 ASM("movq %[carry], 8*" #INDEX "(%[z])")
159 #define LINMUL_OP(WRITE_TO, INDEX) \
160 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
162 ASM("addq %[carry],%%rax") \
163 ASM("adcq $0,%%rdx") \
164 ASM("movq %%rdx,%[carry]") \
165 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
167 #define MULADD_OP(IGNORED, INDEX) \
168 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
170 ASM("addq %[carry],%%rax") \
171 ASM("adcq $0,%%rdx") \
172 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
173 ASM("adcq $0,%%rdx") \
174 ASM("movq %%rdx,%[carry]") \
175 ASM("movq %%rax, 8*" #INDEX " (%[z])")
177 #define ADD_OR_SUBTRACT(CORE_CODE) \
178 ASM("rorq %[carry]") \
180 ASM("sbbq %[carry],%[carry]") \
190#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_addc)
191 if(!std::is_constant_evaluated()) {
192 if constexpr(std::same_as<W, unsigned int>) {
193 return __builtin_addc(x, y, *
carry & 1,
carry);
194 }
else if constexpr(std::same_as<W, unsigned long>) {
195 return __builtin_addcl(x, y, *
carry & 1,
carry);
196 }
else if constexpr(std::same_as<W, unsigned long long>) {
197 return __builtin_addcll(x, y, *
carry & 1,
carry);
208 const W cb = *
carry & 1;
211 return static_cast<W
>(s);
213 const W cb = *
carry & 1;
217 *
carry = c1 | (z < cb);
227#if defined(BOTAN_MP_USE_X86_64_ASM)
228 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
229 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"adcq"))
231 : [x]
"r"(x), [y]
"r"(y),
"0"(
carry)
253#if defined(BOTAN_MP_USE_X86_64_ASM)
254 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
255 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"adcq"))
257 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(
carry)
279#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_subc)
280 if(!std::is_constant_evaluated()) {
281 if constexpr(std::same_as<W, unsigned int>) {
282 return __builtin_subc(x, y, *
carry & 1,
carry);
283 }
else if constexpr(std::same_as<W, unsigned long>) {
284 return __builtin_subcl(x, y, *
carry & 1,
carry);
285 }
else if constexpr(std::same_as<W, unsigned long long>) {
286 return __builtin_subcll(x, y, *
carry & 1,
carry);
291 const W cb = *
carry & 1;
295 *
carry = c1 | (z > t0);
304#if defined(BOTAN_MP_USE_X86_64_ASM)
305 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
306 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"sbbq"))
308 : [x]
"r"(x), [y]
"r"(y),
"0"(
carry)
330#if defined(BOTAN_MP_USE_X86_64_ASM)
331 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
332 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbq"))
334 : [x]
"r"(y), [y]
"r"(x), [z]
"r"(x),
"0"(
carry)
356#if defined(BOTAN_MP_USE_X86_64_ASM)
357 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
358 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbq"))
360 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(
carry)
382#if defined(BOTAN_MP_USE_X86_64_ASM)
383 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
384 asm(DO_8_TIMES(LINMUL_OP,
"x")
386 : [x]
"r"(x), [y]
"rm"(y),
"0"(
carry)
387 :
"cc",
"%rax",
"%rdx");
408#if defined(BOTAN_MP_USE_X86_64_ASM)
409 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
410 asm(DO_8_TIMES(LINMUL_OP,
"z")
412 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(
carry)
413 :
"cc",
"%rax",
"%rdx");
434#if defined(BOTAN_MP_USE_X86_64_ASM)
435 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
436 asm(DO_8_TIMES(MULADD_OP,
"")
438 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(
carry)
439 :
"cc",
"%rax",
"%rdx");
465#if defined(__BITINT_MAXWIDTH__) && (__BITINT_MAXWIDTH__ >= 3 * 64)
468 constexpr word3() { m_w = 0; }
470 inline constexpr void mul(W x, W y) { m_w +=
static_cast<W3
>(x) * y; }
472 inline constexpr void mul_x2(W x, W y) { m_w +=
static_cast<W3
>(x) * y * 2; }
474 inline constexpr void add(W x) { m_w += x; }
477 W r =
static_cast<W
>(m_w);
482 inline constexpr W
monty_step(W p0, W p_dash) {
483 const W w0 =
static_cast<W
>(m_w);
484 const W r = w0 * p_dash;
491 const W r =
static_cast<W
>(m_w);
493 m_w +=
static_cast<W3
>(r);
509 inline constexpr void mul(W x, W y) {
510 #if defined(BOTAN_MP_USE_X86_64_ASM)
511 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
514 asm(
"mulq %[y]" :
"=a"(z0),
"=d"(z1) :
"a"(x), [y]
"rm"(y) :
"cc");
521 : [w0] "=r"(m_w0), [w1]
"=r"(m_w1), [w2]
"=r"(m_w2)
522 : [z0]
"r"(z0), [z1]
"r"(z1),
"0"(m_w0),
"1"(m_w1),
"2"(m_w2)
529 const dword s = dword(x) * y + m_w0;
531 m_w0 =
static_cast<W
>(s);
533 m_w2 += (m_w1 <
carry);
509 inline constexpr void mul(W x, W y) {
…}
537 #if defined(BOTAN_MP_USE_X86_64_ASM)
538 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
541 asm(
"mulq %[y]" :
"=a"(z0),
"=d"(z1) :
"a"(x), [y]
"rm"(y) :
"cc");
552 : [w0] "=r"(m_w0), [w1]
"=r"(m_w1), [w2]
"=r"(m_w2)
553 : [z0]
"r"(z0), [z1]
"r"(z1),
"0"(m_w0),
"1"(m_w1),
"2"(m_w2)
574 inline constexpr void add(W x) {
574 inline constexpr void add(W x) {
…}
604 m_w1 = m_w2 + (m_w0 < m_w1);
617 #undef ADD_OR_SUBTRACT
constexpr W monty_step(W p0, W p_dash)
constexpr void mul(W x, W y)
constexpr W monty_step_pdash1()
constexpr void mul_x2(W x, W y)
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
constexpr auto word_sub(W x, W y, W *carry) -> W
constexpr auto word_add(W x, W y, W *carry) -> W
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
constexpr auto word_madd2(W a, W b, W *c) -> W
constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W
void carry(int64_t &h0, int64_t &h1)
constexpr auto word8_linmul2(W x[8], W y, W carry) -> W
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
static const constexpr bool dword_is_native
static const constexpr uint32_t top_bit
static const constexpr size_t bytes
static const constexpr size_t bits
static const constexpr uint32_t max
static const constexpr size_t bytes
static const constexpr bool dword_is_native
static const constexpr size_t bits
static const constexpr uint64_t max
static const constexpr uint64_t top_bit