9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
12#include <botan/compiler.h>
13#include <botan/types.h>
14#include <botan/internal/target_info.h>
17#if defined(__has_feature)
18 #if __has_feature(memory_sanitizer)
19 #define BOTAN_MP_ASM_HAS_MSAN
20 #include <sanitizer/msan_interface.h>
24#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
25 #include <botan/internal/donna128.h>
32#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_X86_64)
33 #define BOTAN_MP_USE_X86_64_ASM
36#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
37 #define BOTAN_MP_USE_AARCH64_ASM
50static constexpr bool use_dword_for_word_add =
false;
52static constexpr bool use_dword_for_word_add =
true;
59concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
67 static const constexpr size_t bytes = 4;
68 static const constexpr size_t bits = 32;
69 static const constexpr uint32_t
max = 0xFFFFFFFF;
70 static const constexpr uint32_t
top_bit = 0x80000000;
79 static const constexpr size_t bytes = 8;
80 static const constexpr size_t bits = 64;
81 static const constexpr uint64_t
max = 0xFFFFFFFFFFFFFFFF;
82 static const constexpr uint64_t
top_bit = 0x8000000000000000;
84#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
100template <
size_t K, WordType W>
102#if defined(BOTAN_MP_ASM_HAS_MSAN)
103 __msan_unpoison(ptr, K *
sizeof(W));
112#if defined(BOTAN_MP_USE_X86_64_ASM)
113 if constexpr(std::same_as<W, uint64_t>) {
114 if(!std::is_constant_evaluated()) {
120 : [a] "=a"(a), [b]
"=rm"(b), [carry]
"=&d"(*c)
121 :
"0"(a),
"1"(b), [c]
"g"(*c)
127#elif defined(BOTAN_MP_USE_AARCH64_ASM)
128 if constexpr(std::same_as<W, uint64_t>) {
129 if(!std::is_constant_evaluated()) {
133 mul %[lo], %[a], %[b]
134 umulh %[hi], %[a], %[b]
135 adds %[lo], %[lo], %[c]
136 adc %[hi], %[hi], xzr
138 : [lo] "=&r"(lo), [hi]
"=&r"(hi)
139 : [a]
"r"(a), [b]
"r"(b), [c]
"r"(*c)
149 const dword s = dword(a) * b + *c;
151 return static_cast<W
>(s);
159#if defined(BOTAN_MP_USE_X86_64_ASM)
160 if constexpr(std::same_as<W, uint64_t>) {
161 if(!std::is_constant_evaluated()) {
171 : [a] "=a"(a), [b]
"=rm"(b), [carry]
"=&d"(*d)
172 :
"0"(a),
"1"(b), [c]
"g"(c), [d]
"g"(*d)
178#elif defined(BOTAN_MP_USE_AARCH64_ASM)
179 if constexpr(std::same_as<W, uint64_t>) {
180 if(!std::is_constant_evaluated()) {
184 mul %[lo], %[a], %[b]
185 umulh %[hi], %[a], %[b]
186 adds %[lo], %[lo], %[c]
187 adc %[hi], %[hi], xzr
188 adds %[lo], %[lo], %[d]
189 adc %[hi], %[hi], xzr
191 : [lo] "=&r"(lo), [hi]
"=&r"(hi)
192 : [a]
"r"(a), [b]
"r"(b), [c]
"r"(c), [d]
"r"(*d)
202 const dword s = dword(a) * b + c + *d;
204 return static_cast<W
>(s);
207#if defined(BOTAN_MP_USE_X86_64_ASM)
209 #define ASM(x) x "\n\t"
211 #define DO_8_TIMES(MACRO, ARG) \
221 #define ADDSUB2_OP(OPERATION, INDEX) \
222 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
223 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
225 #define ADDSUB3_OP(OPERATION, INDEX) \
226 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
227 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
228 ASM("movq %[carry], 8*" #INDEX "(%[z])")
230 #define LINMUL_OP(WRITE_TO, INDEX) \
231 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
233 ASM("addq %[carry],%%rax") \
234 ASM("adcq $0,%%rdx") \
235 ASM("movq %%rdx,%[carry]") \
236 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
238 #define MULADD_OP(IGNORED, INDEX) \
239 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
241 ASM("addq %[carry],%%rax") \
242 ASM("adcq $0,%%rdx") \
243 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
244 ASM("adcq $0,%%rdx") \
245 ASM("movq %%rdx,%[carry]") \
246 ASM("movq %%rax, 8*" #INDEX " (%[z])")
248 #define ADD_OR_SUBTRACT(CORE_CODE) \
249 ASM("rorq %[carry]") \
251 ASM("sbbq %[carry],%[carry]") \
260inline constexpr auto word_add(W x, W y, W* carry) -> W {
261#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_addc)
262 if(!std::is_constant_evaluated()) {
263 if constexpr(std::same_as<W, unsigned int>) {
264 return __builtin_addc(x, y, *carry & 1, carry);
265 }
else if constexpr(std::same_as<W, unsigned long>) {
266 return __builtin_addcl(x, y, *carry & 1, carry);
267 }
else if constexpr(std::same_as<W, unsigned long long>) {
268 return __builtin_addcll(x, y, *carry & 1, carry);
279 const W cb = *carry & 1;
282 return static_cast<W
>(s);
284 const W cb = *carry & 1;
288 *carry = c1 | (z < cb);
297inline constexpr auto word8_add2(W x[8],
const W y[8], W carry) -> W {
298#if defined(BOTAN_MP_USE_X86_64_ASM)
299 if constexpr(std::same_as<W, uint64_t>) {
300 if(!std::is_constant_evaluated()) {
301 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"adcq"))
302 : [carry]
"=r"(carry)
303 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
311 x[0] =
word_add(x[0], y[0], &carry);
312 x[1] =
word_add(x[1], y[1], &carry);
313 x[2] =
word_add(x[2], y[2], &carry);
314 x[3] =
word_add(x[3], y[3], &carry);
315 x[4] =
word_add(x[4], y[4], &carry);
316 x[5] =
word_add(x[5], y[5], &carry);
317 x[6] =
word_add(x[6], y[6], &carry);
318 x[7] =
word_add(x[7], y[7], &carry);
326inline constexpr auto word8_add3(W z[8],
const W x[8],
const W y[8], W carry) -> W {
327#if defined(BOTAN_MP_USE_X86_64_ASM)
328 if constexpr(std::same_as<W, uint64_t>) {
329 if(!std::is_constant_evaluated()) {
330 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"adcq"))
331 : [carry]
"=r"(carry)
332 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
340 z[0] =
word_add(x[0], y[0], &carry);
341 z[1] =
word_add(x[1], y[1], &carry);
342 z[2] =
word_add(x[2], y[2], &carry);
343 z[3] =
word_add(x[3], y[3], &carry);
344 z[4] =
word_add(x[4], y[4], &carry);
345 z[5] =
word_add(x[5], y[5], &carry);
346 z[6] =
word_add(x[6], y[6], &carry);
347 z[7] =
word_add(x[7], y[7], &carry);
355inline constexpr auto word_sub(W x, W y, W* carry) -> W {
356#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_subc)
357 if(!std::is_constant_evaluated()) {
358 if constexpr(std::same_as<W, unsigned int>) {
359 return __builtin_subc(x, y, *carry & 1, carry);
360 }
else if constexpr(std::same_as<W, unsigned long>) {
361 return __builtin_subcl(x, y, *carry & 1, carry);
362 }
else if constexpr(std::same_as<W, unsigned long long>) {
363 return __builtin_subcll(x, y, *carry & 1, carry);
368 const W cb = *carry & 1;
372 *carry = c1 | (z > t0);
380inline constexpr auto word8_sub2(W x[8],
const W y[8], W carry) -> W {
381#if defined(BOTAN_MP_USE_X86_64_ASM)
382 if constexpr(std::same_as<W, uint64_t>) {
383 if(!std::is_constant_evaluated()) {
384 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"sbbq"))
385 : [carry]
"=r"(carry)
386 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
394 x[0] =
word_sub(x[0], y[0], &carry);
395 x[1] =
word_sub(x[1], y[1], &carry);
396 x[2] =
word_sub(x[2], y[2], &carry);
397 x[3] =
word_sub(x[3], y[3], &carry);
398 x[4] =
word_sub(x[4], y[4], &carry);
399 x[5] =
word_sub(x[5], y[5], &carry);
400 x[6] =
word_sub(x[6], y[6], &carry);
401 x[7] =
word_sub(x[7], y[7], &carry);
409inline constexpr auto word8_sub3(W z[8],
const W x[8],
const W y[8], W carry) -> W {
410#if defined(BOTAN_MP_USE_X86_64_ASM)
411 if constexpr(std::same_as<W, uint64_t>) {
412 if(!std::is_constant_evaluated()) {
413 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbq"))
414 : [carry]
"=r"(carry)
415 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
423 z[0] =
word_sub(x[0], y[0], &carry);
424 z[1] =
word_sub(x[1], y[1], &carry);
425 z[2] =
word_sub(x[2], y[2], &carry);
426 z[3] =
word_sub(x[3], y[3], &carry);
427 z[4] =
word_sub(x[4], y[4], &carry);
428 z[5] =
word_sub(x[5], y[5], &carry);
429 z[6] =
word_sub(x[6], y[6], &carry);
430 z[7] =
word_sub(x[7], y[7], &carry);
438inline constexpr auto word8_linmul3(W z[8],
const W x[8], W y, W carry) -> W {
439#if defined(BOTAN_MP_USE_X86_64_ASM)
440 if constexpr(std::same_as<W, uint64_t>) {
441 if(!std::is_constant_evaluated()) {
442 asm volatile(DO_8_TIMES(LINMUL_OP,
"z")
443 : [carry]
"=r"(carry)
444 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
445 :
"cc",
"%rax",
"%rdx",
"memory");
467inline constexpr auto word8_madd3(W z[8],
const W x[8], W y, W carry) -> W {
468#if defined(BOTAN_MP_USE_X86_64_ASM)
469 if constexpr(std::same_as<W, uint64_t>) {
470 if(!std::is_constant_evaluated()) {
471 asm volatile(DO_8_TIMES(MULADD_OP,
"")
472 : [carry]
"=r"(carry)
473 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
474 :
"cc",
"%rax",
"%rdx",
"memory");
502#if defined(__BITINT_MAXWIDTH__) && (__BITINT_MAXWIDTH__ >= 3 * 64)
505 constexpr word3() : m_w(0) {}
507 inline constexpr void mul(W x, W y) { m_w +=
static_cast<W3
>(x) * y; }
509 inline constexpr void mul_x2(W x, W y) { m_w +=
static_cast<W3
>(x) * y * 2; }
511 inline constexpr void add(W x) { m_w += x; }
514 W r =
static_cast<W
>(m_w);
519 inline constexpr W
monty_step(W p0, W p_dash) {
520 const W w0 =
static_cast<W
>(m_w);
521 const W r = w0 * p_dash;
528 const W r =
static_cast<W
>(m_w);
530 m_w +=
static_cast<W3
>(r);
540 constexpr word3() : m_w0(0), m_w1(0), m_w2(0) {}
542 inline constexpr void mul(W x, W y) {
543 #if defined(BOTAN_MP_USE_X86_64_ASM)
544 if constexpr(std::same_as<W, uint64_t>) {
545 if(!std::is_constant_evaluated()) {
549 asm(
"mulq %[y]" :
"=a"(z0),
"=d"(z1) :
"a"(x), [y]
"rm"(y) :
"cc");
556 : [w0] "=r"(m_w0), [w1]
"=r"(m_w1), [w2]
"=r"(m_w2)
557 : [z0]
"r"(z0), [z1]
"r"(z1),
"0"(m_w0),
"1"(m_w1),
"2"(m_w2)
562 #elif defined(BOTAN_MP_USE_AARCH64_ASM)
563 if constexpr(std::same_as<W, uint64_t>) {
564 if(!std::is_constant_evaluated()) {
568 mul %[t0], %[x], %[y]
569 umulh %[t1], %[x], %[y]
570 adds %[w0], %[w0], %[t0]
571 adcs %[w1], %[w1], %[t1]
572 adc %[w2], %[w2], xzr
574 : [w0] "+r"(m_w0), [w1]
"+r"(m_w1), [w2]
"+r"(m_w2), [t0]
"=&r"(t0), [t1]
"=&r"(t1)
575 : [x]
"r"(x), [y]
"r"(y)
583 const auto z = dword(x) * y;
584 const auto z0 =
static_cast<W
>(z);
594 #if defined(BOTAN_MP_USE_X86_64_ASM)
595 if constexpr(std::same_as<W, uint64_t>) {
596 if(!std::is_constant_evaluated()) {
600 asm(
"mulq %[y]" :
"=a"(z0),
"=d"(z1) :
"a"(x), [y]
"rm"(y) :
"cc");
611 : [w0] "=r"(m_w0), [w1]
"=r"(m_w1), [w2]
"=r"(m_w2)
612 : [z0]
"r"(z0), [z1]
"r"(z1),
"0"(m_w0),
"1"(m_w1),
"2"(m_w2)
617 #elif defined(BOTAN_MP_USE_AARCH64_ASM)
618 if constexpr(std::same_as<W, uint64_t>) {
619 if(!std::is_constant_evaluated()) {
623 mul %[t0], %[x], %[y]
624 umulh %[t1], %[x], %[y]
625 adds %[w0], %[w0], %[t0]
626 adcs %[w1], %[w1], %[t1]
627 adc %[w2], %[w2], xzr
628 adds %[w0], %[w0], %[t0]
629 adcs %[w1], %[w1], %[t1]
630 adc %[w2], %[w2], xzr
632 : [w0] "+r"(m_w0), [w1]
"+r"(m_w1), [w2]
"+r"(m_w2), [t0]
"=&r"(t0), [t1]
"=&r"(t1)
633 : [x]
"r"(x), [y]
"r"(y)
641 const auto z = dword(x) * y;
642 const auto z0 =
static_cast<W
>(z);
656 inline constexpr void add(W x) {
686 m_w1 = m_w2 + (m_w0 < m_w1);
701 #undef ADD_OR_SUBTRACT
constexpr W monty_step(W p0, W p_dash)
constexpr void mul(W x, W y)
constexpr W monty_step_pdash1()
constexpr void mul_x2(W x, W y)
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
constexpr auto word_sub(W x, W y, W *carry) -> W
constexpr auto word_add(W x, W y, W *carry) -> W
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
constexpr auto word_madd2(W a, W b, W *c) -> W
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
void unpoison_asm_output(const W *ptr)
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
static const constexpr bool dword_is_native
static const constexpr uint32_t top_bit
static const constexpr size_t bytes
static const constexpr size_t bits
static const constexpr uint32_t max
static const constexpr size_t bytes
static const constexpr bool dword_is_native
static const constexpr size_t bits
static const constexpr uint64_t max
static const constexpr uint64_t top_bit