7#include <botan/internal/mp_mulx.h>
9#include <botan/mem_ops.h>
10#include <botan/internal/mp_core.h>
25#define BOTAN_MULADD_STEP(D) \
26 "movq " D "(%[t]), %[tmp]\n\t" \
27 "adox %[carry], %[tmp]\n\t" \
28 "mulx " D "(%[x]), %[lo], %[carry]\n\t" \
29 "adcx %[lo], %[tmp]\n\t" \
30 "movq %[tmp], " D "(%[out])\n\t"
37#define BOTAN_MULADD_ASM(STEPS) \
38 asm volatile("xorl %k[zero], %k[zero]\n\t" \
40 "adox %[zero], %[carry]\n\t" \
41 "adcx %[zero], %[carry]\n\t" \
42 : [carry] "+&r"(carry), [tmp] "=&r"(tmp), [lo] "=&r"(lo), [zero] "=&r"(zero) \
43 : [out] "r"(out), [t] "r"(t), [x] "r"(x), "d"(b) \
46#define BOTAN_MULADD_STEPS4 \
47 BOTAN_MULADD_STEP("0") \
48 BOTAN_MULADD_STEP("8") \
49 BOTAN_MULADD_STEP("16") \
50 BOTAN_MULADD_STEP("24")
52#define BOTAN_MULADD_STEPS8 \
54 BOTAN_MULADD_STEP("32") \
55 BOTAN_MULADD_STEP("40") \
56 BOTAN_MULADD_STEP("48") \
57 BOTAN_MULADD_STEP("56")
89 for(
size_t i = N / 8; i != 0; --i) {
90 carry = muladd8(out, t, x, b, carry);
96 carry = muladd4(out, t, x, b, carry);
101 for(
size_t i = N % 4; i != 0; --i) {
106 out[0] =
word_add(t[0], carry, &c2);
118 for(
size_t i = 0; i != N; ++i) {
119 monty_row(t, t, x, N, y[i], 0);
120 const word m = t[0] * p_dash;
121 monty_row(t - 1, t, p, N, m, 0);
143 const size_t lead = N % 8;
145 for(
size_t i = 0; i != lead; ++i) {
146 monty_row(z + 2 * i + 1, z + 2 * i + 1, x + i + 1, N - i - 1, x[i], 0);
149 for(
size_t b = lead; b + 8 < N; b += 8) {
150 for(
size_t i = b; i != b + 8; ++i) {
151 monty_row(z + i + b + 8, z + i + b + 8, x + b + 8, N - b - 8, x[i], 0);
158 auto double_and_add = [&](
size_t k,
word s) {
160 z[k] =
word_add((w << 1) | top, s, &carry);
164 for(
size_t i = 0; i != lead; ++i) {
167 double_and_add(2 * i, lo);
168 double_and_add(2 * i + 1, hi);
171 for(
size_t b = lead; b != N; b += 8) {
172 std::array<word, 16> sq{};
174 for(
size_t k = 0; k != 16; ++k) {
175 double_and_add(2 * b + k, sq[k]);
187 for(
size_t i = 0; i != N; ++i) {
188 const word m = z[i] * p_dash;
189 c = monty_row(z + i, z + i, p, N, m, c);
199#undef BOTAN_MULADD_STEP
200#undef BOTAN_MULADD_ASM
201#undef BOTAN_MULADD_STEPS4
202#undef BOTAN_MULADD_STEPS8
#define BOTAN_DEBUG_ASSERT(expr)
#define BOTAN_FORCE_INLINE
#define BOTAN_MULADD_ASM(STEPS)
#define BOTAN_MULADD_STEPS8
#define BOTAN_MULADD_STEPS4
void bigint_mulx_monty_mul(word z[], const word x[], const word y[], const word p[], size_t N, word p_dash, word ws[])
constexpr auto word_add(W x, W y, W *carry) -> W
void bigint_mulx_monty_redc(word r[], word z[], const word p[], size_t N, word p_dash)
constexpr auto word_madd2(W a, W b, W *c) -> W
void bigint_mulx_sqr(word z[], const word x[], size_t N)
constexpr void bigint_monty_maybe_sub(size_t N, W z[], W x0, const W x[], const W p[])
void bigint_mulx_monty_sqr(word z[], const word x[], const word p[], size_t N, word p_dash, word ws[])
std::conditional_t< HasNative64BitRegisters, std::uint64_t, uint32_t > word
The native machine word, used as the limb type for multiprecision integers.
void bigint_mulx_sqr8(word z[16], const word x[8])
constexpr void clear_mem(T *ptr, size_t n)
void unpoison_asm_output(const W *ptr)
constexpr auto word_madd3(W a, W b, W c, W *d) -> W