Botan 3.14.1
Crypto and TLS for C&
mp_asmi.h
Go to the documentation of this file.
1/*
2* Lowest Level MPI Algorithms
3* (C) 1999-2010,2025 Jack Lloyd
4* 2006 Luca Piccarreta
5*
6* Botan is released under the Simplified BSD License (see license.txt)
7*/
8
9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
11
12#include <botan/compiler.h>
13#include <botan/types.h>
14#include <botan/internal/target_info.h>
15#include <concepts>
16
17#if defined(__has_feature)
18 #if __has_feature(memory_sanitizer)
19 #define BOTAN_MP_ASM_HAS_MSAN
20 #include <sanitizer/msan_interface.h>
21 #endif
22#endif
23
24#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
25 #include <botan/internal/donna128.h>
26#endif
27
28namespace Botan {
29
30// NOLINTBEGIN(*-macro-usage,*-no-assembler)
31
32#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_X86_64)
33 #define BOTAN_MP_USE_X86_64_ASM
34#endif
35
36#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_ARM64)
37 #define BOTAN_MP_USE_AARCH64_ASM
38#endif
39
40/*
41* Expressing an add with carry is sadly quite difficult in standard C/C++.
42*
43* Compilers will recognize various idioms and generate a reasonable carry
44* chain. Unfortunately which idioms the compiler will understand vary, so we
45* have to decide what to do based on the compiler. This is fragile; what will
46* work varies not just based on compiler but also version, target architecture,
47* and optimization flags.
48*/
49#if defined(__clang__)
50static constexpr bool use_dword_for_word_add = false;
51#else
52static constexpr bool use_dword_for_word_add = true;
53#endif
54
55/*
56* Concept for allowed multiprecision word types
57*/
58template <typename T>
59concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
60
61template <WordType W>
62struct WordInfo {};
63
64template <>
65struct WordInfo<uint32_t> {
66 public:
67 static const constexpr size_t bytes = 4;
68 static const constexpr size_t bits = 32;
69 static const constexpr uint32_t max = 0xFFFFFFFF;
70 static const constexpr uint32_t top_bit = 0x80000000;
71
72 typedef uint64_t dword;
73 static const constexpr bool dword_is_native = true;
74};
75
76template <>
77struct WordInfo<uint64_t> {
78 public:
79 static const constexpr size_t bytes = 8;
80 static const constexpr size_t bits = 64;
81 static const constexpr uint64_t max = 0xFFFFFFFFFFFFFFFF;
82 static const constexpr uint64_t top_bit = 0x8000000000000000;
83
84#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
85 typedef uint128_t dword;
86 static const constexpr bool dword_is_native = true;
87#else
88 typedef donna128 dword;
89 static const constexpr bool dword_is_native = false;
90#endif
91};
92
93/**
94* Inform MemorySanitizer that inline asm has written the K words at ptr
95*
96* MSan does not see the stores an asm block makes through a pointer operand,
97* so memory that was uninitialized before the asm would otherwise still be
98* considered uninitialized afterwards.
99*/
100template <size_t K, WordType W>
101inline void unpoison_asm_output([[maybe_unused]] const W* ptr) {
102#if defined(BOTAN_MP_ASM_HAS_MSAN)
103 __msan_unpoison(ptr, K * sizeof(W));
104#endif
105}
106
107/*
108* Word Multiply/Add
109*/
110template <WordType W>
111inline constexpr auto word_madd2(W a, W b, W* c) -> W {
112#if defined(BOTAN_MP_USE_X86_64_ASM)
113 if constexpr(std::same_as<W, uint64_t>) {
114 if(!std::is_constant_evaluated()) {
115 asm(R"(
116 mulq %[b]
117 addq %[c],%[a]
118 adcq $0,%[carry]
119 )"
120 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c)
121 : "0"(a), "1"(b), [c] "g"(*c)
122 : "cc");
123
124 return a;
125 }
126 }
127#elif defined(BOTAN_MP_USE_AARCH64_ASM)
128 if constexpr(std::same_as<W, uint64_t>) {
129 if(!std::is_constant_evaluated()) {
130 W lo = 0;
131 W hi = 0;
132 asm(R"(
133 mul %[lo], %[a], %[b]
134 umulh %[hi], %[a], %[b]
135 adds %[lo], %[lo], %[c]
136 adc %[hi], %[hi], xzr
137 )"
138 : [lo] "=&r"(lo), [hi] "=&r"(hi)
139 : [a] "r"(a), [b] "r"(b), [c] "r"(*c)
140 : "cc");
141
142 *c = hi;
143 return lo;
144 }
145 }
146#endif
147
148 typedef typename WordInfo<W>::dword dword;
149 const dword s = dword(a) * b + *c;
150 *c = static_cast<W>(s >> WordInfo<W>::bits);
151 return static_cast<W>(s);
152}
153
154/*
155* Word Multiply/Add
156*/
157template <WordType W>
158inline constexpr auto word_madd3(W a, W b, W c, W* d) -> W {
159#if defined(BOTAN_MP_USE_X86_64_ASM)
160 if constexpr(std::same_as<W, uint64_t>) {
161 if(!std::is_constant_evaluated()) {
162 asm(R"(
163 mulq %[b]
164
165 addq %[c],%[a]
166 adcq $0,%[carry]
167
168 addq %[d],%[a]
169 adcq $0,%[carry]
170 )"
171 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d)
172 : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d)
173 : "cc");
174
175 return a;
176 }
177 }
178#elif defined(BOTAN_MP_USE_AARCH64_ASM)
179 if constexpr(std::same_as<W, uint64_t>) {
180 if(!std::is_constant_evaluated()) {
181 W lo = 0;
182 W hi = 0;
183 asm(R"(
184 mul %[lo], %[a], %[b]
185 umulh %[hi], %[a], %[b]
186 adds %[lo], %[lo], %[c]
187 adc %[hi], %[hi], xzr
188 adds %[lo], %[lo], %[d]
189 adc %[hi], %[hi], xzr
190 )"
191 : [lo] "=&r"(lo), [hi] "=&r"(hi)
192 : [a] "r"(a), [b] "r"(b), [c] "r"(c), [d] "r"(*d)
193 : "cc");
194
195 *d = hi;
196 return lo;
197 }
198 }
199#endif
200
201 typedef typename WordInfo<W>::dword dword;
202 const dword s = dword(a) * b + c + *d;
203 *d = static_cast<W>(s >> WordInfo<W>::bits);
204 return static_cast<W>(s);
205}
206
207#if defined(BOTAN_MP_USE_X86_64_ASM)
208
209 #define ASM(x) x "\n\t"
210
211 #define DO_8_TIMES(MACRO, ARG) \
212 MACRO(ARG, 0) \
213 MACRO(ARG, 1) \
214 MACRO(ARG, 2) \
215 MACRO(ARG, 3) \
216 MACRO(ARG, 4) \
217 MACRO(ARG, 5) \
218 MACRO(ARG, 6) \
219 MACRO(ARG, 7)
220
221 #define ADDSUB2_OP(OPERATION, INDEX) \
222 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
223 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
224
225 #define ADDSUB3_OP(OPERATION, INDEX) \
226 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
227 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
228 ASM("movq %[carry], 8*" #INDEX "(%[z])")
229
230 #define LINMUL_OP(WRITE_TO, INDEX) \
231 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
232 ASM("mulq %[y]") \
233 ASM("addq %[carry],%%rax") \
234 ASM("adcq $0,%%rdx") \
235 ASM("movq %%rdx,%[carry]") \
236 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
237
238 #define MULADD_OP(IGNORED, INDEX) \
239 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
240 ASM("mulq %[y]") \
241 ASM("addq %[carry],%%rax") \
242 ASM("adcq $0,%%rdx") \
243 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
244 ASM("adcq $0,%%rdx") \
245 ASM("movq %%rdx,%[carry]") \
246 ASM("movq %%rax, 8*" #INDEX " (%[z])")
247
248 #define ADD_OR_SUBTRACT(CORE_CODE) \
249 ASM("rorq %[carry]") \
250 CORE_CODE \
251 ASM("sbbq %[carry],%[carry]") \
252 ASM("negq %[carry]")
253
254#endif
255
256/*
257* Word Addition
258*/
259template <WordType W>
260inline constexpr auto word_add(W x, W y, W* carry) -> W {
261#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_addc)
262 if(!std::is_constant_evaluated()) {
263 if constexpr(std::same_as<W, unsigned int>) {
264 return __builtin_addc(x, y, *carry & 1, carry);
265 } else if constexpr(std::same_as<W, unsigned long>) {
266 return __builtin_addcl(x, y, *carry & 1, carry);
267 } else if constexpr(std::same_as<W, unsigned long long>) {
268 return __builtin_addcll(x, y, *carry & 1, carry);
269 }
270 }
271#endif
272
273 if constexpr(WordInfo<W>::dword_is_native && use_dword_for_word_add) {
274 /*
275 TODO(Botan4) this is largely a performance hack for GCCs that don't
276 support __builtin_addc, if we increase the minimum supported version of
277 GCC to GCC 14 then we can remove this and not worry about it
278 */
279 const W cb = *carry & 1;
280 const auto s = typename WordInfo<W>::dword(x) + y + cb;
281 *carry = static_cast<W>(s >> WordInfo<W>::bits);
282 return static_cast<W>(s);
283 } else {
284 const W cb = *carry & 1;
285 W z = x + y;
286 W c1 = (z < x);
287 z += cb;
288 *carry = c1 | (z < cb);
289 return z;
290 }
291}
292
293/*
294* Eight Word Block Addition, Two Argument
295*/
296template <WordType W>
297inline constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W {
298#if defined(BOTAN_MP_USE_X86_64_ASM)
299 if constexpr(std::same_as<W, uint64_t>) {
300 if(!std::is_constant_evaluated()) {
301 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq"))
302 : [carry] "=r"(carry)
303 : [x] "r"(x), [y] "r"(y), "0"(carry)
304 : "cc", "memory");
306 return carry;
307 }
308 }
309#endif
310
311 x[0] = word_add(x[0], y[0], &carry);
312 x[1] = word_add(x[1], y[1], &carry);
313 x[2] = word_add(x[2], y[2], &carry);
314 x[3] = word_add(x[3], y[3], &carry);
315 x[4] = word_add(x[4], y[4], &carry);
316 x[5] = word_add(x[5], y[5], &carry);
317 x[6] = word_add(x[6], y[6], &carry);
318 x[7] = word_add(x[7], y[7], &carry);
319 return carry;
320}
321
322/*
323* Eight Word Block Addition, Three Argument
324*/
325template <WordType W>
326inline constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W {
327#if defined(BOTAN_MP_USE_X86_64_ASM)
328 if constexpr(std::same_as<W, uint64_t>) {
329 if(!std::is_constant_evaluated()) {
330 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq"))
331 : [carry] "=r"(carry)
332 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
333 : "cc", "memory");
335 return carry;
336 }
337 }
338#endif
339
340 z[0] = word_add(x[0], y[0], &carry);
341 z[1] = word_add(x[1], y[1], &carry);
342 z[2] = word_add(x[2], y[2], &carry);
343 z[3] = word_add(x[3], y[3], &carry);
344 z[4] = word_add(x[4], y[4], &carry);
345 z[5] = word_add(x[5], y[5], &carry);
346 z[6] = word_add(x[6], y[6], &carry);
347 z[7] = word_add(x[7], y[7], &carry);
348 return carry;
349}
350
351/*
352* Word Subtraction
353*/
354template <WordType W>
355inline constexpr auto word_sub(W x, W y, W* carry) -> W {
356#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_subc)
357 if(!std::is_constant_evaluated()) {
358 if constexpr(std::same_as<W, unsigned int>) {
359 return __builtin_subc(x, y, *carry & 1, carry);
360 } else if constexpr(std::same_as<W, unsigned long>) {
361 return __builtin_subcl(x, y, *carry & 1, carry);
362 } else if constexpr(std::same_as<W, unsigned long long>) {
363 return __builtin_subcll(x, y, *carry & 1, carry);
364 }
365 }
366#endif
367
368 const W cb = *carry & 1;
369 W t0 = x - y;
370 W c1 = (t0 > x);
371 W z = t0 - cb;
372 *carry = c1 | (z > t0);
373 return z;
374}
375
376/*
377* Eight Word Block Subtraction, Two Argument
378*/
379template <WordType W>
380inline constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W {
381#if defined(BOTAN_MP_USE_X86_64_ASM)
382 if constexpr(std::same_as<W, uint64_t>) {
383 if(!std::is_constant_evaluated()) {
384 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq"))
385 : [carry] "=r"(carry)
386 : [x] "r"(x), [y] "r"(y), "0"(carry)
387 : "cc", "memory");
389 return carry;
390 }
391 }
392#endif
393
394 x[0] = word_sub(x[0], y[0], &carry);
395 x[1] = word_sub(x[1], y[1], &carry);
396 x[2] = word_sub(x[2], y[2], &carry);
397 x[3] = word_sub(x[3], y[3], &carry);
398 x[4] = word_sub(x[4], y[4], &carry);
399 x[5] = word_sub(x[5], y[5], &carry);
400 x[6] = word_sub(x[6], y[6], &carry);
401 x[7] = word_sub(x[7], y[7], &carry);
402 return carry;
403}
404
405/*
406* Eight Word Block Subtraction, Three Argument
407*/
408template <WordType W>
409inline constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W {
410#if defined(BOTAN_MP_USE_X86_64_ASM)
411 if constexpr(std::same_as<W, uint64_t>) {
412 if(!std::is_constant_evaluated()) {
413 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
414 : [carry] "=r"(carry)
415 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
416 : "cc", "memory");
418 return carry;
419 }
420 }
421#endif
422
423 z[0] = word_sub(x[0], y[0], &carry);
424 z[1] = word_sub(x[1], y[1], &carry);
425 z[2] = word_sub(x[2], y[2], &carry);
426 z[3] = word_sub(x[3], y[3], &carry);
427 z[4] = word_sub(x[4], y[4], &carry);
428 z[5] = word_sub(x[5], y[5], &carry);
429 z[6] = word_sub(x[6], y[6], &carry);
430 z[7] = word_sub(x[7], y[7], &carry);
431 return carry;
432}
433
434/*
435* Eight Word Block Linear Multiplication
436*/
437template <WordType W>
438inline constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W {
439#if defined(BOTAN_MP_USE_X86_64_ASM)
440 if constexpr(std::same_as<W, uint64_t>) {
441 if(!std::is_constant_evaluated()) {
442 asm volatile(DO_8_TIMES(LINMUL_OP, "z")
443 : [carry] "=r"(carry)
444 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
445 : "cc", "%rax", "%rdx", "memory");
447 return carry;
448 }
449 }
450#endif
451
452 z[0] = word_madd2(x[0], y, &carry);
453 z[1] = word_madd2(x[1], y, &carry);
454 z[2] = word_madd2(x[2], y, &carry);
455 z[3] = word_madd2(x[3], y, &carry);
456 z[4] = word_madd2(x[4], y, &carry);
457 z[5] = word_madd2(x[5], y, &carry);
458 z[6] = word_madd2(x[6], y, &carry);
459 z[7] = word_madd2(x[7], y, &carry);
460 return carry;
461}
462
463/*
464* Eight Word Block Multiply/Add
465*/
466template <WordType W>
467inline constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W {
468#if defined(BOTAN_MP_USE_X86_64_ASM)
469 if constexpr(std::same_as<W, uint64_t>) {
470 if(!std::is_constant_evaluated()) {
471 asm volatile(DO_8_TIMES(MULADD_OP, "")
472 : [carry] "=r"(carry)
473 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
474 : "cc", "%rax", "%rdx", "memory");
476 return carry;
477 }
478 }
479#endif
480
481 z[0] = word_madd3(x[0], y, z[0], &carry);
482 z[1] = word_madd3(x[1], y, z[1], &carry);
483 z[2] = word_madd3(x[2], y, z[2], &carry);
484 z[3] = word_madd3(x[3], y, z[3], &carry);
485 z[4] = word_madd3(x[4], y, z[4], &carry);
486 z[5] = word_madd3(x[5], y, z[5], &carry);
487 z[6] = word_madd3(x[6], y, z[6], &carry);
488 z[7] = word_madd3(x[7], y, z[7], &carry);
489 return carry;
490}
491
492/**
493* Helper for 3-word accumulators
494*
495* A number of algorithms especially Comba multiplication and
496* Montgomery reduction can take advantage of wide accumulators, which
497* consume inputs via addition with outputs extracted from the low
498* bits.
499*/
500template <WordType W>
501class word3 final {
502#if defined(__BITINT_MAXWIDTH__) && (__BITINT_MAXWIDTH__ >= 3 * 64)
503
504 public:
505 constexpr word3() : m_w(0) {}
506
507 inline constexpr void mul(W x, W y) { m_w += static_cast<W3>(x) * y; }
508
509 inline constexpr void mul_x2(W x, W y) { m_w += static_cast<W3>(x) * y * 2; }
510
511 inline constexpr void add(W x) { m_w += x; }
512
513 inline constexpr W extract() {
514 W r = static_cast<W>(m_w);
515 m_w >>= WordInfo<W>::bits;
516 return r;
517 }
518
519 inline constexpr W monty_step(W p0, W p_dash) {
520 const W w0 = static_cast<W>(m_w);
521 const W r = w0 * p_dash;
522 mul(r, p0);
523 m_w >>= WordInfo<W>::bits;
524 return r;
525 }
526
527 inline constexpr W monty_step_pdash1() {
528 const W r = static_cast<W>(m_w);
529 m_w >>= WordInfo<W>::bits;
530 m_w += static_cast<W3>(r);
531 return r;
532 }
533
534 private:
535 __extension__ typedef unsigned _BitInt(WordInfo<W>::bits * 3) W3;
536 W3 m_w;
537#else
538
539 public:
540 constexpr word3() : m_w0(0), m_w1(0), m_w2(0) {}
541
542 inline constexpr void mul(W x, W y) {
543 #if defined(BOTAN_MP_USE_X86_64_ASM)
544 if constexpr(std::same_as<W, uint64_t>) {
545 if(!std::is_constant_evaluated()) {
546 W z0 = 0;
547 W z1 = 0;
548
549 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
550
551 asm(R"(
552 addq %[z0],%[w0]
553 adcq %[z1],%[w1]
554 adcq $0,%[w2]
555 )"
556 : [w0] "=r"(m_w0), [w1] "=r"(m_w1), [w2] "=r"(m_w2)
557 : [z0] "r"(z0), [z1] "r"(z1), "0"(m_w0), "1"(m_w1), "2"(m_w2)
558 : "cc");
559 return;
560 }
561 }
562 #elif defined(BOTAN_MP_USE_AARCH64_ASM)
563 if constexpr(std::same_as<W, uint64_t>) {
564 if(!std::is_constant_evaluated()) {
565 W t0 = 0;
566 W t1 = 0;
567 asm(R"(
568 mul %[t0], %[x], %[y]
569 umulh %[t1], %[x], %[y]
570 adds %[w0], %[w0], %[t0]
571 adcs %[w1], %[w1], %[t1]
572 adc %[w2], %[w2], xzr
573 )"
574 : [w0] "+r"(m_w0), [w1] "+r"(m_w1), [w2] "+r"(m_w2), [t0] "=&r"(t0), [t1] "=&r"(t1)
575 : [x] "r"(x), [y] "r"(y)
576 : "cc");
577 return;
578 }
579 }
580 #endif
581
582 typedef typename WordInfo<W>::dword dword;
583 const auto z = dword(x) * y;
584 const auto z0 = static_cast<W>(z);
585 const auto z1 = static_cast<W>(z >> WordInfo<W>::bits);
586
587 W carry = 0;
588 m_w0 = word_add(m_w0, z0, &carry);
589 m_w1 = word_add(m_w1, z1, &carry);
590 m_w2 += carry;
591 }
592
593 inline constexpr void mul_x2(W x, W y) {
594 #if defined(BOTAN_MP_USE_X86_64_ASM)
595 if constexpr(std::same_as<W, uint64_t>) {
596 if(!std::is_constant_evaluated()) {
597 W z0 = 0;
598 W z1 = 0;
599
600 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
601
602 asm(R"(
603 addq %[z0],%[w0]
604 adcq %[z1],%[w1]
605 adcq $0,%[w2]
606
607 addq %[z0],%[w0]
608 adcq %[z1],%[w1]
609 adcq $0,%[w2]
610 )"
611 : [w0] "=r"(m_w0), [w1] "=r"(m_w1), [w2] "=r"(m_w2)
612 : [z0] "r"(z0), [z1] "r"(z1), "0"(m_w0), "1"(m_w1), "2"(m_w2)
613 : "cc");
614 return;
615 }
616 }
617 #elif defined(BOTAN_MP_USE_AARCH64_ASM)
618 if constexpr(std::same_as<W, uint64_t>) {
619 if(!std::is_constant_evaluated()) {
620 W t0 = 0;
621 W t1 = 0;
622 asm(R"(
623 mul %[t0], %[x], %[y]
624 umulh %[t1], %[x], %[y]
625 adds %[w0], %[w0], %[t0]
626 adcs %[w1], %[w1], %[t1]
627 adc %[w2], %[w2], xzr
628 adds %[w0], %[w0], %[t0]
629 adcs %[w1], %[w1], %[t1]
630 adc %[w2], %[w2], xzr
631 )"
632 : [w0] "+r"(m_w0), [w1] "+r"(m_w1), [w2] "+r"(m_w2), [t0] "=&r"(t0), [t1] "=&r"(t1)
633 : [x] "r"(x), [y] "r"(y)
634 : "cc");
635 return;
636 }
637 }
638 #endif
639
640 typedef typename WordInfo<W>::dword dword;
641 const auto z = dword(x) * y;
642 const auto z0 = static_cast<W>(z);
643 const auto z1 = static_cast<W>(z >> WordInfo<W>::bits);
644
645 W carry = 0;
646 m_w0 = word_add(m_w0, z0, &carry);
647 m_w1 = word_add(m_w1, z1, &carry);
648 m_w2 += carry;
649
650 carry = 0;
651 m_w0 = word_add(m_w0, z0, &carry);
652 m_w1 = word_add(m_w1, z1, &carry);
653 m_w2 += carry;
654 }
655
656 inline constexpr void add(W x) {
657 constexpr W z = 0;
658
659 W carry = 0;
660 m_w0 = word_add(m_w0, x, &carry);
661 m_w1 = word_add(m_w1, z, &carry);
662 m_w2 += carry;
663 }
664
665 inline constexpr W extract() {
666 W r = m_w0;
667 m_w0 = m_w1;
668 m_w1 = m_w2;
669 m_w2 = 0;
670 return r;
671 }
672
673 inline constexpr W monty_step(W p0, W p_dash) {
674 W r = m_w0 * p_dash;
675 mul(r, p0);
676 m_w0 = m_w1;
677 m_w1 = m_w2;
678 m_w2 = 0;
679 return r;
680 }
681
682 inline constexpr W monty_step_pdash1() {
683 // If p_dash == 1 then p[0] = -1 and everything simplifies
684 const W r = m_w0;
685 m_w0 += m_w1;
686 m_w1 = m_w2 + (m_w0 < m_w1);
687 m_w2 = 0;
688 return r;
689 }
690
691 private:
692 W m_w0;
693 W m_w1;
694 W m_w2;
695#endif
696};
697
698#if defined(ASM)
699 #undef ASM
700 #undef DO_8_TIMES
701 #undef ADD_OR_SUBTRACT
702 #undef ADDSUB2_OP
703 #undef ADDSUB3_OP
704 #undef LINMUL_OP
705 #undef MULADD_OP
706#endif
707
708// NOLINTEND(*-macro-usage,*-no-assembler)
709
710} // namespace Botan
711
712#endif
constexpr void add(W x)
Definition mp_asmi.h:656
constexpr W monty_step(W p0, W p_dash)
Definition mp_asmi.h:673
constexpr word3()
Definition mp_asmi.h:540
constexpr W extract()
Definition mp_asmi.h:665
constexpr void mul(W x, W y)
Definition mp_asmi.h:542
constexpr W monty_step_pdash1()
Definition mp_asmi.h:682
constexpr void mul_x2(W x, W y)
Definition mp_asmi.h:593
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:409
constexpr auto word_sub(W x, W y, W *carry) -> W
Definition mp_asmi.h:355
constexpr auto word_add(W x, W y, W *carry) -> W
Definition mp_asmi.h:260
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:467
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:326
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:380
constexpr auto word_madd2(W a, W b, W *c) -> W
Definition mp_asmi.h:111
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:297
donna128 uint128_t
Definition x25519_fe.h:37
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:438
void unpoison_asm_output(const W *ptr)
Definition mp_asmi.h:101
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
Definition mp_asmi.h:158
static const constexpr bool dword_is_native
Definition mp_asmi.h:73
static const constexpr uint32_t top_bit
Definition mp_asmi.h:70
static const constexpr size_t bytes
Definition mp_asmi.h:67
static const constexpr size_t bits
Definition mp_asmi.h:68
static const constexpr uint32_t max
Definition mp_asmi.h:69
static const constexpr size_t bytes
Definition mp_asmi.h:79
static const constexpr bool dword_is_native
Definition mp_asmi.h:89
static const constexpr size_t bits
Definition mp_asmi.h:80
static const constexpr uint64_t max
Definition mp_asmi.h:81
static const constexpr uint64_t top_bit
Definition mp_asmi.h:82