Botan 3.6.1
Crypto and TLS for C&
mp_asmi.h
Go to the documentation of this file.
1/*
2* Lowest Level MPI Algorithms
3* (C) 1999-2010 Jack Lloyd
4* 2006 Luca Piccarreta
5*
6* Botan is released under the Simplified BSD License (see license.txt)
7*/
8
9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
11
12#include <botan/types.h>
13
14#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
15 #include <botan/internal/donna128.h>
16#endif
17
18namespace Botan {
19
20#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_X86_64)
21 #define BOTAN_MP_USE_X86_64_ASM
22#endif
23
24/*
25* Concept for allowed multiprecision word types
26*/
27template <typename T>
28concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
29
30template <WordType W>
31struct WordInfo {};
32
33template <>
34struct WordInfo<uint32_t> {
35 public:
36 static const constexpr size_t bytes = 4;
37 static const constexpr size_t bits = 32;
38 static const constexpr uint32_t max = 0xFFFFFFFF;
39 static const constexpr uint32_t top_bit = 0x80000000;
40
41 typedef uint64_t dword;
42 static const constexpr bool dword_is_native = true;
43};
44
45template <>
46struct WordInfo<uint64_t> {
47 public:
48 static const constexpr size_t bytes = 8;
49 static const constexpr size_t bits = 64;
50 static const constexpr uint64_t max = 0xFFFFFFFFFFFFFFFF;
51 static const constexpr uint64_t top_bit = 0x8000000000000000;
52
53#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
54 typedef uint128_t dword;
55 static const constexpr bool dword_is_native = true;
56#else
57 typedef donna128 dword;
58 static const constexpr bool dword_is_native = false;
59#endif
60};
61
62/*
63* Word Multiply/Add
64*/
65template <WordType W>
66inline constexpr auto word_madd2(W a, W b, W* c) -> W {
67#if defined(BOTAN_MP_USE_X86_64_ASM)
68 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
69 asm(R"(
70 mulq %[b]
71 addq %[c],%[a]
72 adcq $0,%[carry]
73 )"
74 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c)
75 : "0"(a), "1"(b), [c] "g"(*c)
76 : "cc");
77
78 return a;
79 }
80#endif
81
82 typedef typename WordInfo<W>::dword dword;
83 const dword s = dword(a) * b + *c;
84 *c = static_cast<W>(s >> WordInfo<W>::bits);
85 return static_cast<W>(s);
86}
87
88/*
89* Word Multiply/Add
90*/
91template <WordType W>
92inline constexpr auto word_madd3(W a, W b, W c, W* d) -> W {
93#if defined(BOTAN_MP_USE_X86_64_ASM)
94 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
95 asm(R"(
96 mulq %[b]
97
98 addq %[c],%[a]
99 adcq $0,%[carry]
100
101 addq %[d],%[a]
102 adcq $0,%[carry]
103 )"
104 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d)
105 : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d)
106 : "cc");
107
108 return a;
109 }
110#endif
111
112 typedef typename WordInfo<W>::dword dword;
113 const dword s = dword(a) * b + c + *d;
114 *d = static_cast<W>(s >> WordInfo<W>::bits);
115 return static_cast<W>(s);
116}
117
118#if defined(BOTAN_MP_USE_X86_64_ASM)
119
120 #define ASM(x) x "\n\t"
121
122 #define DO_4_TIMES(MACRO, ARG) \
123 MACRO(ARG, 0) \
124 MACRO(ARG, 1) \
125 MACRO(ARG, 2) \
126 MACRO(ARG, 3)
127
128 #define DO_8_TIMES(MACRO, ARG) \
129 MACRO(ARG, 0) \
130 MACRO(ARG, 1) \
131 MACRO(ARG, 2) \
132 MACRO(ARG, 3) \
133 MACRO(ARG, 4) \
134 MACRO(ARG, 5) \
135 MACRO(ARG, 6) \
136 MACRO(ARG, 7)
137
138 #define ADDSUB2_OP(OPERATION, INDEX) \
139 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
140 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
141
142 #define ADDSUB3_OP(OPERATION, INDEX) \
143 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
144 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
145 ASM("movq %[carry], 8*" #INDEX "(%[z])")
146
147 #define LINMUL_OP(WRITE_TO, INDEX) \
148 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
149 ASM("mulq %[y]") \
150 ASM("addq %[carry],%%rax") \
151 ASM("adcq $0,%%rdx") \
152 ASM("movq %%rdx,%[carry]") \
153 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
154
155 #define MULADD_OP(IGNORED, INDEX) \
156 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
157 ASM("mulq %[y]") \
158 ASM("addq %[carry],%%rax") \
159 ASM("adcq $0,%%rdx") \
160 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
161 ASM("adcq $0,%%rdx") \
162 ASM("movq %%rdx,%[carry]") \
163 ASM("movq %%rax, 8*" #INDEX " (%[z])")
164
165 #define ADD_OR_SUBTRACT(CORE_CODE) \
166 ASM("rorq %[carry]") \
167 CORE_CODE \
168 ASM("sbbq %[carry],%[carry]") \
169 ASM("negq %[carry]")
170
171#endif
172
173/*
174* Word Addition
175*/
176template <WordType W>
177inline constexpr auto word_add(W x, W y, W* carry) -> W {
178 if(!std::is_constant_evaluated()) {
179#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_addc)
180 if constexpr(std::same_as<W, unsigned int>) {
181 return __builtin_addc(x, y, *carry & 1, carry);
182 } else if constexpr(std::same_as<W, unsigned long>) {
183 return __builtin_addcl(x, y, *carry & 1, carry);
184 } else if constexpr(std::same_as<W, unsigned long long>) {
185 return __builtin_addcll(x, y, *carry & 1, carry);
186 }
187#elif defined(BOTAN_MP_USE_X86_64_ASM)
188 if(std::same_as<W, uint64_t>) {
189 asm(ADD_OR_SUBTRACT(ASM("adcq %[y],%[x]"))
190 : [x] "=r"(x), [carry] "=r"(*carry)
191 : "0"(x), [y] "rm"(y), "1"(*carry)
192 : "cc");
193 return x;
194 }
195#endif
196 }
197
198 const W cb = *carry & 1;
199 W z = x + y;
200 W c1 = (z < x);
201 z += cb;
202 *carry = c1 | (z < cb);
203 return z;
204}
205
206/*
207* Eight Word Block Addition, Two Argument
208*/
209template <WordType W>
210inline constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W {
211#if defined(BOTAN_MP_USE_X86_64_ASM)
212 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
213 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq"))
214 : [carry] "=r"(carry)
215 : [x] "r"(x), [y] "r"(y), "0"(carry)
216 : "cc", "memory");
217 return carry;
218 }
219#endif
220
221 x[0] = word_add(x[0], y[0], &carry);
222 x[1] = word_add(x[1], y[1], &carry);
223 x[2] = word_add(x[2], y[2], &carry);
224 x[3] = word_add(x[3], y[3], &carry);
225 x[4] = word_add(x[4], y[4], &carry);
226 x[5] = word_add(x[5], y[5], &carry);
227 x[6] = word_add(x[6], y[6], &carry);
228 x[7] = word_add(x[7], y[7], &carry);
229 return carry;
230}
231
232/*
233* Eight Word Block Addition, Three Argument
234*/
235template <WordType W>
236inline constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W {
237#if defined(BOTAN_MP_USE_X86_64_ASM)
238 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
239 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq"))
240 : [carry] "=r"(carry)
241 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
242 : "cc", "memory");
243 return carry;
244 }
245#endif
246
247 z[0] = word_add(x[0], y[0], &carry);
248 z[1] = word_add(x[1], y[1], &carry);
249 z[2] = word_add(x[2], y[2], &carry);
250 z[3] = word_add(x[3], y[3], &carry);
251 z[4] = word_add(x[4], y[4], &carry);
252 z[5] = word_add(x[5], y[5], &carry);
253 z[6] = word_add(x[6], y[6], &carry);
254 z[7] = word_add(x[7], y[7], &carry);
255 return carry;
256}
257
258template <WordType W>
259inline constexpr auto word4_add3(W z[4], const W x[4], const W y[4], W carry) -> W {
260#if defined(BOTAN_MP_USE_X86_64_ASM)
261 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
262 asm volatile(ADD_OR_SUBTRACT(DO_4_TIMES(ADDSUB3_OP, "adcq"))
263 : [carry] "=r"(carry)
264 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
265 : "cc", "memory");
266 return carry;
267 }
268#endif
269
270 z[0] = word_add(x[0], y[0], &carry);
271 z[1] = word_add(x[1], y[1], &carry);
272 z[2] = word_add(x[2], y[2], &carry);
273 z[3] = word_add(x[3], y[3], &carry);
274 return carry;
275}
276
277/*
278* Word Subtraction
279*/
280template <WordType W>
281inline constexpr auto word_sub(W x, W y, W* carry) -> W {
282 if(!std::is_constant_evaluated()) {
283#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_subc)
284 if constexpr(std::same_as<W, unsigned int>) {
285 return __builtin_subc(x, y, *carry & 1, carry);
286 } else if constexpr(std::same_as<W, unsigned long>) {
287 return __builtin_subcl(x, y, *carry & 1, carry);
288 } else if constexpr(std::same_as<W, unsigned long long>) {
289 return __builtin_subcll(x, y, *carry & 1, carry);
290 }
291#elif defined(BOTAN_MP_USE_X86_64_ASM)
292 if(std::same_as<W, uint64_t>) {
293 asm(ADD_OR_SUBTRACT(ASM("sbbq %[y],%[x]"))
294 : [x] "=r"(x), [carry] "=r"(*carry)
295 : "0"(x), [y] "rm"(y), "1"(*carry)
296 : "cc");
297 return x;
298 }
299#endif
300 }
301
302 const W cb = *carry & 1;
303 W t0 = x - y;
304 W c1 = (t0 > x);
305 W z = t0 - cb;
306 *carry = c1 | (z > t0);
307 return z;
308}
309
310/*
311* Eight Word Block Subtraction, Two Argument
312*/
313template <WordType W>
314inline constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W {
315#if defined(BOTAN_MP_USE_X86_64_ASM)
316 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
317 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq"))
318 : [carry] "=r"(carry)
319 : [x] "r"(x), [y] "r"(y), "0"(carry)
320 : "cc", "memory");
321 return carry;
322 }
323#endif
324
325 x[0] = word_sub(x[0], y[0], &carry);
326 x[1] = word_sub(x[1], y[1], &carry);
327 x[2] = word_sub(x[2], y[2], &carry);
328 x[3] = word_sub(x[3], y[3], &carry);
329 x[4] = word_sub(x[4], y[4], &carry);
330 x[5] = word_sub(x[5], y[5], &carry);
331 x[6] = word_sub(x[6], y[6], &carry);
332 x[7] = word_sub(x[7], y[7], &carry);
333 return carry;
334}
335
336/*
337* Eight Word Block Subtraction, Two Argument
338*/
339template <WordType W>
340inline constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W {
341#if defined(BOTAN_MP_USE_X86_64_ASM)
342 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
343 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
344 : [carry] "=r"(carry)
345 : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry)
346 : "cc", "memory");
347 return carry;
348 }
349#endif
350
351 x[0] = word_sub(y[0], x[0], &carry);
352 x[1] = word_sub(y[1], x[1], &carry);
353 x[2] = word_sub(y[2], x[2], &carry);
354 x[3] = word_sub(y[3], x[3], &carry);
355 x[4] = word_sub(y[4], x[4], &carry);
356 x[5] = word_sub(y[5], x[5], &carry);
357 x[6] = word_sub(y[6], x[6], &carry);
358 x[7] = word_sub(y[7], x[7], &carry);
359 return carry;
360}
361
362/*
363* Eight Word Block Subtraction, Three Argument
364*/
365template <WordType W>
366inline constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W {
367#if defined(BOTAN_MP_USE_X86_64_ASM)
368 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
369 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
370 : [carry] "=r"(carry)
371 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
372 : "cc", "memory");
373 return carry;
374 }
375#endif
376
377 z[0] = word_sub(x[0], y[0], &carry);
378 z[1] = word_sub(x[1], y[1], &carry);
379 z[2] = word_sub(x[2], y[2], &carry);
380 z[3] = word_sub(x[3], y[3], &carry);
381 z[4] = word_sub(x[4], y[4], &carry);
382 z[5] = word_sub(x[5], y[5], &carry);
383 z[6] = word_sub(x[6], y[6], &carry);
384 z[7] = word_sub(x[7], y[7], &carry);
385 return carry;
386}
387
388template <WordType W>
389inline constexpr auto word4_sub3(W z[4], const W x[4], const W y[4], W carry) -> W {
390#if defined(BOTAN_MP_USE_X86_64_ASM)
391 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
392 asm volatile(ADD_OR_SUBTRACT(DO_4_TIMES(ADDSUB3_OP, "sbbq"))
393 : [carry] "=r"(carry)
394 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
395 : "cc", "memory");
396 return carry;
397 }
398#endif
399
400 z[0] = word_sub(x[0], y[0], &carry);
401 z[1] = word_sub(x[1], y[1], &carry);
402 z[2] = word_sub(x[2], y[2], &carry);
403 z[3] = word_sub(x[3], y[3], &carry);
404 return carry;
405}
406
407/*
408* Eight Word Block Linear Multiplication
409*/
410template <WordType W>
411inline constexpr auto word8_linmul2(W x[8], W y, W carry) -> W {
412#if defined(BOTAN_MP_USE_X86_64_ASM)
413 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
414 asm(DO_8_TIMES(LINMUL_OP, "x")
415 : [carry] "=r"(carry)
416 : [x] "r"(x), [y] "rm"(y), "0"(carry)
417 : "cc", "%rax", "%rdx");
418 return carry;
419 }
420#endif
421
422 x[0] = word_madd2(x[0], y, &carry);
423 x[1] = word_madd2(x[1], y, &carry);
424 x[2] = word_madd2(x[2], y, &carry);
425 x[3] = word_madd2(x[3], y, &carry);
426 x[4] = word_madd2(x[4], y, &carry);
427 x[5] = word_madd2(x[5], y, &carry);
428 x[6] = word_madd2(x[6], y, &carry);
429 x[7] = word_madd2(x[7], y, &carry);
430 return carry;
431}
432
433/*
434* Eight Word Block Linear Multiplication
435*/
436template <WordType W>
437inline constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W {
438#if defined(BOTAN_MP_USE_X86_64_ASM)
439 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
440 asm(DO_8_TIMES(LINMUL_OP, "z")
441 : [carry] "=r"(carry)
442 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
443 : "cc", "%rax", "%rdx");
444 return carry;
445 }
446#endif
447
448 z[0] = word_madd2(x[0], y, &carry);
449 z[1] = word_madd2(x[1], y, &carry);
450 z[2] = word_madd2(x[2], y, &carry);
451 z[3] = word_madd2(x[3], y, &carry);
452 z[4] = word_madd2(x[4], y, &carry);
453 z[5] = word_madd2(x[5], y, &carry);
454 z[6] = word_madd2(x[6], y, &carry);
455 z[7] = word_madd2(x[7], y, &carry);
456 return carry;
457}
458
459/*
460* Eight Word Block Multiply/Add
461*/
462template <WordType W>
463inline constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W {
464#if defined(BOTAN_MP_USE_X86_64_ASM)
465 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
466 asm(DO_8_TIMES(MULADD_OP, "")
467 : [carry] "=r"(carry)
468 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
469 : "cc", "%rax", "%rdx");
470 return carry;
471 }
472#endif
473
474 z[0] = word_madd3(x[0], y, z[0], &carry);
475 z[1] = word_madd3(x[1], y, z[1], &carry);
476 z[2] = word_madd3(x[2], y, z[2], &carry);
477 z[3] = word_madd3(x[3], y, z[3], &carry);
478 z[4] = word_madd3(x[4], y, z[4], &carry);
479 z[5] = word_madd3(x[5], y, z[5], &carry);
480 z[6] = word_madd3(x[6], y, z[6], &carry);
481 z[7] = word_madd3(x[7], y, z[7], &carry);
482 return carry;
483}
484
485/*
486* Multiply-Add Accumulator
487* (w2,w1,w0) += x * y
488*/
489template <WordType W>
490inline constexpr void word3_muladd(W* w2, W* w1, W* w0, W x, W y) {
491#if defined(BOTAN_MP_USE_X86_64_ASM)
492 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
493 W z0 = 0, z1 = 0;
494
495 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
496
497 asm(R"(
498 addq %[z0],%[w0]
499 adcq %[z1],%[w1]
500 adcq $0,%[w2]
501 )"
502 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
503 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
504 : "cc");
505 return;
506 }
507#endif
508
509 W carry = *w0;
510 *w0 = word_madd2(x, y, &carry);
511 *w1 += carry;
512 *w2 += (*w1 < carry);
513}
514
515/*
516* 3-word addition
517* (w2,w1,w0) += x
518*/
519template <WordType W>
520inline constexpr void word3_add(W* w2, W* w1, W* w0, W x) {
521#if defined(BOTAN_MP_USE_X86_64_ASM)
522 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
523 asm(R"(
524 addq %[x],%[w0]
525 adcq $0,%[w1]
526 adcq $0,%[w2]
527 )"
528 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
529 : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2)
530 : "cc");
531 return;
532 }
533#endif
534
535 *w0 += x;
536 W c1 = (*w0 < x);
537 *w1 += c1;
538 W c2 = (*w1 < c1);
539 *w2 += c2;
540}
541
542/*
543* Multiply-Add Accumulator
544* (w2,w1,w0) += 2 * x * y
545*/
546template <WordType W>
547inline constexpr void word3_muladd_2(W* w2, W* w1, W* w0, W x, W y) {
548#if defined(BOTAN_MP_USE_X86_64_ASM)
549 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
550 W z0 = 0, z1 = 0;
551
552 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
553
554 asm(R"(
555 addq %[z0],%[w0]
556 adcq %[z1],%[w1]
557 adcq $0,%[w2]
558
559 addq %[z0],%[w0]
560 adcq %[z1],%[w1]
561 adcq $0,%[w2]
562 )"
563 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
564 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
565 : "cc");
566 return;
567 }
568#endif
569
570 W carry = 0;
571 x = word_madd2(x, y, &carry);
572 y = carry;
573
574 const size_t top_bit_shift = WordInfo<W>::bits - 1;
575
576 W top = (y >> top_bit_shift);
577 y <<= 1;
578 y |= (x >> top_bit_shift);
579 x <<= 1;
580
581 carry = 0;
582 *w0 = word_add(*w0, x, &carry);
583 *w1 = word_add(*w1, y, &carry);
584 *w2 = word_add(*w2, top, &carry);
585}
586
587/**
588* Helper for 3-word accumulators
589*
590* A number of algorithms especially Comba multiplication and
591* Montgomery reduction can take advantage of wide accumulators, which
592* consume inputs via addition with outputs extracted from the low
593* bits.
594*/
595template <WordType W>
596class word3 {
597#if defined(__BITINT_MAXWIDTH__) && (__BITINT_MAXWIDTH__ >= 3 * 64)
598
599 public:
600 constexpr word3() { m_w = 0; }
601
602 inline constexpr void mul(W x, W y) { m_w += static_cast<W3>(x) * y; }
603
604 inline constexpr void mul_x2(W x, W y) { m_w += static_cast<W3>(x) * y * 2; }
605
606 inline constexpr void add(W x) { m_w += x; }
607
608 inline constexpr W extract() {
609 W r = static_cast<W>(m_w);
610 m_w >>= WordInfo<W>::bits;
611 return r;
612 }
613
614 inline constexpr W monty_step(W p0, W p_dash) {
615 const W w0 = static_cast<W>(m_w);
616 const W r = w0 * p_dash;
617 mul(r, p0);
618 m_w >>= WordInfo<W>::bits;
619 return r;
620 }
621
622 inline constexpr W monty_step_pdash1() {
623 const W r = static_cast<W>(m_w);
624 m_w >>= WordInfo<W>::bits;
625 m_w += static_cast<W3>(r);
626 return r;
627 }
628
629 private:
630 __extension__ typedef unsigned _BitInt(WordInfo<W>::bits * 3) W3;
631 W3 m_w;
632#else
633
634 public:
635 constexpr word3() {
636 m_w2 = 0;
637 m_w1 = 0;
638 m_w0 = 0;
639 }
640
641 inline constexpr void mul(W x, W y) { word3_muladd(&m_w2, &m_w1, &m_w0, x, y); }
642
643 inline constexpr void mul_x2(W x, W y) { word3_muladd_2(&m_w2, &m_w1, &m_w0, x, y); }
644
645 inline constexpr void add(W x) { word3_add(&m_w2, &m_w1, &m_w0, x); }
646
647 inline constexpr W extract() {
648 W r = m_w0;
649 m_w0 = m_w1;
650 m_w1 = m_w2;
651 m_w2 = 0;
652 return r;
653 }
654
655 inline constexpr W monty_step(W p0, W p_dash) {
656 W r = m_w0 * p_dash;
657 mul(r, p0);
658 m_w0 = m_w1;
659 m_w1 = m_w2;
660 m_w2 = 0;
661 return r;
662 }
663
664 inline constexpr W monty_step_pdash1() {
665 // If p_dash == 1 then p[0] = -1 and everything simplifies
666 const W r = m_w0;
667 m_w0 += m_w1;
668 m_w1 = m_w2 + (m_w0 < m_w1);
669 m_w2 = 0;
670 return r;
671 }
672
673 private:
674 W m_w0, m_w1, m_w2;
675#endif
676};
677
678#if defined(ASM)
679 #undef ASM
680 #undef DO_4_TIMES
681 #undef DO_8_TIMES
682 #undef ADD_OR_SUBTRACT
683 #undef ADDSUB2_OP
684 #undef ADDSUB3_OP
685 #undef LINMUL_OP
686 #undef MULADD_OP
687#endif
688
689} // namespace Botan
690
691#endif
Botan::word3::add
constexpr void add(W x)
Definition mp_asmi.h:645
Botan::word3::monty_step
constexpr W monty_step(W p0, W p_dash)
Definition mp_asmi.h:655
Botan::word3::word3
constexpr word3()
Definition mp_asmi.h:635
Botan::word3::extract
constexpr W extract()
Definition mp_asmi.h:647
Botan::word3::mul
constexpr void mul(W x, W y)
Definition mp_asmi.h:641
Botan::word3::monty_step_pdash1
constexpr W monty_step_pdash1()
Definition mp_asmi.h:664
Botan::word3::mul_x2
constexpr void mul_x2(W x, W y)
Definition mp_asmi.h:643
Botan::word8_sub3
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:366
Botan::word_sub
constexpr auto word_sub(W x, W y, W *carry) -> W
Definition mp_asmi.h:281
Botan::word_add
constexpr auto word_add(W x, W y, W *carry) -> W
Definition mp_asmi.h:177
Botan::word8_madd3
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:463
Botan::word8_add3
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:236
Botan::word4_sub3
constexpr auto word4_sub3(W z[4], const W x[4], const W y[4], W carry) -> W
Definition mp_asmi.h:389
Botan::word8_sub2
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:314
Botan::word_madd2
constexpr auto word_madd2(W a, W b, W *c) -> W
Definition mp_asmi.h:66
Botan::word8_sub2_rev
constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:340
Botan::word4_add3
constexpr auto word4_add3(W z[4], const W x[4], const W y[4], W carry) -> W
Definition mp_asmi.h:259
Botan::word3_add
constexpr void word3_add(W *w2, W *w1, W *w0, W x)
Definition mp_asmi.h:520
Botan::word3_muladd_2
constexpr void word3_muladd_2(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:547
Botan::word3_muladd
constexpr void word3_muladd(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:490
Botan::carry
void carry(int64_t &h0, int64_t &h1)
Definition ed25519_internal.h:28
Botan::word8_linmul2
constexpr auto word8_linmul2(W x[8], W y, W carry) -> W
Definition mp_asmi.h:411
Botan::b
const SIMD_8x32 & b
Definition simd_avx2_gfni.h:63
Botan::word8_add2
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:210
Botan::word8_linmul3
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:437
Botan::word_madd3
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
Definition mp_asmi.h:92
Botan::WordInfo< uint32_t >::dword
uint64_t dword
Definition mp_asmi.h:41
Botan::WordInfo< uint64_t >::dword
donna128 dword
Definition mp_asmi.h:57
Generated by doxygen 1.12.0