Botan 3.7.1
Crypto and TLS for C&
mp_asmi.h
Go to the documentation of this file.
1/*
2* Lowest Level MPI Algorithms
3* (C) 1999-2010 Jack Lloyd
4* 2006 Luca Piccarreta
5*
6* Botan is released under the Simplified BSD License (see license.txt)
7*/
8
9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
11
12#include <botan/compiler.h>
13#include <botan/types.h>
14
15#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
16 #include <botan/internal/donna128.h>
17#endif
18
19namespace Botan {
20
21#if defined(BOTAN_USE_GCC_INLINE_ASM) && defined(BOTAN_TARGET_ARCH_IS_X86_64)
22 #define BOTAN_MP_USE_X86_64_ASM
23#endif
24
25/*
26* Concept for allowed multiprecision word types
27*/
28template <typename T>
29concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
30
31template <WordType W>
32struct WordInfo {};
33
34template <>
35struct WordInfo<uint32_t> {
36 public:
37 static const constexpr size_t bytes = 4;
38 static const constexpr size_t bits = 32;
39 static const constexpr uint32_t max = 0xFFFFFFFF;
40 static const constexpr uint32_t top_bit = 0x80000000;
41
42 typedef uint64_t dword;
43 static const constexpr bool dword_is_native = true;
44};
45
46template <>
47struct WordInfo<uint64_t> {
48 public:
49 static const constexpr size_t bytes = 8;
50 static const constexpr size_t bits = 64;
51 static const constexpr uint64_t max = 0xFFFFFFFFFFFFFFFF;
52 static const constexpr uint64_t top_bit = 0x8000000000000000;
53
54#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
55 typedef uint128_t dword;
56 static const constexpr bool dword_is_native = true;
57#else
58 typedef donna128 dword;
59 static const constexpr bool dword_is_native = false;
60#endif
61};
62
63/*
64* Word Multiply/Add
65*/
66template <WordType W>
67inline constexpr auto word_madd2(W a, W b, W* c) -> W {
68#if defined(BOTAN_MP_USE_X86_64_ASM)
69 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
70 asm(R"(
71 mulq %[b]
72 addq %[c],%[a]
73 adcq $0,%[carry]
74 )"
75 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c)
76 : "0"(a), "1"(b), [c] "g"(*c)
77 : "cc");
78
79 return a;
80 }
81#endif
82
83 typedef typename WordInfo<W>::dword dword;
84 const dword s = dword(a) * b + *c;
85 *c = static_cast<W>(s >> WordInfo<W>::bits);
86 return static_cast<W>(s);
87}
88
89/*
90* Word Multiply/Add
91*/
92template <WordType W>
93inline constexpr auto word_madd3(W a, W b, W c, W* d) -> W {
94#if defined(BOTAN_MP_USE_X86_64_ASM)
95 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
96 asm(R"(
97 mulq %[b]
98
99 addq %[c],%[a]
100 adcq $0,%[carry]
101
102 addq %[d],%[a]
103 adcq $0,%[carry]
104 )"
105 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d)
106 : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d)
107 : "cc");
108
109 return a;
110 }
111#endif
112
113 typedef typename WordInfo<W>::dword dword;
114 const dword s = dword(a) * b + c + *d;
115 *d = static_cast<W>(s >> WordInfo<W>::bits);
116 return static_cast<W>(s);
117}
118
119#if defined(BOTAN_MP_USE_X86_64_ASM)
120
121 #define ASM(x) x "\n\t"
122
123 #define DO_4_TIMES(MACRO, ARG) \
124 MACRO(ARG, 0) \
125 MACRO(ARG, 1) \
126 MACRO(ARG, 2) \
127 MACRO(ARG, 3)
128
129 #define DO_8_TIMES(MACRO, ARG) \
130 MACRO(ARG, 0) \
131 MACRO(ARG, 1) \
132 MACRO(ARG, 2) \
133 MACRO(ARG, 3) \
134 MACRO(ARG, 4) \
135 MACRO(ARG, 5) \
136 MACRO(ARG, 6) \
137 MACRO(ARG, 7)
138
139 #define ADDSUB2_OP(OPERATION, INDEX) \
140 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
141 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
142
143 #define ADDSUB3_OP(OPERATION, INDEX) \
144 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
145 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
146 ASM("movq %[carry], 8*" #INDEX "(%[z])")
147
148 #define LINMUL_OP(WRITE_TO, INDEX) \
149 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
150 ASM("mulq %[y]") \
151 ASM("addq %[carry],%%rax") \
152 ASM("adcq $0,%%rdx") \
153 ASM("movq %%rdx,%[carry]") \
154 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
155
156 #define MULADD_OP(IGNORED, INDEX) \
157 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
158 ASM("mulq %[y]") \
159 ASM("addq %[carry],%%rax") \
160 ASM("adcq $0,%%rdx") \
161 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
162 ASM("adcq $0,%%rdx") \
163 ASM("movq %%rdx,%[carry]") \
164 ASM("movq %%rax, 8*" #INDEX " (%[z])")
165
166 #define ADD_OR_SUBTRACT(CORE_CODE) \
167 ASM("rorq %[carry]") \
168 CORE_CODE \
169 ASM("sbbq %[carry],%[carry]") \
170 ASM("negq %[carry]")
171
172#endif
173
174/*
175* Word Addition
176*/
177template <WordType W>
178inline constexpr auto word_add(W x, W y, W* carry) -> W {
179 if(!std::is_constant_evaluated()) {
180#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_addc)
181 if constexpr(std::same_as<W, unsigned int>) {
182 return __builtin_addc(x, y, *carry & 1, carry);
183 } else if constexpr(std::same_as<W, unsigned long>) {
184 return __builtin_addcl(x, y, *carry & 1, carry);
185 } else if constexpr(std::same_as<W, unsigned long long>) {
186 return __builtin_addcll(x, y, *carry & 1, carry);
187 }
188#elif defined(BOTAN_MP_USE_X86_64_ASM)
189 if(std::same_as<W, uint64_t>) {
190 asm(ADD_OR_SUBTRACT(ASM("adcq %[y],%[x]"))
191 : [x] "=r"(x), [carry] "=r"(*carry)
192 : "0"(x), [y] "rm"(y), "1"(*carry)
193 : "cc");
194 return x;
195 }
196#endif
197 }
198
199 const W cb = *carry & 1;
200 W z = x + y;
201 W c1 = (z < x);
202 z += cb;
203 *carry = c1 | (z < cb);
204 return z;
205}
206
207/*
208* Eight Word Block Addition, Two Argument
209*/
210template <WordType W>
211inline constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W {
212#if defined(BOTAN_MP_USE_X86_64_ASM)
213 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
214 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq"))
215 : [carry] "=r"(carry)
216 : [x] "r"(x), [y] "r"(y), "0"(carry)
217 : "cc", "memory");
218 return carry;
219 }
220#endif
221
222 x[0] = word_add(x[0], y[0], &carry);
223 x[1] = word_add(x[1], y[1], &carry);
224 x[2] = word_add(x[2], y[2], &carry);
225 x[3] = word_add(x[3], y[3], &carry);
226 x[4] = word_add(x[4], y[4], &carry);
227 x[5] = word_add(x[5], y[5], &carry);
228 x[6] = word_add(x[6], y[6], &carry);
229 x[7] = word_add(x[7], y[7], &carry);
230 return carry;
231}
232
233/*
234* Eight Word Block Addition, Three Argument
235*/
236template <WordType W>
237inline constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W {
238#if defined(BOTAN_MP_USE_X86_64_ASM)
239 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
240 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq"))
241 : [carry] "=r"(carry)
242 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
243 : "cc", "memory");
244 return carry;
245 }
246#endif
247
248 z[0] = word_add(x[0], y[0], &carry);
249 z[1] = word_add(x[1], y[1], &carry);
250 z[2] = word_add(x[2], y[2], &carry);
251 z[3] = word_add(x[3], y[3], &carry);
252 z[4] = word_add(x[4], y[4], &carry);
253 z[5] = word_add(x[5], y[5], &carry);
254 z[6] = word_add(x[6], y[6], &carry);
255 z[7] = word_add(x[7], y[7], &carry);
256 return carry;
257}
258
259template <WordType W>
260inline constexpr auto word4_add3(W z[4], const W x[4], const W y[4], W carry) -> W {
261#if defined(BOTAN_MP_USE_X86_64_ASM)
262 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
263 asm volatile(ADD_OR_SUBTRACT(DO_4_TIMES(ADDSUB3_OP, "adcq"))
264 : [carry] "=r"(carry)
265 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
266 : "cc", "memory");
267 return carry;
268 }
269#endif
270
271 z[0] = word_add(x[0], y[0], &carry);
272 z[1] = word_add(x[1], y[1], &carry);
273 z[2] = word_add(x[2], y[2], &carry);
274 z[3] = word_add(x[3], y[3], &carry);
275 return carry;
276}
277
278/*
279* Word Subtraction
280*/
281template <WordType W>
282inline constexpr auto word_sub(W x, W y, W* carry) -> W {
283 if(!std::is_constant_evaluated()) {
284#if BOTAN_COMPILER_HAS_BUILTIN(__builtin_subc)
285 if constexpr(std::same_as<W, unsigned int>) {
286 return __builtin_subc(x, y, *carry & 1, carry);
287 } else if constexpr(std::same_as<W, unsigned long>) {
288 return __builtin_subcl(x, y, *carry & 1, carry);
289 } else if constexpr(std::same_as<W, unsigned long long>) {
290 return __builtin_subcll(x, y, *carry & 1, carry);
291 }
292#elif defined(BOTAN_MP_USE_X86_64_ASM)
293 if(std::same_as<W, uint64_t>) {
294 asm(ADD_OR_SUBTRACT(ASM("sbbq %[y],%[x]"))
295 : [x] "=r"(x), [carry] "=r"(*carry)
296 : "0"(x), [y] "rm"(y), "1"(*carry)
297 : "cc");
298 return x;
299 }
300#endif
301 }
302
303 const W cb = *carry & 1;
304 W t0 = x - y;
305 W c1 = (t0 > x);
306 W z = t0 - cb;
307 *carry = c1 | (z > t0);
308 return z;
309}
310
311/*
312* Eight Word Block Subtraction, Two Argument
313*/
314template <WordType W>
315inline constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W {
316#if defined(BOTAN_MP_USE_X86_64_ASM)
317 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
318 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq"))
319 : [carry] "=r"(carry)
320 : [x] "r"(x), [y] "r"(y), "0"(carry)
321 : "cc", "memory");
322 return carry;
323 }
324#endif
325
326 x[0] = word_sub(x[0], y[0], &carry);
327 x[1] = word_sub(x[1], y[1], &carry);
328 x[2] = word_sub(x[2], y[2], &carry);
329 x[3] = word_sub(x[3], y[3], &carry);
330 x[4] = word_sub(x[4], y[4], &carry);
331 x[5] = word_sub(x[5], y[5], &carry);
332 x[6] = word_sub(x[6], y[6], &carry);
333 x[7] = word_sub(x[7], y[7], &carry);
334 return carry;
335}
336
337/*
338* Eight Word Block Subtraction, Two Argument
339*/
340template <WordType W>
341inline constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W {
342#if defined(BOTAN_MP_USE_X86_64_ASM)
343 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
344 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
345 : [carry] "=r"(carry)
346 : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry)
347 : "cc", "memory");
348 return carry;
349 }
350#endif
351
352 x[0] = word_sub(y[0], x[0], &carry);
353 x[1] = word_sub(y[1], x[1], &carry);
354 x[2] = word_sub(y[2], x[2], &carry);
355 x[3] = word_sub(y[3], x[3], &carry);
356 x[4] = word_sub(y[4], x[4], &carry);
357 x[5] = word_sub(y[5], x[5], &carry);
358 x[6] = word_sub(y[6], x[6], &carry);
359 x[7] = word_sub(y[7], x[7], &carry);
360 return carry;
361}
362
363/*
364* Eight Word Block Subtraction, Three Argument
365*/
366template <WordType W>
367inline constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W {
368#if defined(BOTAN_MP_USE_X86_64_ASM)
369 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
370 asm volatile(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
371 : [carry] "=r"(carry)
372 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
373 : "cc", "memory");
374 return carry;
375 }
376#endif
377
378 z[0] = word_sub(x[0], y[0], &carry);
379 z[1] = word_sub(x[1], y[1], &carry);
380 z[2] = word_sub(x[2], y[2], &carry);
381 z[3] = word_sub(x[3], y[3], &carry);
382 z[4] = word_sub(x[4], y[4], &carry);
383 z[5] = word_sub(x[5], y[5], &carry);
384 z[6] = word_sub(x[6], y[6], &carry);
385 z[7] = word_sub(x[7], y[7], &carry);
386 return carry;
387}
388
389template <WordType W>
390inline constexpr auto word4_sub3(W z[4], const W x[4], const W y[4], W carry) -> W {
391#if defined(BOTAN_MP_USE_X86_64_ASM)
392 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
393 asm volatile(ADD_OR_SUBTRACT(DO_4_TIMES(ADDSUB3_OP, "sbbq"))
394 : [carry] "=r"(carry)
395 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
396 : "cc", "memory");
397 return carry;
398 }
399#endif
400
401 z[0] = word_sub(x[0], y[0], &carry);
402 z[1] = word_sub(x[1], y[1], &carry);
403 z[2] = word_sub(x[2], y[2], &carry);
404 z[3] = word_sub(x[3], y[3], &carry);
405 return carry;
406}
407
408/*
409* Eight Word Block Linear Multiplication
410*/
411template <WordType W>
412inline constexpr auto word8_linmul2(W x[8], W y, W carry) -> W {
413#if defined(BOTAN_MP_USE_X86_64_ASM)
414 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
415 asm(DO_8_TIMES(LINMUL_OP, "x")
416 : [carry] "=r"(carry)
417 : [x] "r"(x), [y] "rm"(y), "0"(carry)
418 : "cc", "%rax", "%rdx");
419 return carry;
420 }
421#endif
422
423 x[0] = word_madd2(x[0], y, &carry);
424 x[1] = word_madd2(x[1], y, &carry);
425 x[2] = word_madd2(x[2], y, &carry);
426 x[3] = word_madd2(x[3], y, &carry);
427 x[4] = word_madd2(x[4], y, &carry);
428 x[5] = word_madd2(x[5], y, &carry);
429 x[6] = word_madd2(x[6], y, &carry);
430 x[7] = word_madd2(x[7], y, &carry);
431 return carry;
432}
433
434/*
435* Eight Word Block Linear Multiplication
436*/
437template <WordType W>
438inline constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W {
439#if defined(BOTAN_MP_USE_X86_64_ASM)
440 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
441 asm(DO_8_TIMES(LINMUL_OP, "z")
442 : [carry] "=r"(carry)
443 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
444 : "cc", "%rax", "%rdx");
445 return carry;
446 }
447#endif
448
449 z[0] = word_madd2(x[0], y, &carry);
450 z[1] = word_madd2(x[1], y, &carry);
451 z[2] = word_madd2(x[2], y, &carry);
452 z[3] = word_madd2(x[3], y, &carry);
453 z[4] = word_madd2(x[4], y, &carry);
454 z[5] = word_madd2(x[5], y, &carry);
455 z[6] = word_madd2(x[6], y, &carry);
456 z[7] = word_madd2(x[7], y, &carry);
457 return carry;
458}
459
460/*
461* Eight Word Block Multiply/Add
462*/
463template <WordType W>
464inline constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W {
465#if defined(BOTAN_MP_USE_X86_64_ASM)
466 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
467 asm(DO_8_TIMES(MULADD_OP, "")
468 : [carry] "=r"(carry)
469 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
470 : "cc", "%rax", "%rdx");
471 return carry;
472 }
473#endif
474
475 z[0] = word_madd3(x[0], y, z[0], &carry);
476 z[1] = word_madd3(x[1], y, z[1], &carry);
477 z[2] = word_madd3(x[2], y, z[2], &carry);
478 z[3] = word_madd3(x[3], y, z[3], &carry);
479 z[4] = word_madd3(x[4], y, z[4], &carry);
480 z[5] = word_madd3(x[5], y, z[5], &carry);
481 z[6] = word_madd3(x[6], y, z[6], &carry);
482 z[7] = word_madd3(x[7], y, z[7], &carry);
483 return carry;
484}
485
486/*
487* Multiply-Add Accumulator
488* (w2,w1,w0) += x * y
489*/
490template <WordType W>
491inline constexpr void word3_muladd(W* w2, W* w1, W* w0, W x, W y) {
492#if defined(BOTAN_MP_USE_X86_64_ASM)
493 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
494 W z0 = 0, z1 = 0;
495
496 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
497
498 asm(R"(
499 addq %[z0],%[w0]
500 adcq %[z1],%[w1]
501 adcq $0,%[w2]
502 )"
503 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
504 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
505 : "cc");
506 return;
507 }
508#endif
509
510 W carry = *w0;
511 *w0 = word_madd2(x, y, &carry);
512 *w1 += carry;
513 *w2 += (*w1 < carry);
514}
515
516/*
517* 3-word addition
518* (w2,w1,w0) += x
519*/
520template <WordType W>
521inline constexpr void word3_add(W* w2, W* w1, W* w0, W x) {
522#if defined(BOTAN_MP_USE_X86_64_ASM)
523 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
524 asm(R"(
525 addq %[x],%[w0]
526 adcq $0,%[w1]
527 adcq $0,%[w2]
528 )"
529 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
530 : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2)
531 : "cc");
532 return;
533 }
534#endif
535
536 *w0 += x;
537 W c1 = (*w0 < x);
538 *w1 += c1;
539 W c2 = (*w1 < c1);
540 *w2 += c2;
541}
542
543/*
544* Multiply-Add Accumulator
545* (w2,w1,w0) += 2 * x * y
546*/
547template <WordType W>
548inline constexpr void word3_muladd_2(W* w2, W* w1, W* w0, W x, W y) {
549#if defined(BOTAN_MP_USE_X86_64_ASM)
550 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
551 W z0 = 0, z1 = 0;
552
553 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
554
555 asm(R"(
556 addq %[z0],%[w0]
557 adcq %[z1],%[w1]
558 adcq $0,%[w2]
559
560 addq %[z0],%[w0]
561 adcq %[z1],%[w1]
562 adcq $0,%[w2]
563 )"
564 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
565 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
566 : "cc");
567 return;
568 }
569#endif
570
571 W carry = 0;
572 x = word_madd2(x, y, &carry);
573 y = carry;
574
575 const size_t top_bit_shift = WordInfo<W>::bits - 1;
576
577 W top = (y >> top_bit_shift);
578 y <<= 1;
579 y |= (x >> top_bit_shift);
580 x <<= 1;
581
582 carry = 0;
583 *w0 = word_add(*w0, x, &carry);
584 *w1 = word_add(*w1, y, &carry);
585 *w2 = word_add(*w2, top, &carry);
586}
587
588/**
589* Helper for 3-word accumulators
590*
591* A number of algorithms especially Comba multiplication and
592* Montgomery reduction can take advantage of wide accumulators, which
593* consume inputs via addition with outputs extracted from the low
594* bits.
595*/
596template <WordType W>
597class word3 final {
598#if defined(__BITINT_MAXWIDTH__) && (__BITINT_MAXWIDTH__ >= 3 * 64)
599
600 public:
601 constexpr word3() { m_w = 0; }
602
603 inline constexpr void mul(W x, W y) { m_w += static_cast<W3>(x) * y; }
604
605 inline constexpr void mul_x2(W x, W y) { m_w += static_cast<W3>(x) * y * 2; }
606
607 inline constexpr void add(W x) { m_w += x; }
608
609 inline constexpr W extract() {
610 W r = static_cast<W>(m_w);
611 m_w >>= WordInfo<W>::bits;
612 return r;
613 }
614
615 inline constexpr W monty_step(W p0, W p_dash) {
616 const W w0 = static_cast<W>(m_w);
617 const W r = w0 * p_dash;
618 mul(r, p0);
619 m_w >>= WordInfo<W>::bits;
620 return r;
621 }
622
623 inline constexpr W monty_step_pdash1() {
624 const W r = static_cast<W>(m_w);
625 m_w >>= WordInfo<W>::bits;
626 m_w += static_cast<W3>(r);
627 return r;
628 }
629
630 private:
631 __extension__ typedef unsigned _BitInt(WordInfo<W>::bits * 3) W3;
632 W3 m_w;
633#else
634
635 public:
636 constexpr word3() {
637 m_w2 = 0;
638 m_w1 = 0;
639 m_w0 = 0;
640 }
641
642 inline constexpr void mul(W x, W y) { word3_muladd(&m_w2, &m_w1, &m_w0, x, y); }
643
644 inline constexpr void mul_x2(W x, W y) { word3_muladd_2(&m_w2, &m_w1, &m_w0, x, y); }
645
646 inline constexpr void add(W x) { word3_add(&m_w2, &m_w1, &m_w0, x); }
647
648 inline constexpr W extract() {
649 W r = m_w0;
650 m_w0 = m_w1;
651 m_w1 = m_w2;
652 m_w2 = 0;
653 return r;
654 }
655
656 inline constexpr W monty_step(W p0, W p_dash) {
657 W r = m_w0 * p_dash;
658 mul(r, p0);
659 m_w0 = m_w1;
660 m_w1 = m_w2;
661 m_w2 = 0;
662 return r;
663 }
664
665 inline constexpr W monty_step_pdash1() {
666 // If p_dash == 1 then p[0] = -1 and everything simplifies
667 const W r = m_w0;
668 m_w0 += m_w1;
669 m_w1 = m_w2 + (m_w0 < m_w1);
670 m_w2 = 0;
671 return r;
672 }
673
674 private:
675 W m_w0, m_w1, m_w2;
676#endif
677};
678
679#if defined(ASM)
680 #undef ASM
681 #undef DO_4_TIMES
682 #undef DO_8_TIMES
683 #undef ADD_OR_SUBTRACT
684 #undef ADDSUB2_OP
685 #undef ADDSUB3_OP
686 #undef LINMUL_OP
687 #undef MULADD_OP
688#endif
689
690} // namespace Botan
691
692#endif
Botan::word3::add
constexpr void add(W x)
Definition mp_asmi.h:646
Botan::word3::monty_step
constexpr W monty_step(W p0, W p_dash)
Definition mp_asmi.h:656
Botan::word3::word3
constexpr word3()
Definition mp_asmi.h:636
Botan::word3::extract
constexpr W extract()
Definition mp_asmi.h:648
Botan::word3::mul
constexpr void mul(W x, W y)
Definition mp_asmi.h:642
Botan::word3::monty_step_pdash1
constexpr W monty_step_pdash1()
Definition mp_asmi.h:665
Botan::word3::mul_x2
constexpr void mul_x2(W x, W y)
Definition mp_asmi.h:644
final
int(* final)(unsigned char *, CTX *)
Definition commoncrypto_hash.cpp:29
Botan::word8_sub3
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:367
Botan::word_sub
constexpr auto word_sub(W x, W y, W *carry) -> W
Definition mp_asmi.h:282
Botan::word_add
constexpr auto word_add(W x, W y, W *carry) -> W
Definition mp_asmi.h:178
Botan::word8_madd3
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:464
Botan::word8_add3
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:237
Botan::word4_sub3
constexpr auto word4_sub3(W z[4], const W x[4], const W y[4], W carry) -> W
Definition mp_asmi.h:390
Botan::word8_sub2
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:315
Botan::word_madd2
constexpr auto word_madd2(W a, W b, W *c) -> W
Definition mp_asmi.h:67
Botan::word8_sub2_rev
constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:341
Botan::word4_add3
constexpr auto word4_add3(W z[4], const W x[4], const W y[4], W carry) -> W
Definition mp_asmi.h:260
Botan::word3_add
constexpr void word3_add(W *w2, W *w1, W *w0, W x)
Definition mp_asmi.h:521
Botan::word3_muladd_2
constexpr void word3_muladd_2(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:548
Botan::word3_muladd
constexpr void word3_muladd(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:491
Botan::carry
void carry(int64_t &h0, int64_t &h1)
Definition ed25519_internal.h:28
Botan::word8_linmul2
constexpr auto word8_linmul2(W x[8], W y, W carry) -> W
Definition mp_asmi.h:412
Botan::b
const SIMD_8x32 & b
Definition simd_avx2_gfni.h:63
Botan::word8_add2
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:211
Botan::word8_linmul3
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:438
Botan::word_madd3
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
Definition mp_asmi.h:93
Botan::WordInfo< uint32_t >::dword
uint64_t dword
Definition mp_asmi.h:42
Botan::WordInfo< uint64_t >::dword
donna128 dword
Definition mp_asmi.h:58
Generated by doxygen 1.12.0