Botan 3.4.0
Crypto and TLS for C&
mp_asmi.h
Go to the documentation of this file.
1/*
2* Lowest Level MPI Algorithms
3* (C) 1999-2010 Jack Lloyd
4* 2006 Luca Piccarreta
5*
6* Botan is released under the Simplified BSD License (see license.txt)
7*/
8
9#ifndef BOTAN_MP_ASM_INTERNAL_H_
10#define BOTAN_MP_ASM_INTERNAL_H_
11
12#include <botan/types.h>
13
14#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
15 #include <botan/internal/donna128.h>
16#endif
17
18namespace Botan {
19
20#if defined(BOTAN_USE_GCC_INLINE_ASM)
21 #if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY)
22 #define BOTAN_MP_USE_X86_32_ASM
23 #endif
24
25 #if defined(BOTAN_TARGET_ARCH_IS_X86_64)
26 #define BOTAN_MP_USE_X86_64_ASM
27 #endif
28#endif
29
30/*
31* Concept for allowed multiprecision word types
32*/
33template <typename T>
34concept WordType = (std::same_as<T, uint32_t> || std::same_as<T, uint64_t>);
35
36template <WordType W>
37struct WordInfo {};
38
39template <>
40struct WordInfo<uint32_t> {
41 public:
42 static const constexpr size_t bytes = 4;
43 static const constexpr size_t bits = 32;
44 static const constexpr uint32_t max = 0xFFFFFFFF;
45
46 typedef uint64_t dword;
47 static const constexpr bool dword_is_native = true;
48};
49
50template <>
51struct WordInfo<uint64_t> {
52 public:
53 static const constexpr size_t bytes = 8;
54 static const constexpr size_t bits = 64;
55 static const constexpr uint64_t max = 0xFFFFFFFFFFFFFFFF;
56
57#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
58 typedef uint128_t dword;
59 static const constexpr bool dword_is_native = true;
60#else
61 typedef donna128 dword;
62 static const constexpr bool dword_is_native = false;
63#endif
64};
65
66/*
67* Word Multiply/Add
68*/
69template <WordType W>
70inline constexpr auto word_madd2(W a, W b, W* c) -> W {
71#if defined(BOTAN_MP_USE_X86_32_ASM)
72 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
73 asm(R"(
74 mull %[b]
75 addl %[c],%[a]
76 adcl $0,%[carry]
77 )"
78 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c)
79 : "0"(a), "1"(b), [c] "g"(*c)
80 : "cc");
81
82 return a;
83 }
84#endif
85
86#if defined(BOTAN_MP_USE_X86_64_ASM)
87 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
88 asm(R"(
89 mulq %[b]
90 addq %[c],%[a]
91 adcq $0,%[carry]
92 )"
93 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c)
94 : "0"(a), "1"(b), [c] "g"(*c)
95 : "cc");
96
97 return a;
98 }
99#endif
100
101 typedef typename WordInfo<W>::dword dword;
102 const dword s = dword(a) * b + *c;
103 *c = static_cast<W>(s >> WordInfo<W>::bits);
104 return static_cast<W>(s);
105}
106
107/*
108* Word Multiply/Add
109*/
110template <WordType W>
111inline constexpr auto word_madd3(W a, W b, W c, W* d) -> W {
112#if defined(BOTAN_MP_USE_X86_32_ASM)
113 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
114 asm(R"(
115 mull %[b]
116
117 addl %[c],%[a]
118 adcl $0,%[carry]
119
120 addl %[d],%[a]
121 adcl $0,%[carry]
122 )"
123 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d)
124 : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d)
125 : "cc");
126
127 return a;
128 }
129#endif
130
131#if defined(BOTAN_MP_USE_X86_64_ASM)
132 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
133 asm(R"(
134 mulq %[b]
135
136 addq %[c],%[a]
137 adcq $0,%[carry]
138
139 addq %[d],%[a]
140 adcq $0,%[carry]
141 )"
142 : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d)
143 : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d)
144 : "cc");
145
146 return a;
147 }
148#endif
149
150 typedef typename WordInfo<W>::dword dword;
151 const dword s = dword(a) * b + c + *d;
152 *d = static_cast<W>(s >> WordInfo<W>::bits);
153 return static_cast<W>(s);
154}
155
156#if defined(BOTAN_MP_USE_X86_32_ASM)
157
158 #define ADDSUB2_OP(OPERATION, INDEX) \
159 ASM("movl 4*" #INDEX "(%[y]), %[carry]") \
160 ASM(OPERATION " %[carry], 4*" #INDEX "(%[x])")
161
162 #define ADDSUB3_OP(OPERATION, INDEX) \
163 ASM("movl 4*" #INDEX "(%[x]), %[carry]") \
164 ASM(OPERATION " 4*" #INDEX "(%[y]), %[carry]") \
165 ASM("movl %[carry], 4*" #INDEX "(%[z])")
166
167 #define LINMUL_OP(WRITE_TO, INDEX) \
168 ASM("movl 4*" #INDEX "(%[x]),%%eax") \
169 ASM("mull %[y]") \
170 ASM("addl %[carry],%%eax") \
171 ASM("adcl $0,%%edx") \
172 ASM("movl %%edx,%[carry]") \
173 ASM("movl %%eax, 4*" #INDEX "(%[" WRITE_TO "])")
174
175 #define MULADD_OP(IGNORED, INDEX) \
176 ASM("movl 4*" #INDEX "(%[x]),%%eax") \
177 ASM("mull %[y]") \
178 ASM("addl %[carry],%%eax") \
179 ASM("adcl $0,%%edx") \
180 ASM("addl 4*" #INDEX "(%[z]),%%eax") \
181 ASM("adcl $0,%%edx") \
182 ASM("movl %%edx,%[carry]") \
183 ASM("movl %%eax, 4*" #INDEX " (%[z])")
184
185 #define ADD_OR_SUBTRACT(CORE_CODE) \
186 ASM("rorl %[carry]") \
187 CORE_CODE \
188 ASM("sbbl %[carry],%[carry]") \
189 ASM("negl %[carry]")
190
191#elif defined(BOTAN_MP_USE_X86_64_ASM)
192
193 #define ADDSUB2_OP(OPERATION, INDEX) \
194 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
195 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])")
196
197 #define ADDSUB3_OP(OPERATION, INDEX) \
198 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
199 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
200 ASM("movq %[carry], 8*" #INDEX "(%[z])")
201
202 #define LINMUL_OP(WRITE_TO, INDEX) \
203 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
204 ASM("mulq %[y]") \
205 ASM("addq %[carry],%%rax") \
206 ASM("adcq $0,%%rdx") \
207 ASM("movq %%rdx,%[carry]") \
208 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
209
210 #define MULADD_OP(IGNORED, INDEX) \
211 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
212 ASM("mulq %[y]") \
213 ASM("addq %[carry],%%rax") \
214 ASM("adcq $0,%%rdx") \
215 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
216 ASM("adcq $0,%%rdx") \
217 ASM("movq %%rdx,%[carry]") \
218 ASM("movq %%rax, 8*" #INDEX " (%[z])")
219
220 #define ADD_OR_SUBTRACT(CORE_CODE) \
221 ASM("rorq %[carry]") \
222 CORE_CODE \
223 ASM("sbbq %[carry],%[carry]") \
224 ASM("negq %[carry]")
225
226#endif
227
228#if defined(ADD_OR_SUBTRACT)
229
230 #define ASM(x) x "\n\t"
231
232 #define DO_8_TIMES(MACRO, ARG) \
233 MACRO(ARG, 0) \
234 MACRO(ARG, 1) \
235 MACRO(ARG, 2) \
236 MACRO(ARG, 3) \
237 MACRO(ARG, 4) \
238 MACRO(ARG, 5) \
239 MACRO(ARG, 6) \
240 MACRO(ARG, 7)
241
242#endif
243
244/*
245* Word Addition
246*/
247template <WordType W>
248inline constexpr auto word_add(W x, W y, W* carry) -> W {
249#if defined(BOTAN_MP_USE_X86_32_ASM)
250 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
251 asm(ADD_OR_SUBTRACT(ASM("adcl %[y],%[x]"))
252 : [x] "=r"(x), [carry] "=r"(*carry)
253 : "0"(x), [y] "rm"(y), "1"(*carry)
254 : "cc");
255 return x;
256 }
257#endif
258
259#if defined(BOTAN_MP_USE_X86_64_ASM)
260 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
261 asm(ADD_OR_SUBTRACT(ASM("adcq %[y],%[x]"))
262 : [x] "=r"(x), [carry] "=r"(*carry)
263 : "0"(x), [y] "rm"(y), "1"(*carry)
264 : "cc");
265 return x;
266 }
267#endif
268
269 W z = x + y;
270 W c1 = (z < x);
271 z += *carry;
272 *carry = c1 | (z < *carry);
273 return z;
274}
275
276/*
277* Eight Word Block Addition, Two Argument
278*/
279template <WordType W>
280inline constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W {
281#if defined(BOTAN_MP_USE_X86_32_ASM)
282 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
283 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcl"))
284 : [carry] "=r"(carry)
285 : [x] "r"(x), [y] "r"(y), "0"(carry)
286 : "cc", "memory");
287 return carry;
288 }
289#endif
290
291#if defined(BOTAN_MP_USE_X86_64_ASM)
292 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
293 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq"))
294 : [carry] "=r"(carry)
295 : [x] "r"(x), [y] "r"(y), "0"(carry)
296 : "cc", "memory");
297 return carry;
298 }
299#endif
300
301 x[0] = word_add(x[0], y[0], &carry);
302 x[1] = word_add(x[1], y[1], &carry);
303 x[2] = word_add(x[2], y[2], &carry);
304 x[3] = word_add(x[3], y[3], &carry);
305 x[4] = word_add(x[4], y[4], &carry);
306 x[5] = word_add(x[5], y[5], &carry);
307 x[6] = word_add(x[6], y[6], &carry);
308 x[7] = word_add(x[7], y[7], &carry);
309 return carry;
310}
311
312/*
313* Eight Word Block Addition, Three Argument
314*/
315template <WordType W>
316inline constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W {
317#if defined(BOTAN_MP_USE_X86_32_ASM)
318 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
319 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcl"))
320 : [carry] "=r"(carry)
321 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
322 : "cc", "memory");
323 return carry;
324 }
325#endif
326
327#if defined(BOTAN_MP_USE_X86_64_ASM)
328 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
329 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq"))
330 : [carry] "=r"(carry)
331 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
332 : "cc", "memory");
333 return carry;
334 }
335#endif
336
337 z[0] = word_add(x[0], y[0], &carry);
338 z[1] = word_add(x[1], y[1], &carry);
339 z[2] = word_add(x[2], y[2], &carry);
340 z[3] = word_add(x[3], y[3], &carry);
341 z[4] = word_add(x[4], y[4], &carry);
342 z[5] = word_add(x[5], y[5], &carry);
343 z[6] = word_add(x[6], y[6], &carry);
344 z[7] = word_add(x[7], y[7], &carry);
345 return carry;
346}
347
348/*
349* Word Subtraction
350*/
351template <WordType W>
352inline constexpr auto word_sub(W x, W y, W* carry) -> W {
353#if defined(BOTAN_MP_USE_X86_32_ASM)
354 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
355 asm(ADD_OR_SUBTRACT(ASM("sbbl %[y],%[x]"))
356 : [x] "=r"(x), [carry] "=r"(*carry)
357 : "0"(x), [y] "rm"(y), "1"(*carry)
358 : "cc");
359 return x;
360 }
361#endif
362
363#if defined(BOTAN_MP_USE_X86_64_ASM)
364 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
365 asm(ADD_OR_SUBTRACT(ASM("sbbq %[y],%[x]"))
366 : [x] "=r"(x), [carry] "=r"(*carry)
367 : "0"(x), [y] "rm"(y), "1"(*carry)
368 : "cc");
369 return x;
370 }
371#endif
372
373 W t0 = x - y;
374 W c1 = (t0 > x);
375 W z = t0 - *carry;
376 *carry = c1 | (z > t0);
377 return z;
378}
379
380/*
381* Eight Word Block Subtraction, Two Argument
382*/
383template <WordType W>
384inline constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W {
385#if defined(BOTAN_MP_USE_X86_32_ASM)
386 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
387 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbl"))
388 : [carry] "=r"(carry)
389 : [x] "r"(x), [y] "r"(y), "0"(carry)
390 : "cc", "memory");
391 return carry;
392 }
393#endif
394
395#if defined(BOTAN_MP_USE_X86_64_ASM)
396 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
397 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq"))
398 : [carry] "=r"(carry)
399 : [x] "r"(x), [y] "r"(y), "0"(carry)
400 : "cc", "memory");
401 return carry;
402 }
403#endif
404
405 x[0] = word_sub(x[0], y[0], &carry);
406 x[1] = word_sub(x[1], y[1], &carry);
407 x[2] = word_sub(x[2], y[2], &carry);
408 x[3] = word_sub(x[3], y[3], &carry);
409 x[4] = word_sub(x[4], y[4], &carry);
410 x[5] = word_sub(x[5], y[5], &carry);
411 x[6] = word_sub(x[6], y[6], &carry);
412 x[7] = word_sub(x[7], y[7], &carry);
413 return carry;
414}
415
416/*
417* Eight Word Block Subtraction, Two Argument
418*/
419template <WordType W>
420inline constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W {
421#if defined(BOTAN_MP_USE_X86_32_ASM)
422 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
423 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl"))
424 : [carry] "=r"(carry)
425 : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry)
426 : "cc", "memory");
427 return carry;
428 }
429#endif
430
431#if defined(BOTAN_MP_USE_X86_64_ASM)
432 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
433 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
434 : [carry] "=r"(carry)
435 : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry)
436 : "cc", "memory");
437 return carry;
438 }
439#endif
440
441 x[0] = word_sub(y[0], x[0], &carry);
442 x[1] = word_sub(y[1], x[1], &carry);
443 x[2] = word_sub(y[2], x[2], &carry);
444 x[3] = word_sub(y[3], x[3], &carry);
445 x[4] = word_sub(y[4], x[4], &carry);
446 x[5] = word_sub(y[5], x[5], &carry);
447 x[6] = word_sub(y[6], x[6], &carry);
448 x[7] = word_sub(y[7], x[7], &carry);
449 return carry;
450}
451
452/*
453* Eight Word Block Subtraction, Three Argument
454*/
455template <WordType W>
456inline constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W {
457#if defined(BOTAN_MP_USE_X86_32_ASM)
458 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
459 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl"))
460 : [carry] "=r"(carry)
461 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
462 : "cc", "memory");
463 return carry;
464 }
465#endif
466
467#if defined(BOTAN_MP_USE_X86_64_ASM)
468 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
469 asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq"))
470 : [carry] "=r"(carry)
471 : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry)
472 : "cc", "memory");
473 return carry;
474 }
475#endif
476
477 z[0] = word_sub(x[0], y[0], &carry);
478 z[1] = word_sub(x[1], y[1], &carry);
479 z[2] = word_sub(x[2], y[2], &carry);
480 z[3] = word_sub(x[3], y[3], &carry);
481 z[4] = word_sub(x[4], y[4], &carry);
482 z[5] = word_sub(x[5], y[5], &carry);
483 z[6] = word_sub(x[6], y[6], &carry);
484 z[7] = word_sub(x[7], y[7], &carry);
485 return carry;
486}
487
488/*
489* Eight Word Block Linear Multiplication
490*/
491template <WordType W>
492inline constexpr auto word8_linmul2(W x[8], W y, W carry) -> W {
493#if defined(BOTAN_MP_USE_X86_32_ASM)
494 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
495 asm(DO_8_TIMES(LINMUL_OP, "x")
496 : [carry] "=r"(carry)
497 : [x] "r"(x), [y] "rm"(y), "0"(carry)
498 : "cc", "%eax", "%edx");
499 return carry;
500 }
501#endif
502
503#if defined(BOTAN_MP_USE_X86_64_ASM)
504 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
505 asm(DO_8_TIMES(LINMUL_OP, "x")
506 : [carry] "=r"(carry)
507 : [x] "r"(x), [y] "rm"(y), "0"(carry)
508 : "cc", "%rax", "%rdx");
509 return carry;
510 }
511#endif
512
513 x[0] = word_madd2(x[0], y, &carry);
514 x[1] = word_madd2(x[1], y, &carry);
515 x[2] = word_madd2(x[2], y, &carry);
516 x[3] = word_madd2(x[3], y, &carry);
517 x[4] = word_madd2(x[4], y, &carry);
518 x[5] = word_madd2(x[5], y, &carry);
519 x[6] = word_madd2(x[6], y, &carry);
520 x[7] = word_madd2(x[7], y, &carry);
521 return carry;
522}
523
524/*
525* Eight Word Block Linear Multiplication
526*/
527template <WordType W>
528inline constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W {
529#if defined(BOTAN_MP_USE_X86_32_ASM)
530 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
531 asm(DO_8_TIMES(LINMUL_OP, "z")
532 : [carry] "=r"(carry)
533 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
534 : "cc", "%eax", "%edx");
535 return carry;
536 }
537#endif
538
539#if defined(BOTAN_MP_USE_X86_64_ASM)
540 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
541 asm(DO_8_TIMES(LINMUL_OP, "z")
542 : [carry] "=r"(carry)
543 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
544 : "cc", "%rax", "%rdx");
545 return carry;
546 }
547#endif
548
549 z[0] = word_madd2(x[0], y, &carry);
550 z[1] = word_madd2(x[1], y, &carry);
551 z[2] = word_madd2(x[2], y, &carry);
552 z[3] = word_madd2(x[3], y, &carry);
553 z[4] = word_madd2(x[4], y, &carry);
554 z[5] = word_madd2(x[5], y, &carry);
555 z[6] = word_madd2(x[6], y, &carry);
556 z[7] = word_madd2(x[7], y, &carry);
557 return carry;
558}
559
560/*
561* Eight Word Block Multiply/Add
562*/
563template <WordType W>
564inline constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W {
565#if defined(BOTAN_MP_USE_X86_32_ASM)
566 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
567 asm(DO_8_TIMES(MULADD_OP, "")
568 : [carry] "=r"(carry)
569 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
570 : "cc", "%eax", "%edx");
571 return carry;
572 }
573#endif
574
575#if defined(BOTAN_MP_USE_X86_64_ASM)
576 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
577 asm(DO_8_TIMES(MULADD_OP, "")
578 : [carry] "=r"(carry)
579 : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry)
580 : "cc", "%rax", "%rdx");
581 return carry;
582 }
583#endif
584
585 z[0] = word_madd3(x[0], y, z[0], &carry);
586 z[1] = word_madd3(x[1], y, z[1], &carry);
587 z[2] = word_madd3(x[2], y, z[2], &carry);
588 z[3] = word_madd3(x[3], y, z[3], &carry);
589 z[4] = word_madd3(x[4], y, z[4], &carry);
590 z[5] = word_madd3(x[5], y, z[5], &carry);
591 z[6] = word_madd3(x[6], y, z[6], &carry);
592 z[7] = word_madd3(x[7], y, z[7], &carry);
593 return carry;
594}
595
596/*
597* Multiply-Add Accumulator
598* (w2,w1,w0) += x * y
599*/
600template <WordType W>
601inline constexpr void word3_muladd(W* w2, W* w1, W* w0, W x, W y) {
602#if defined(BOTAN_MP_USE_X86_32_ASM)
603 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
604 W z0 = 0, z1 = 0;
605
606 asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
607
608 asm(R"(
609 addl %[z0],%[w0]
610 adcl %[z1],%[w1]
611 adcl $0,%[w2]
612 )"
613 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
614 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
615 : "cc");
616 return;
617 }
618#endif
619
620#if defined(BOTAN_MP_USE_X86_64_ASM)
621 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
622 W z0 = 0, z1 = 0;
623
624 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
625
626 asm(R"(
627 addq %[z0],%[w0]
628 adcq %[z1],%[w1]
629 adcq $0,%[w2]
630 )"
631 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
632 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
633 : "cc");
634 return;
635 }
636#endif
637
638 W carry = *w0;
639 *w0 = word_madd2(x, y, &carry);
640 *w1 += carry;
641 *w2 += (*w1 < carry);
642}
643
644/*
645* 3-word addition
646* (w2,w1,w0) += x
647*/
648template <WordType W>
649inline constexpr void word3_add(W* w2, W* w1, W* w0, W x) {
650#if defined(BOTAN_MP_USE_X86_32_ASM)
651 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
652 asm(R"(
653 addl %[x],%[w0]
654 adcl $0,%[w1]
655 adcl $0,%[w2]
656 )"
657 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
658 : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2)
659 : "cc");
660 return;
661 }
662#endif
663
664#if defined(BOTAN_MP_USE_X86_64_ASM)
665 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
666 asm(R"(
667 addq %[x],%[w0]
668 adcq $0,%[w1]
669 adcq $0,%[w2]
670 )"
671 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
672 : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2)
673 : "cc");
674 return;
675 }
676#endif
677
678 *w0 += x;
679 W c1 = (*w0 < x);
680 *w1 += c1;
681 W c2 = (*w1 < c1);
682 *w2 += c2;
683}
684
685/*
686* Multiply-Add Accumulator
687* (w2,w1,w0) += 2 * x * y
688*/
689template <WordType W>
690inline constexpr void word3_muladd_2(W* w2, W* w1, W* w0, W x, W y) {
691#if defined(BOTAN_MP_USE_X86_32_ASM)
692 if(std::same_as<W, uint32_t> && !std::is_constant_evaluated()) {
693 W z0 = 0, z1 = 0;
694
695 asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
696
697 asm(R"(
698 addl %[z0],%[w0]
699 adcl %[z1],%[w1]
700 adcl $0,%[w2]
701
702 addl %[z0],%[w0]
703 adcl %[z1],%[w1]
704 adcl $0,%[w2]
705 )"
706 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
707 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
708 : "cc");
709 return;
710 }
711#endif
712
713#if defined(BOTAN_MP_USE_X86_64_ASM)
714 if(std::same_as<W, uint64_t> && !std::is_constant_evaluated()) {
715 W z0 = 0, z1 = 0;
716
717 asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc");
718
719 asm(R"(
720 addq %[z0],%[w0]
721 adcq %[z1],%[w1]
722 adcq $0,%[w2]
723
724 addq %[z0],%[w0]
725 adcq %[z1],%[w1]
726 adcq $0,%[w2]
727 )"
728 : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2)
729 : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2)
730 : "cc");
731 return;
732 }
733#endif
734
735 W carry = 0;
736 x = word_madd2(x, y, &carry);
737 y = carry;
738
739 const size_t top_bit_shift = WordInfo<W>::bits - 1;
740
741 W top = (y >> top_bit_shift);
742 y <<= 1;
743 y |= (x >> top_bit_shift);
744 x <<= 1;
745
746 carry = 0;
747 *w0 = word_add(*w0, x, &carry);
748 *w1 = word_add(*w1, y, &carry);
749 *w2 = word_add(*w2, top, &carry);
750}
751
752#if defined(ASM)
753 #undef ASM
754 #undef DO_8_TIMES
755 #undef ADD_OR_SUBTRACT
756 #undef ADDSUB2_OP
757 #undef ADDSUB3_OP
758 #undef LINMUL_OP
759 #undef MULADD_OP
760#endif
761
762} // namespace Botan
763
764#endif
Botan::word8_sub3
constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:456
Botan::word_sub
constexpr auto word_sub(W x, W y, W *carry) -> W
Definition mp_asmi.h:352
Botan::word_add
constexpr auto word_add(W x, W y, W *carry) -> W
Definition mp_asmi.h:248
Botan::word8_madd3
constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:564
Botan::word8_add3
constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:316
Botan::word8_sub2
constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:384
Botan::word_madd2
constexpr auto word_madd2(W a, W b, W *c) -> W
Definition mp_asmi.h:70
Botan::word8_sub2_rev
constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:420
Botan::word3_add
constexpr void word3_add(W *w2, W *w1, W *w0, W x)
Definition mp_asmi.h:649
Botan::word3_muladd_2
constexpr void word3_muladd_2(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:690
Botan::word3_muladd
constexpr void word3_muladd(W *w2, W *w1, W *w0, W x, W y)
Definition mp_asmi.h:601
Botan::carry
void carry(int64_t &h0, int64_t &h1)
Definition ed25519_internal.h:28
Botan::word8_linmul2
constexpr auto word8_linmul2(W x[8], W y, W carry) -> W
Definition mp_asmi.h:492
Botan::word8_add2
constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W
Definition mp_asmi.h:280
Botan::word8_linmul3
constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W
Definition mp_asmi.h:528
Botan::word_madd3
constexpr auto word_madd3(W a, W b, W c, W *d) -> W
Definition mp_asmi.h:111
Botan::WordInfo< uint32_t >::dword
uint64_t dword
Definition mp_asmi.h:46
Botan::WordInfo< uint64_t >::dword
donna128 dword
Definition mp_asmi.h:61
Generated by doxygen 1.10.0