Botan 3.14.1
Crypto and TLS for C&
mp_mulx.cpp
Go to the documentation of this file.
1/*
2* Multiplication and squaring using MULX with ADCX/ADOX carry chains
3*
4* This file was automatically generated by ./src/scripts/dev_tools/gen_mp_mulx.py on 2026-10-13
5* All manual changes will be lost. Edit the script instead.
6*
7* Botan is released under the Simplified BSD License (see license.txt)
8*/
9
10#include <botan/internal/mp_mulx.h>
11
12#include <botan/mem_ops.h>
13#include <botan/internal/mp_asmi.h>
14
15namespace Botan {
16
17/*
18* Each row multiplies x by one word and adds it into a window of accumulator
19* registers. The row's own carries (high half of one product into the low
20* half of the next) run on the overflow flag via ADOX, while adding the row
21* into the accumulator runs on the carry flag via ADCX; the two chains do not
22* wait on each other. The lowest word of a row is final after its first
23* product, so it is stored at once and its register takes the row's top word.
24*/
25
26// NOLINTBEGIN(*-no-assembler)
27
28void bigint_mulx_sqr4(word z[8], const word x[4]) {
29 // clang-format off
30 asm(
31 "xorl %%r9d, %%r9d\n\t"
32 "xorl %%r10d, %%r10d\n\t"
33 "xorl %%r8d, %%r8d\n\t"
34
35 "movq 0(%[x]), %%rdx\n\t" // x[0] * x[1..3]
36 "xorl %%eax, %%eax\n\t"
37 "mulx 8(%[x]), %%rax, %%rcx\n\t"
38 "adcx %%rax, %%r9\n\t"
39 "movq %%r9, 8(%[z])\n\t"
40 "mulx 16(%[x]), %%rax, %%r9\n\t"
41 "adox %%rcx, %%rax\n\t"
42 "adcx %%rax, %%r10\n\t"
43 "mulx 24(%[x]), %%rax, %%rcx\n\t"
44 "adox %%r9, %%rax\n\t"
45 "adcx %%rax, %%r8\n\t"
46 "movl $0, %%eax\n\t"
47 "adox %%rax, %%rcx\n\t"
48 "adcx %%rax, %%rcx\n\t"
49 "movq %%rcx, %%r9\n\t"
50 "movq %%r10, 16(%[z])\n\t"
51
52 "movq 8(%[x]), %%rdx\n\t" // x[1] * x[2..3]
53 "xorl %%eax, %%eax\n\t"
54 "mulx 16(%[x]), %%rax, %%rcx\n\t"
55 "adcx %%rax, %%r8\n\t"
56 "movq %%r8, 24(%[z])\n\t"
57 "mulx 24(%[x]), %%rax, %%r10\n\t"
58 "adox %%rcx, %%rax\n\t"
59 "adcx %%rax, %%r9\n\t"
60 "movl $0, %%eax\n\t"
61 "adox %%rax, %%r10\n\t"
62 "adcx %%rax, %%r10\n\t"
63 "movq %%r9, 32(%[z])\n\t"
64
65 "movq 16(%[x]), %%rdx\n\t" // x[2] * x[3..3]
66 "xorl %%eax, %%eax\n\t"
67 "mulx 24(%[x]), %%rax, %%rcx\n\t"
68 "adcx %%rax, %%r10\n\t"
69 "movq %%r10, 40(%[z])\n\t"
70 "movl $0, %%eax\n\t"
71 "adox %%rax, %%rcx\n\t"
72 "adcx %%rax, %%rcx\n\t"
73 "movq %%rcx, %%r8\n\t"
74 "movq %%r8, 48(%[z])\n\t"
75
76 "movq $0, 56(%[z])\n\t"
77 "xorl %%eax, %%eax\n\t" // double the cross products and add the squares
78 "movq 0(%[x]), %%rdx\n\t"
79 "mulx %%rdx, %%rax, %%rcx\n\t"
80 "movq %%rax, 0(%[z])\n\t"
81 "movq 8(%[z]), %%r8\n\t"
82 "adcx %%r8, %%r8\n\t"
83 "adox %%rcx, %%r8\n\t"
84 "movq %%r8, 8(%[z])\n\t"
85 "movq 8(%[x]), %%rdx\n\t"
86 "mulx %%rdx, %%rax, %%rcx\n\t"
87 "movq 16(%[z]), %%r8\n\t"
88 "adcx %%r8, %%r8\n\t"
89 "adox %%rax, %%r8\n\t"
90 "movq %%r8, 16(%[z])\n\t"
91 "movq 24(%[z]), %%r8\n\t"
92 "adcx %%r8, %%r8\n\t"
93 "adox %%rcx, %%r8\n\t"
94 "movq %%r8, 24(%[z])\n\t"
95 "movq 16(%[x]), %%rdx\n\t"
96 "mulx %%rdx, %%rax, %%rcx\n\t"
97 "movq 32(%[z]), %%r8\n\t"
98 "adcx %%r8, %%r8\n\t"
99 "adox %%rax, %%r8\n\t"
100 "movq %%r8, 32(%[z])\n\t"
101 "movq 40(%[z]), %%r8\n\t"
102 "adcx %%r8, %%r8\n\t"
103 "adox %%rcx, %%r8\n\t"
104 "movq %%r8, 40(%[z])\n\t"
105 "movq 24(%[x]), %%rdx\n\t"
106 "mulx %%rdx, %%rax, %%rcx\n\t"
107 "movq 48(%[z]), %%r8\n\t"
108 "adcx %%r8, %%r8\n\t"
109 "adox %%rax, %%r8\n\t"
110 "movq %%r8, 48(%[z])\n\t"
111 "movq 56(%[z]), %%r8\n\t"
112 "adcx %%r8, %%r8\n\t"
113 "adox %%rcx, %%r8\n\t"
114 "movq %%r8, 56(%[z])\n\t"
115 :
116 : [z] "r"(z), [x] "r"(x)
117 : "rax", "rcx", "rdx", "r8", "r9", "r10", "cc", "memory");
118 // clang-format on
120}
121
122void bigint_mulx_mul4(word z[8], const word x[4], const word y[4]) {
123 // clang-format off
124 asm(
125 "xorl %%r8d, %%r8d\n\t"
126 "xorl %%r9d, %%r9d\n\t"
127 "xorl %%r10d, %%r10d\n\t"
128 "xorl %%r11d, %%r11d\n\t"
129
130 "movq 0(%[y]), %%rdx\n\t" // row 0: x * y[0]
131 "xorl %%eax, %%eax\n\t"
132 "mulx 0(%[x]), %%rax, %%rcx\n\t"
133 "adcx %%rax, %%r8\n\t"
134 "movq %%r8, 0(%[z])\n\t"
135 "mulx 8(%[x]), %%rax, %%r8\n\t"
136 "adox %%rcx, %%rax\n\t"
137 "adcx %%rax, %%r9\n\t"
138 "mulx 16(%[x]), %%rax, %%rcx\n\t"
139 "adox %%r8, %%rax\n\t"
140 "adcx %%rax, %%r10\n\t"
141 "mulx 24(%[x]), %%rax, %%r8\n\t"
142 "adox %%rcx, %%rax\n\t"
143 "adcx %%rax, %%r11\n\t"
144 "movl $0, %%eax\n\t"
145 "adox %%rax, %%r8\n\t"
146 "adcx %%rax, %%r8\n\t"
147
148 "movq 8(%[y]), %%rdx\n\t" // row 1: x * y[1]
149 "xorl %%eax, %%eax\n\t"
150 "mulx 0(%[x]), %%rax, %%rcx\n\t"
151 "adcx %%rax, %%r9\n\t"
152 "movq %%r9, 8(%[z])\n\t"
153 "mulx 8(%[x]), %%rax, %%r9\n\t"
154 "adox %%rcx, %%rax\n\t"
155 "adcx %%rax, %%r10\n\t"
156 "mulx 16(%[x]), %%rax, %%rcx\n\t"
157 "adox %%r9, %%rax\n\t"
158 "adcx %%rax, %%r11\n\t"
159 "mulx 24(%[x]), %%rax, %%r9\n\t"
160 "adox %%rcx, %%rax\n\t"
161 "adcx %%rax, %%r8\n\t"
162 "movl $0, %%eax\n\t"
163 "adox %%rax, %%r9\n\t"
164 "adcx %%rax, %%r9\n\t"
165
166 "movq 16(%[y]), %%rdx\n\t" // row 2: x * y[2]
167 "xorl %%eax, %%eax\n\t"
168 "mulx 0(%[x]), %%rax, %%rcx\n\t"
169 "adcx %%rax, %%r10\n\t"
170 "movq %%r10, 16(%[z])\n\t"
171 "mulx 8(%[x]), %%rax, %%r10\n\t"
172 "adox %%rcx, %%rax\n\t"
173 "adcx %%rax, %%r11\n\t"
174 "mulx 16(%[x]), %%rax, %%rcx\n\t"
175 "adox %%r10, %%rax\n\t"
176 "adcx %%rax, %%r8\n\t"
177 "mulx 24(%[x]), %%rax, %%r10\n\t"
178 "adox %%rcx, %%rax\n\t"
179 "adcx %%rax, %%r9\n\t"
180 "movl $0, %%eax\n\t"
181 "adox %%rax, %%r10\n\t"
182 "adcx %%rax, %%r10\n\t"
183
184 "movq 24(%[y]), %%rdx\n\t" // row 3: x * y[3]
185 "xorl %%eax, %%eax\n\t"
186 "mulx 0(%[x]), %%rax, %%rcx\n\t"
187 "adcx %%rax, %%r11\n\t"
188 "movq %%r11, 24(%[z])\n\t"
189 "mulx 8(%[x]), %%rax, %%r11\n\t"
190 "adox %%rcx, %%rax\n\t"
191 "adcx %%rax, %%r8\n\t"
192 "mulx 16(%[x]), %%rax, %%rcx\n\t"
193 "adox %%r11, %%rax\n\t"
194 "adcx %%rax, %%r9\n\t"
195 "mulx 24(%[x]), %%rax, %%r11\n\t"
196 "adox %%rcx, %%rax\n\t"
197 "adcx %%rax, %%r10\n\t"
198 "movl $0, %%eax\n\t"
199 "adox %%rax, %%r11\n\t"
200 "adcx %%rax, %%r11\n\t"
201
202 "movq %%r8, 32(%[z])\n\t"
203 "movq %%r9, 40(%[z])\n\t"
204 "movq %%r10, 48(%[z])\n\t"
205 "movq %%r11, 56(%[z])\n\t"
206 :
207 : [z] "r"(z), [x] "r"(x), [y] "r"(y)
208 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
209 // clang-format on
211}
212
213void bigint_mulx_sqr6(word z[12], const word x[6]) {
214 // clang-format off
215 asm(
216 "xorl %%r9d, %%r9d\n\t"
217 "xorl %%r10d, %%r10d\n\t"
218 "xorl %%r11d, %%r11d\n\t"
219 "xorl %%r12d, %%r12d\n\t"
220 "xorl %%r8d, %%r8d\n\t"
221
222 "movq 0(%[x]), %%rdx\n\t" // x[0] * x[1..5]
223 "xorl %%eax, %%eax\n\t"
224 "mulx 8(%[x]), %%rax, %%rcx\n\t"
225 "adcx %%rax, %%r9\n\t"
226 "movq %%r9, 8(%[z])\n\t"
227 "mulx 16(%[x]), %%rax, %%r9\n\t"
228 "adox %%rcx, %%rax\n\t"
229 "adcx %%rax, %%r10\n\t"
230 "mulx 24(%[x]), %%rax, %%rcx\n\t"
231 "adox %%r9, %%rax\n\t"
232 "adcx %%rax, %%r11\n\t"
233 "mulx 32(%[x]), %%rax, %%r9\n\t"
234 "adox %%rcx, %%rax\n\t"
235 "adcx %%rax, %%r12\n\t"
236 "mulx 40(%[x]), %%rax, %%rcx\n\t"
237 "adox %%r9, %%rax\n\t"
238 "adcx %%rax, %%r8\n\t"
239 "movl $0, %%eax\n\t"
240 "adox %%rax, %%rcx\n\t"
241 "adcx %%rax, %%rcx\n\t"
242 "movq %%rcx, %%r9\n\t"
243 "movq %%r10, 16(%[z])\n\t"
244
245 "movq 8(%[x]), %%rdx\n\t" // x[1] * x[2..5]
246 "xorl %%eax, %%eax\n\t"
247 "mulx 16(%[x]), %%rax, %%rcx\n\t"
248 "adcx %%rax, %%r11\n\t"
249 "movq %%r11, 24(%[z])\n\t"
250 "mulx 24(%[x]), %%rax, %%r10\n\t"
251 "adox %%rcx, %%rax\n\t"
252 "adcx %%rax, %%r12\n\t"
253 "mulx 32(%[x]), %%rax, %%rcx\n\t"
254 "adox %%r10, %%rax\n\t"
255 "adcx %%rax, %%r8\n\t"
256 "mulx 40(%[x]), %%rax, %%r10\n\t"
257 "adox %%rcx, %%rax\n\t"
258 "adcx %%rax, %%r9\n\t"
259 "movl $0, %%eax\n\t"
260 "adox %%rax, %%r10\n\t"
261 "adcx %%rax, %%r10\n\t"
262 "movq %%r12, 32(%[z])\n\t"
263
264 "movq 16(%[x]), %%rdx\n\t" // x[2] * x[3..5]
265 "xorl %%eax, %%eax\n\t"
266 "mulx 24(%[x]), %%rax, %%rcx\n\t"
267 "adcx %%rax, %%r8\n\t"
268 "movq %%r8, 40(%[z])\n\t"
269 "mulx 32(%[x]), %%rax, %%r11\n\t"
270 "adox %%rcx, %%rax\n\t"
271 "adcx %%rax, %%r9\n\t"
272 "mulx 40(%[x]), %%rax, %%rcx\n\t"
273 "adox %%r11, %%rax\n\t"
274 "adcx %%rax, %%r10\n\t"
275 "movl $0, %%eax\n\t"
276 "adox %%rax, %%rcx\n\t"
277 "adcx %%rax, %%rcx\n\t"
278 "movq %%rcx, %%r11\n\t"
279 "movq %%r9, 48(%[z])\n\t"
280
281 "movq 24(%[x]), %%rdx\n\t" // x[3] * x[4..5]
282 "xorl %%eax, %%eax\n\t"
283 "mulx 32(%[x]), %%rax, %%rcx\n\t"
284 "adcx %%rax, %%r10\n\t"
285 "movq %%r10, 56(%[z])\n\t"
286 "mulx 40(%[x]), %%rax, %%r12\n\t"
287 "adox %%rcx, %%rax\n\t"
288 "adcx %%rax, %%r11\n\t"
289 "movl $0, %%eax\n\t"
290 "adox %%rax, %%r12\n\t"
291 "adcx %%rax, %%r12\n\t"
292 "movq %%r11, 64(%[z])\n\t"
293
294 "movq 32(%[x]), %%rdx\n\t" // x[4] * x[5..5]
295 "xorl %%eax, %%eax\n\t"
296 "mulx 40(%[x]), %%rax, %%rcx\n\t"
297 "adcx %%rax, %%r12\n\t"
298 "movq %%r12, 72(%[z])\n\t"
299 "movl $0, %%eax\n\t"
300 "adox %%rax, %%rcx\n\t"
301 "adcx %%rax, %%rcx\n\t"
302 "movq %%rcx, %%r8\n\t"
303 "movq %%r8, 80(%[z])\n\t"
304
305 "movq $0, 88(%[z])\n\t"
306 "xorl %%eax, %%eax\n\t" // double the cross products and add the squares
307 "movq 0(%[x]), %%rdx\n\t"
308 "mulx %%rdx, %%rax, %%rcx\n\t"
309 "movq %%rax, 0(%[z])\n\t"
310 "movq 8(%[z]), %%r8\n\t"
311 "adcx %%r8, %%r8\n\t"
312 "adox %%rcx, %%r8\n\t"
313 "movq %%r8, 8(%[z])\n\t"
314 "movq 8(%[x]), %%rdx\n\t"
315 "mulx %%rdx, %%rax, %%rcx\n\t"
316 "movq 16(%[z]), %%r8\n\t"
317 "adcx %%r8, %%r8\n\t"
318 "adox %%rax, %%r8\n\t"
319 "movq %%r8, 16(%[z])\n\t"
320 "movq 24(%[z]), %%r8\n\t"
321 "adcx %%r8, %%r8\n\t"
322 "adox %%rcx, %%r8\n\t"
323 "movq %%r8, 24(%[z])\n\t"
324 "movq 16(%[x]), %%rdx\n\t"
325 "mulx %%rdx, %%rax, %%rcx\n\t"
326 "movq 32(%[z]), %%r8\n\t"
327 "adcx %%r8, %%r8\n\t"
328 "adox %%rax, %%r8\n\t"
329 "movq %%r8, 32(%[z])\n\t"
330 "movq 40(%[z]), %%r8\n\t"
331 "adcx %%r8, %%r8\n\t"
332 "adox %%rcx, %%r8\n\t"
333 "movq %%r8, 40(%[z])\n\t"
334 "movq 24(%[x]), %%rdx\n\t"
335 "mulx %%rdx, %%rax, %%rcx\n\t"
336 "movq 48(%[z]), %%r8\n\t"
337 "adcx %%r8, %%r8\n\t"
338 "adox %%rax, %%r8\n\t"
339 "movq %%r8, 48(%[z])\n\t"
340 "movq 56(%[z]), %%r8\n\t"
341 "adcx %%r8, %%r8\n\t"
342 "adox %%rcx, %%r8\n\t"
343 "movq %%r8, 56(%[z])\n\t"
344 "movq 32(%[x]), %%rdx\n\t"
345 "mulx %%rdx, %%rax, %%rcx\n\t"
346 "movq 64(%[z]), %%r8\n\t"
347 "adcx %%r8, %%r8\n\t"
348 "adox %%rax, %%r8\n\t"
349 "movq %%r8, 64(%[z])\n\t"
350 "movq 72(%[z]), %%r8\n\t"
351 "adcx %%r8, %%r8\n\t"
352 "adox %%rcx, %%r8\n\t"
353 "movq %%r8, 72(%[z])\n\t"
354 "movq 40(%[x]), %%rdx\n\t"
355 "mulx %%rdx, %%rax, %%rcx\n\t"
356 "movq 80(%[z]), %%r8\n\t"
357 "adcx %%r8, %%r8\n\t"
358 "adox %%rax, %%r8\n\t"
359 "movq %%r8, 80(%[z])\n\t"
360 "movq 88(%[z]), %%r8\n\t"
361 "adcx %%r8, %%r8\n\t"
362 "adox %%rcx, %%r8\n\t"
363 "movq %%r8, 88(%[z])\n\t"
364 :
365 : [z] "r"(z), [x] "r"(x)
366 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "cc", "memory");
367 // clang-format on
369}
370
371void bigint_mulx_mul6(word z[12], const word x[6], const word y[6]) {
372 // clang-format off
373 asm(
374 "xorl %%r8d, %%r8d\n\t"
375 "xorl %%r9d, %%r9d\n\t"
376 "xorl %%r10d, %%r10d\n\t"
377 "xorl %%r11d, %%r11d\n\t"
378 "xorl %%r12d, %%r12d\n\t"
379 "xorl %%r13d, %%r13d\n\t"
380
381 "movq 0(%[y]), %%rdx\n\t" // row 0: x * y[0]
382 "xorl %%eax, %%eax\n\t"
383 "mulx 0(%[x]), %%rax, %%rcx\n\t"
384 "adcx %%rax, %%r8\n\t"
385 "movq %%r8, 0(%[z])\n\t"
386 "mulx 8(%[x]), %%rax, %%r8\n\t"
387 "adox %%rcx, %%rax\n\t"
388 "adcx %%rax, %%r9\n\t"
389 "mulx 16(%[x]), %%rax, %%rcx\n\t"
390 "adox %%r8, %%rax\n\t"
391 "adcx %%rax, %%r10\n\t"
392 "mulx 24(%[x]), %%rax, %%r8\n\t"
393 "adox %%rcx, %%rax\n\t"
394 "adcx %%rax, %%r11\n\t"
395 "mulx 32(%[x]), %%rax, %%rcx\n\t"
396 "adox %%r8, %%rax\n\t"
397 "adcx %%rax, %%r12\n\t"
398 "mulx 40(%[x]), %%rax, %%r8\n\t"
399 "adox %%rcx, %%rax\n\t"
400 "adcx %%rax, %%r13\n\t"
401 "movl $0, %%eax\n\t"
402 "adox %%rax, %%r8\n\t"
403 "adcx %%rax, %%r8\n\t"
404
405 "movq 8(%[y]), %%rdx\n\t" // row 1: x * y[1]
406 "xorl %%eax, %%eax\n\t"
407 "mulx 0(%[x]), %%rax, %%rcx\n\t"
408 "adcx %%rax, %%r9\n\t"
409 "movq %%r9, 8(%[z])\n\t"
410 "mulx 8(%[x]), %%rax, %%r9\n\t"
411 "adox %%rcx, %%rax\n\t"
412 "adcx %%rax, %%r10\n\t"
413 "mulx 16(%[x]), %%rax, %%rcx\n\t"
414 "adox %%r9, %%rax\n\t"
415 "adcx %%rax, %%r11\n\t"
416 "mulx 24(%[x]), %%rax, %%r9\n\t"
417 "adox %%rcx, %%rax\n\t"
418 "adcx %%rax, %%r12\n\t"
419 "mulx 32(%[x]), %%rax, %%rcx\n\t"
420 "adox %%r9, %%rax\n\t"
421 "adcx %%rax, %%r13\n\t"
422 "mulx 40(%[x]), %%rax, %%r9\n\t"
423 "adox %%rcx, %%rax\n\t"
424 "adcx %%rax, %%r8\n\t"
425 "movl $0, %%eax\n\t"
426 "adox %%rax, %%r9\n\t"
427 "adcx %%rax, %%r9\n\t"
428
429 "movq 16(%[y]), %%rdx\n\t" // row 2: x * y[2]
430 "xorl %%eax, %%eax\n\t"
431 "mulx 0(%[x]), %%rax, %%rcx\n\t"
432 "adcx %%rax, %%r10\n\t"
433 "movq %%r10, 16(%[z])\n\t"
434 "mulx 8(%[x]), %%rax, %%r10\n\t"
435 "adox %%rcx, %%rax\n\t"
436 "adcx %%rax, %%r11\n\t"
437 "mulx 16(%[x]), %%rax, %%rcx\n\t"
438 "adox %%r10, %%rax\n\t"
439 "adcx %%rax, %%r12\n\t"
440 "mulx 24(%[x]), %%rax, %%r10\n\t"
441 "adox %%rcx, %%rax\n\t"
442 "adcx %%rax, %%r13\n\t"
443 "mulx 32(%[x]), %%rax, %%rcx\n\t"
444 "adox %%r10, %%rax\n\t"
445 "adcx %%rax, %%r8\n\t"
446 "mulx 40(%[x]), %%rax, %%r10\n\t"
447 "adox %%rcx, %%rax\n\t"
448 "adcx %%rax, %%r9\n\t"
449 "movl $0, %%eax\n\t"
450 "adox %%rax, %%r10\n\t"
451 "adcx %%rax, %%r10\n\t"
452
453 "movq 24(%[y]), %%rdx\n\t" // row 3: x * y[3]
454 "xorl %%eax, %%eax\n\t"
455 "mulx 0(%[x]), %%rax, %%rcx\n\t"
456 "adcx %%rax, %%r11\n\t"
457 "movq %%r11, 24(%[z])\n\t"
458 "mulx 8(%[x]), %%rax, %%r11\n\t"
459 "adox %%rcx, %%rax\n\t"
460 "adcx %%rax, %%r12\n\t"
461 "mulx 16(%[x]), %%rax, %%rcx\n\t"
462 "adox %%r11, %%rax\n\t"
463 "adcx %%rax, %%r13\n\t"
464 "mulx 24(%[x]), %%rax, %%r11\n\t"
465 "adox %%rcx, %%rax\n\t"
466 "adcx %%rax, %%r8\n\t"
467 "mulx 32(%[x]), %%rax, %%rcx\n\t"
468 "adox %%r11, %%rax\n\t"
469 "adcx %%rax, %%r9\n\t"
470 "mulx 40(%[x]), %%rax, %%r11\n\t"
471 "adox %%rcx, %%rax\n\t"
472 "adcx %%rax, %%r10\n\t"
473 "movl $0, %%eax\n\t"
474 "adox %%rax, %%r11\n\t"
475 "adcx %%rax, %%r11\n\t"
476
477 "movq 32(%[y]), %%rdx\n\t" // row 4: x * y[4]
478 "xorl %%eax, %%eax\n\t"
479 "mulx 0(%[x]), %%rax, %%rcx\n\t"
480 "adcx %%rax, %%r12\n\t"
481 "movq %%r12, 32(%[z])\n\t"
482 "mulx 8(%[x]), %%rax, %%r12\n\t"
483 "adox %%rcx, %%rax\n\t"
484 "adcx %%rax, %%r13\n\t"
485 "mulx 16(%[x]), %%rax, %%rcx\n\t"
486 "adox %%r12, %%rax\n\t"
487 "adcx %%rax, %%r8\n\t"
488 "mulx 24(%[x]), %%rax, %%r12\n\t"
489 "adox %%rcx, %%rax\n\t"
490 "adcx %%rax, %%r9\n\t"
491 "mulx 32(%[x]), %%rax, %%rcx\n\t"
492 "adox %%r12, %%rax\n\t"
493 "adcx %%rax, %%r10\n\t"
494 "mulx 40(%[x]), %%rax, %%r12\n\t"
495 "adox %%rcx, %%rax\n\t"
496 "adcx %%rax, %%r11\n\t"
497 "movl $0, %%eax\n\t"
498 "adox %%rax, %%r12\n\t"
499 "adcx %%rax, %%r12\n\t"
500
501 "movq 40(%[y]), %%rdx\n\t" // row 5: x * y[5]
502 "xorl %%eax, %%eax\n\t"
503 "mulx 0(%[x]), %%rax, %%rcx\n\t"
504 "adcx %%rax, %%r13\n\t"
505 "movq %%r13, 40(%[z])\n\t"
506 "mulx 8(%[x]), %%rax, %%r13\n\t"
507 "adox %%rcx, %%rax\n\t"
508 "adcx %%rax, %%r8\n\t"
509 "mulx 16(%[x]), %%rax, %%rcx\n\t"
510 "adox %%r13, %%rax\n\t"
511 "adcx %%rax, %%r9\n\t"
512 "mulx 24(%[x]), %%rax, %%r13\n\t"
513 "adox %%rcx, %%rax\n\t"
514 "adcx %%rax, %%r10\n\t"
515 "mulx 32(%[x]), %%rax, %%rcx\n\t"
516 "adox %%r13, %%rax\n\t"
517 "adcx %%rax, %%r11\n\t"
518 "mulx 40(%[x]), %%rax, %%r13\n\t"
519 "adox %%rcx, %%rax\n\t"
520 "adcx %%rax, %%r12\n\t"
521 "movl $0, %%eax\n\t"
522 "adox %%rax, %%r13\n\t"
523 "adcx %%rax, %%r13\n\t"
524
525 "movq %%r8, 48(%[z])\n\t"
526 "movq %%r9, 56(%[z])\n\t"
527 "movq %%r10, 64(%[z])\n\t"
528 "movq %%r11, 72(%[z])\n\t"
529 "movq %%r12, 80(%[z])\n\t"
530 "movq %%r13, 88(%[z])\n\t"
531 :
532 : [z] "r"(z), [x] "r"(x), [y] "r"(y)
533 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "cc", "memory");
534 // clang-format on
536}
537
538void bigint_mulx_sqr8(word z[16], const word x[8]) {
539 // clang-format off
540 asm(
541 "xorl %%r9d, %%r9d\n\t"
542 "xorl %%r10d, %%r10d\n\t"
543 "xorl %%r11d, %%r11d\n\t"
544 "xorl %%r12d, %%r12d\n\t"
545 "xorl %%r13d, %%r13d\n\t"
546 "xorl %%r14d, %%r14d\n\t"
547 "xorl %%r8d, %%r8d\n\t"
548
549 "movq 0(%[x]), %%rdx\n\t" // x[0] * x[1..7]
550 "xorl %%eax, %%eax\n\t"
551 "mulx 8(%[x]), %%rax, %%rcx\n\t"
552 "adcx %%rax, %%r9\n\t"
553 "movq %%r9, 8(%[z])\n\t"
554 "mulx 16(%[x]), %%rax, %%r9\n\t"
555 "adox %%rcx, %%rax\n\t"
556 "adcx %%rax, %%r10\n\t"
557 "mulx 24(%[x]), %%rax, %%rcx\n\t"
558 "adox %%r9, %%rax\n\t"
559 "adcx %%rax, %%r11\n\t"
560 "mulx 32(%[x]), %%rax, %%r9\n\t"
561 "adox %%rcx, %%rax\n\t"
562 "adcx %%rax, %%r12\n\t"
563 "mulx 40(%[x]), %%rax, %%rcx\n\t"
564 "adox %%r9, %%rax\n\t"
565 "adcx %%rax, %%r13\n\t"
566 "mulx 48(%[x]), %%rax, %%r9\n\t"
567 "adox %%rcx, %%rax\n\t"
568 "adcx %%rax, %%r14\n\t"
569 "mulx 56(%[x]), %%rax, %%rcx\n\t"
570 "adox %%r9, %%rax\n\t"
571 "adcx %%rax, %%r8\n\t"
572 "movl $0, %%eax\n\t"
573 "adox %%rax, %%rcx\n\t"
574 "adcx %%rax, %%rcx\n\t"
575 "movq %%rcx, %%r9\n\t"
576 "movq %%r10, 16(%[z])\n\t"
577
578 "movq 8(%[x]), %%rdx\n\t" // x[1] * x[2..7]
579 "xorl %%eax, %%eax\n\t"
580 "mulx 16(%[x]), %%rax, %%rcx\n\t"
581 "adcx %%rax, %%r11\n\t"
582 "movq %%r11, 24(%[z])\n\t"
583 "mulx 24(%[x]), %%rax, %%r10\n\t"
584 "adox %%rcx, %%rax\n\t"
585 "adcx %%rax, %%r12\n\t"
586 "mulx 32(%[x]), %%rax, %%rcx\n\t"
587 "adox %%r10, %%rax\n\t"
588 "adcx %%rax, %%r13\n\t"
589 "mulx 40(%[x]), %%rax, %%r10\n\t"
590 "adox %%rcx, %%rax\n\t"
591 "adcx %%rax, %%r14\n\t"
592 "mulx 48(%[x]), %%rax, %%rcx\n\t"
593 "adox %%r10, %%rax\n\t"
594 "adcx %%rax, %%r8\n\t"
595 "mulx 56(%[x]), %%rax, %%r10\n\t"
596 "adox %%rcx, %%rax\n\t"
597 "adcx %%rax, %%r9\n\t"
598 "movl $0, %%eax\n\t"
599 "adox %%rax, %%r10\n\t"
600 "adcx %%rax, %%r10\n\t"
601 "movq %%r12, 32(%[z])\n\t"
602
603 "movq 16(%[x]), %%rdx\n\t" // x[2] * x[3..7]
604 "xorl %%eax, %%eax\n\t"
605 "mulx 24(%[x]), %%rax, %%rcx\n\t"
606 "adcx %%rax, %%r13\n\t"
607 "movq %%r13, 40(%[z])\n\t"
608 "mulx 32(%[x]), %%rax, %%r11\n\t"
609 "adox %%rcx, %%rax\n\t"
610 "adcx %%rax, %%r14\n\t"
611 "mulx 40(%[x]), %%rax, %%rcx\n\t"
612 "adox %%r11, %%rax\n\t"
613 "adcx %%rax, %%r8\n\t"
614 "mulx 48(%[x]), %%rax, %%r11\n\t"
615 "adox %%rcx, %%rax\n\t"
616 "adcx %%rax, %%r9\n\t"
617 "mulx 56(%[x]), %%rax, %%rcx\n\t"
618 "adox %%r11, %%rax\n\t"
619 "adcx %%rax, %%r10\n\t"
620 "movl $0, %%eax\n\t"
621 "adox %%rax, %%rcx\n\t"
622 "adcx %%rax, %%rcx\n\t"
623 "movq %%rcx, %%r11\n\t"
624 "movq %%r14, 48(%[z])\n\t"
625
626 "movq 24(%[x]), %%rdx\n\t" // x[3] * x[4..7]
627 "xorl %%eax, %%eax\n\t"
628 "mulx 32(%[x]), %%rax, %%rcx\n\t"
629 "adcx %%rax, %%r8\n\t"
630 "movq %%r8, 56(%[z])\n\t"
631 "mulx 40(%[x]), %%rax, %%r12\n\t"
632 "adox %%rcx, %%rax\n\t"
633 "adcx %%rax, %%r9\n\t"
634 "mulx 48(%[x]), %%rax, %%rcx\n\t"
635 "adox %%r12, %%rax\n\t"
636 "adcx %%rax, %%r10\n\t"
637 "mulx 56(%[x]), %%rax, %%r12\n\t"
638 "adox %%rcx, %%rax\n\t"
639 "adcx %%rax, %%r11\n\t"
640 "movl $0, %%eax\n\t"
641 "adox %%rax, %%r12\n\t"
642 "adcx %%rax, %%r12\n\t"
643 "movq %%r9, 64(%[z])\n\t"
644
645 "movq 32(%[x]), %%rdx\n\t" // x[4] * x[5..7]
646 "xorl %%eax, %%eax\n\t"
647 "mulx 40(%[x]), %%rax, %%rcx\n\t"
648 "adcx %%rax, %%r10\n\t"
649 "movq %%r10, 72(%[z])\n\t"
650 "mulx 48(%[x]), %%rax, %%r13\n\t"
651 "adox %%rcx, %%rax\n\t"
652 "adcx %%rax, %%r11\n\t"
653 "mulx 56(%[x]), %%rax, %%rcx\n\t"
654 "adox %%r13, %%rax\n\t"
655 "adcx %%rax, %%r12\n\t"
656 "movl $0, %%eax\n\t"
657 "adox %%rax, %%rcx\n\t"
658 "adcx %%rax, %%rcx\n\t"
659 "movq %%rcx, %%r13\n\t"
660 "movq %%r11, 80(%[z])\n\t"
661
662 "movq 40(%[x]), %%rdx\n\t" // x[5] * x[6..7]
663 "xorl %%eax, %%eax\n\t"
664 "mulx 48(%[x]), %%rax, %%rcx\n\t"
665 "adcx %%rax, %%r12\n\t"
666 "movq %%r12, 88(%[z])\n\t"
667 "mulx 56(%[x]), %%rax, %%r14\n\t"
668 "adox %%rcx, %%rax\n\t"
669 "adcx %%rax, %%r13\n\t"
670 "movl $0, %%eax\n\t"
671 "adox %%rax, %%r14\n\t"
672 "adcx %%rax, %%r14\n\t"
673 "movq %%r13, 96(%[z])\n\t"
674
675 "movq 48(%[x]), %%rdx\n\t" // x[6] * x[7..7]
676 "xorl %%eax, %%eax\n\t"
677 "mulx 56(%[x]), %%rax, %%rcx\n\t"
678 "adcx %%rax, %%r14\n\t"
679 "movq %%r14, 104(%[z])\n\t"
680 "movl $0, %%eax\n\t"
681 "adox %%rax, %%rcx\n\t"
682 "adcx %%rax, %%rcx\n\t"
683 "movq %%rcx, %%r8\n\t"
684 "movq %%r8, 112(%[z])\n\t"
685
686 "movq $0, 120(%[z])\n\t"
687 "xorl %%eax, %%eax\n\t" // double the cross products and add the squares
688 "movq 0(%[x]), %%rdx\n\t"
689 "mulx %%rdx, %%rax, %%rcx\n\t"
690 "movq %%rax, 0(%[z])\n\t"
691 "movq 8(%[z]), %%r8\n\t"
692 "adcx %%r8, %%r8\n\t"
693 "adox %%rcx, %%r8\n\t"
694 "movq %%r8, 8(%[z])\n\t"
695 "movq 8(%[x]), %%rdx\n\t"
696 "mulx %%rdx, %%rax, %%rcx\n\t"
697 "movq 16(%[z]), %%r8\n\t"
698 "adcx %%r8, %%r8\n\t"
699 "adox %%rax, %%r8\n\t"
700 "movq %%r8, 16(%[z])\n\t"
701 "movq 24(%[z]), %%r8\n\t"
702 "adcx %%r8, %%r8\n\t"
703 "adox %%rcx, %%r8\n\t"
704 "movq %%r8, 24(%[z])\n\t"
705 "movq 16(%[x]), %%rdx\n\t"
706 "mulx %%rdx, %%rax, %%rcx\n\t"
707 "movq 32(%[z]), %%r8\n\t"
708 "adcx %%r8, %%r8\n\t"
709 "adox %%rax, %%r8\n\t"
710 "movq %%r8, 32(%[z])\n\t"
711 "movq 40(%[z]), %%r8\n\t"
712 "adcx %%r8, %%r8\n\t"
713 "adox %%rcx, %%r8\n\t"
714 "movq %%r8, 40(%[z])\n\t"
715 "movq 24(%[x]), %%rdx\n\t"
716 "mulx %%rdx, %%rax, %%rcx\n\t"
717 "movq 48(%[z]), %%r8\n\t"
718 "adcx %%r8, %%r8\n\t"
719 "adox %%rax, %%r8\n\t"
720 "movq %%r8, 48(%[z])\n\t"
721 "movq 56(%[z]), %%r8\n\t"
722 "adcx %%r8, %%r8\n\t"
723 "adox %%rcx, %%r8\n\t"
724 "movq %%r8, 56(%[z])\n\t"
725 "movq 32(%[x]), %%rdx\n\t"
726 "mulx %%rdx, %%rax, %%rcx\n\t"
727 "movq 64(%[z]), %%r8\n\t"
728 "adcx %%r8, %%r8\n\t"
729 "adox %%rax, %%r8\n\t"
730 "movq %%r8, 64(%[z])\n\t"
731 "movq 72(%[z]), %%r8\n\t"
732 "adcx %%r8, %%r8\n\t"
733 "adox %%rcx, %%r8\n\t"
734 "movq %%r8, 72(%[z])\n\t"
735 "movq 40(%[x]), %%rdx\n\t"
736 "mulx %%rdx, %%rax, %%rcx\n\t"
737 "movq 80(%[z]), %%r8\n\t"
738 "adcx %%r8, %%r8\n\t"
739 "adox %%rax, %%r8\n\t"
740 "movq %%r8, 80(%[z])\n\t"
741 "movq 88(%[z]), %%r8\n\t"
742 "adcx %%r8, %%r8\n\t"
743 "adox %%rcx, %%r8\n\t"
744 "movq %%r8, 88(%[z])\n\t"
745 "movq 48(%[x]), %%rdx\n\t"
746 "mulx %%rdx, %%rax, %%rcx\n\t"
747 "movq 96(%[z]), %%r8\n\t"
748 "adcx %%r8, %%r8\n\t"
749 "adox %%rax, %%r8\n\t"
750 "movq %%r8, 96(%[z])\n\t"
751 "movq 104(%[z]), %%r8\n\t"
752 "adcx %%r8, %%r8\n\t"
753 "adox %%rcx, %%r8\n\t"
754 "movq %%r8, 104(%[z])\n\t"
755 "movq 56(%[x]), %%rdx\n\t"
756 "mulx %%rdx, %%rax, %%rcx\n\t"
757 "movq 112(%[z]), %%r8\n\t"
758 "adcx %%r8, %%r8\n\t"
759 "adox %%rax, %%r8\n\t"
760 "movq %%r8, 112(%[z])\n\t"
761 "movq 120(%[z]), %%r8\n\t"
762 "adcx %%r8, %%r8\n\t"
763 "adox %%rcx, %%r8\n\t"
764 "movq %%r8, 120(%[z])\n\t"
765 :
766 : [z] "r"(z), [x] "r"(x)
767 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "cc", "memory");
768 // clang-format on
770}
771
772void bigint_mulx_mul8(word z[16], const word x[8], const word y[8]) {
773 // clang-format off
774 asm(
775 "xorl %%r8d, %%r8d\n\t"
776 "xorl %%r9d, %%r9d\n\t"
777 "xorl %%r10d, %%r10d\n\t"
778 "xorl %%r11d, %%r11d\n\t"
779 "xorl %%r12d, %%r12d\n\t"
780 "xorl %%r13d, %%r13d\n\t"
781 "xorl %%r14d, %%r14d\n\t"
782 "xorl %%r15d, %%r15d\n\t"
783
784 "movq 0(%[y]), %%rdx\n\t" // row 0: x * y[0]
785 "xorl %%eax, %%eax\n\t"
786 "mulx 0(%[x]), %%rax, %%rcx\n\t"
787 "adcx %%rax, %%r8\n\t"
788 "movq %%r8, 0(%[z])\n\t"
789 "mulx 8(%[x]), %%rax, %%r8\n\t"
790 "adox %%rcx, %%rax\n\t"
791 "adcx %%rax, %%r9\n\t"
792 "mulx 16(%[x]), %%rax, %%rcx\n\t"
793 "adox %%r8, %%rax\n\t"
794 "adcx %%rax, %%r10\n\t"
795 "mulx 24(%[x]), %%rax, %%r8\n\t"
796 "adox %%rcx, %%rax\n\t"
797 "adcx %%rax, %%r11\n\t"
798 "mulx 32(%[x]), %%rax, %%rcx\n\t"
799 "adox %%r8, %%rax\n\t"
800 "adcx %%rax, %%r12\n\t"
801 "mulx 40(%[x]), %%rax, %%r8\n\t"
802 "adox %%rcx, %%rax\n\t"
803 "adcx %%rax, %%r13\n\t"
804 "mulx 48(%[x]), %%rax, %%rcx\n\t"
805 "adox %%r8, %%rax\n\t"
806 "adcx %%rax, %%r14\n\t"
807 "mulx 56(%[x]), %%rax, %%r8\n\t"
808 "adox %%rcx, %%rax\n\t"
809 "adcx %%rax, %%r15\n\t"
810 "movl $0, %%eax\n\t"
811 "adox %%rax, %%r8\n\t"
812 "adcx %%rax, %%r8\n\t"
813
814 "movq 8(%[y]), %%rdx\n\t" // row 1: x * y[1]
815 "xorl %%eax, %%eax\n\t"
816 "mulx 0(%[x]), %%rax, %%rcx\n\t"
817 "adcx %%rax, %%r9\n\t"
818 "movq %%r9, 8(%[z])\n\t"
819 "mulx 8(%[x]), %%rax, %%r9\n\t"
820 "adox %%rcx, %%rax\n\t"
821 "adcx %%rax, %%r10\n\t"
822 "mulx 16(%[x]), %%rax, %%rcx\n\t"
823 "adox %%r9, %%rax\n\t"
824 "adcx %%rax, %%r11\n\t"
825 "mulx 24(%[x]), %%rax, %%r9\n\t"
826 "adox %%rcx, %%rax\n\t"
827 "adcx %%rax, %%r12\n\t"
828 "mulx 32(%[x]), %%rax, %%rcx\n\t"
829 "adox %%r9, %%rax\n\t"
830 "adcx %%rax, %%r13\n\t"
831 "mulx 40(%[x]), %%rax, %%r9\n\t"
832 "adox %%rcx, %%rax\n\t"
833 "adcx %%rax, %%r14\n\t"
834 "mulx 48(%[x]), %%rax, %%rcx\n\t"
835 "adox %%r9, %%rax\n\t"
836 "adcx %%rax, %%r15\n\t"
837 "mulx 56(%[x]), %%rax, %%r9\n\t"
838 "adox %%rcx, %%rax\n\t"
839 "adcx %%rax, %%r8\n\t"
840 "movl $0, %%eax\n\t"
841 "adox %%rax, %%r9\n\t"
842 "adcx %%rax, %%r9\n\t"
843
844 "movq 16(%[y]), %%rdx\n\t" // row 2: x * y[2]
845 "xorl %%eax, %%eax\n\t"
846 "mulx 0(%[x]), %%rax, %%rcx\n\t"
847 "adcx %%rax, %%r10\n\t"
848 "movq %%r10, 16(%[z])\n\t"
849 "mulx 8(%[x]), %%rax, %%r10\n\t"
850 "adox %%rcx, %%rax\n\t"
851 "adcx %%rax, %%r11\n\t"
852 "mulx 16(%[x]), %%rax, %%rcx\n\t"
853 "adox %%r10, %%rax\n\t"
854 "adcx %%rax, %%r12\n\t"
855 "mulx 24(%[x]), %%rax, %%r10\n\t"
856 "adox %%rcx, %%rax\n\t"
857 "adcx %%rax, %%r13\n\t"
858 "mulx 32(%[x]), %%rax, %%rcx\n\t"
859 "adox %%r10, %%rax\n\t"
860 "adcx %%rax, %%r14\n\t"
861 "mulx 40(%[x]), %%rax, %%r10\n\t"
862 "adox %%rcx, %%rax\n\t"
863 "adcx %%rax, %%r15\n\t"
864 "mulx 48(%[x]), %%rax, %%rcx\n\t"
865 "adox %%r10, %%rax\n\t"
866 "adcx %%rax, %%r8\n\t"
867 "mulx 56(%[x]), %%rax, %%r10\n\t"
868 "adox %%rcx, %%rax\n\t"
869 "adcx %%rax, %%r9\n\t"
870 "movl $0, %%eax\n\t"
871 "adox %%rax, %%r10\n\t"
872 "adcx %%rax, %%r10\n\t"
873
874 "movq 24(%[y]), %%rdx\n\t" // row 3: x * y[3]
875 "xorl %%eax, %%eax\n\t"
876 "mulx 0(%[x]), %%rax, %%rcx\n\t"
877 "adcx %%rax, %%r11\n\t"
878 "movq %%r11, 24(%[z])\n\t"
879 "mulx 8(%[x]), %%rax, %%r11\n\t"
880 "adox %%rcx, %%rax\n\t"
881 "adcx %%rax, %%r12\n\t"
882 "mulx 16(%[x]), %%rax, %%rcx\n\t"
883 "adox %%r11, %%rax\n\t"
884 "adcx %%rax, %%r13\n\t"
885 "mulx 24(%[x]), %%rax, %%r11\n\t"
886 "adox %%rcx, %%rax\n\t"
887 "adcx %%rax, %%r14\n\t"
888 "mulx 32(%[x]), %%rax, %%rcx\n\t"
889 "adox %%r11, %%rax\n\t"
890 "adcx %%rax, %%r15\n\t"
891 "mulx 40(%[x]), %%rax, %%r11\n\t"
892 "adox %%rcx, %%rax\n\t"
893 "adcx %%rax, %%r8\n\t"
894 "mulx 48(%[x]), %%rax, %%rcx\n\t"
895 "adox %%r11, %%rax\n\t"
896 "adcx %%rax, %%r9\n\t"
897 "mulx 56(%[x]), %%rax, %%r11\n\t"
898 "adox %%rcx, %%rax\n\t"
899 "adcx %%rax, %%r10\n\t"
900 "movl $0, %%eax\n\t"
901 "adox %%rax, %%r11\n\t"
902 "adcx %%rax, %%r11\n\t"
903
904 "movq 32(%[y]), %%rdx\n\t" // row 4: x * y[4]
905 "xorl %%eax, %%eax\n\t"
906 "mulx 0(%[x]), %%rax, %%rcx\n\t"
907 "adcx %%rax, %%r12\n\t"
908 "movq %%r12, 32(%[z])\n\t"
909 "mulx 8(%[x]), %%rax, %%r12\n\t"
910 "adox %%rcx, %%rax\n\t"
911 "adcx %%rax, %%r13\n\t"
912 "mulx 16(%[x]), %%rax, %%rcx\n\t"
913 "adox %%r12, %%rax\n\t"
914 "adcx %%rax, %%r14\n\t"
915 "mulx 24(%[x]), %%rax, %%r12\n\t"
916 "adox %%rcx, %%rax\n\t"
917 "adcx %%rax, %%r15\n\t"
918 "mulx 32(%[x]), %%rax, %%rcx\n\t"
919 "adox %%r12, %%rax\n\t"
920 "adcx %%rax, %%r8\n\t"
921 "mulx 40(%[x]), %%rax, %%r12\n\t"
922 "adox %%rcx, %%rax\n\t"
923 "adcx %%rax, %%r9\n\t"
924 "mulx 48(%[x]), %%rax, %%rcx\n\t"
925 "adox %%r12, %%rax\n\t"
926 "adcx %%rax, %%r10\n\t"
927 "mulx 56(%[x]), %%rax, %%r12\n\t"
928 "adox %%rcx, %%rax\n\t"
929 "adcx %%rax, %%r11\n\t"
930 "movl $0, %%eax\n\t"
931 "adox %%rax, %%r12\n\t"
932 "adcx %%rax, %%r12\n\t"
933
934 "movq 40(%[y]), %%rdx\n\t" // row 5: x * y[5]
935 "xorl %%eax, %%eax\n\t"
936 "mulx 0(%[x]), %%rax, %%rcx\n\t"
937 "adcx %%rax, %%r13\n\t"
938 "movq %%r13, 40(%[z])\n\t"
939 "mulx 8(%[x]), %%rax, %%r13\n\t"
940 "adox %%rcx, %%rax\n\t"
941 "adcx %%rax, %%r14\n\t"
942 "mulx 16(%[x]), %%rax, %%rcx\n\t"
943 "adox %%r13, %%rax\n\t"
944 "adcx %%rax, %%r15\n\t"
945 "mulx 24(%[x]), %%rax, %%r13\n\t"
946 "adox %%rcx, %%rax\n\t"
947 "adcx %%rax, %%r8\n\t"
948 "mulx 32(%[x]), %%rax, %%rcx\n\t"
949 "adox %%r13, %%rax\n\t"
950 "adcx %%rax, %%r9\n\t"
951 "mulx 40(%[x]), %%rax, %%r13\n\t"
952 "adox %%rcx, %%rax\n\t"
953 "adcx %%rax, %%r10\n\t"
954 "mulx 48(%[x]), %%rax, %%rcx\n\t"
955 "adox %%r13, %%rax\n\t"
956 "adcx %%rax, %%r11\n\t"
957 "mulx 56(%[x]), %%rax, %%r13\n\t"
958 "adox %%rcx, %%rax\n\t"
959 "adcx %%rax, %%r12\n\t"
960 "movl $0, %%eax\n\t"
961 "adox %%rax, %%r13\n\t"
962 "adcx %%rax, %%r13\n\t"
963
964 "movq 48(%[y]), %%rdx\n\t" // row 6: x * y[6]
965 "xorl %%eax, %%eax\n\t"
966 "mulx 0(%[x]), %%rax, %%rcx\n\t"
967 "adcx %%rax, %%r14\n\t"
968 "movq %%r14, 48(%[z])\n\t"
969 "mulx 8(%[x]), %%rax, %%r14\n\t"
970 "adox %%rcx, %%rax\n\t"
971 "adcx %%rax, %%r15\n\t"
972 "mulx 16(%[x]), %%rax, %%rcx\n\t"
973 "adox %%r14, %%rax\n\t"
974 "adcx %%rax, %%r8\n\t"
975 "mulx 24(%[x]), %%rax, %%r14\n\t"
976 "adox %%rcx, %%rax\n\t"
977 "adcx %%rax, %%r9\n\t"
978 "mulx 32(%[x]), %%rax, %%rcx\n\t"
979 "adox %%r14, %%rax\n\t"
980 "adcx %%rax, %%r10\n\t"
981 "mulx 40(%[x]), %%rax, %%r14\n\t"
982 "adox %%rcx, %%rax\n\t"
983 "adcx %%rax, %%r11\n\t"
984 "mulx 48(%[x]), %%rax, %%rcx\n\t"
985 "adox %%r14, %%rax\n\t"
986 "adcx %%rax, %%r12\n\t"
987 "mulx 56(%[x]), %%rax, %%r14\n\t"
988 "adox %%rcx, %%rax\n\t"
989 "adcx %%rax, %%r13\n\t"
990 "movl $0, %%eax\n\t"
991 "adox %%rax, %%r14\n\t"
992 "adcx %%rax, %%r14\n\t"
993
994 "movq 56(%[y]), %%rdx\n\t" // row 7: x * y[7]
995 "xorl %%eax, %%eax\n\t"
996 "mulx 0(%[x]), %%rax, %%rcx\n\t"
997 "adcx %%rax, %%r15\n\t"
998 "movq %%r15, 56(%[z])\n\t"
999 "mulx 8(%[x]), %%rax, %%r15\n\t"
1000 "adox %%rcx, %%rax\n\t"
1001 "adcx %%rax, %%r8\n\t"
1002 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1003 "adox %%r15, %%rax\n\t"
1004 "adcx %%rax, %%r9\n\t"
1005 "mulx 24(%[x]), %%rax, %%r15\n\t"
1006 "adox %%rcx, %%rax\n\t"
1007 "adcx %%rax, %%r10\n\t"
1008 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1009 "adox %%r15, %%rax\n\t"
1010 "adcx %%rax, %%r11\n\t"
1011 "mulx 40(%[x]), %%rax, %%r15\n\t"
1012 "adox %%rcx, %%rax\n\t"
1013 "adcx %%rax, %%r12\n\t"
1014 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1015 "adox %%r15, %%rax\n\t"
1016 "adcx %%rax, %%r13\n\t"
1017 "mulx 56(%[x]), %%rax, %%r15\n\t"
1018 "adox %%rcx, %%rax\n\t"
1019 "adcx %%rax, %%r14\n\t"
1020 "movl $0, %%eax\n\t"
1021 "adox %%rax, %%r15\n\t"
1022 "adcx %%rax, %%r15\n\t"
1023
1024 "movq %%r8, 64(%[z])\n\t"
1025 "movq %%r9, 72(%[z])\n\t"
1026 "movq %%r10, 80(%[z])\n\t"
1027 "movq %%r11, 88(%[z])\n\t"
1028 "movq %%r12, 96(%[z])\n\t"
1029 "movq %%r13, 104(%[z])\n\t"
1030 "movq %%r14, 112(%[z])\n\t"
1031 "movq %%r15, 120(%[z])\n\t"
1032 :
1033 : [z] "r"(z), [x] "r"(x), [y] "r"(y)
1034 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "cc", "memory");
1035 // clang-format on
1037}
1038
1039void bigint_mulx_sqr9(word z[18], const word x[9]) {
1040 // clang-format off
1041 asm(
1042 "xorl %%r9d, %%r9d\n\t"
1043 "xorl %%r10d, %%r10d\n\t"
1044 "xorl %%r11d, %%r11d\n\t"
1045 "xorl %%r12d, %%r12d\n\t"
1046 "xorl %%r13d, %%r13d\n\t"
1047 "xorl %%r14d, %%r14d\n\t"
1048 "xorl %%r15d, %%r15d\n\t"
1049 "xorl %%r8d, %%r8d\n\t"
1050
1051 "movq 0(%[x]), %%rdx\n\t" // x[0] * x[1..8]
1052 "xorl %%eax, %%eax\n\t"
1053 "mulx 8(%[x]), %%rax, %%rcx\n\t"
1054 "adcx %%rax, %%r9\n\t"
1055 "movq %%r9, 8(%[z])\n\t"
1056 "mulx 16(%[x]), %%rax, %%r9\n\t"
1057 "adox %%rcx, %%rax\n\t"
1058 "adcx %%rax, %%r10\n\t"
1059 "mulx 24(%[x]), %%rax, %%rcx\n\t"
1060 "adox %%r9, %%rax\n\t"
1061 "adcx %%rax, %%r11\n\t"
1062 "mulx 32(%[x]), %%rax, %%r9\n\t"
1063 "adox %%rcx, %%rax\n\t"
1064 "adcx %%rax, %%r12\n\t"
1065 "mulx 40(%[x]), %%rax, %%rcx\n\t"
1066 "adox %%r9, %%rax\n\t"
1067 "adcx %%rax, %%r13\n\t"
1068 "mulx 48(%[x]), %%rax, %%r9\n\t"
1069 "adox %%rcx, %%rax\n\t"
1070 "adcx %%rax, %%r14\n\t"
1071 "mulx 56(%[x]), %%rax, %%rcx\n\t"
1072 "adox %%r9, %%rax\n\t"
1073 "adcx %%rax, %%r15\n\t"
1074 "mulx 64(%[x]), %%rax, %%r9\n\t"
1075 "adox %%rcx, %%rax\n\t"
1076 "adcx %%rax, %%r8\n\t"
1077 "movl $0, %%eax\n\t"
1078 "adox %%rax, %%r9\n\t"
1079 "adcx %%rax, %%r9\n\t"
1080 "movq %%r10, 16(%[z])\n\t"
1081
1082 "movq 8(%[x]), %%rdx\n\t" // x[1] * x[2..8]
1083 "xorl %%eax, %%eax\n\t"
1084 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1085 "adcx %%rax, %%r11\n\t"
1086 "movq %%r11, 24(%[z])\n\t"
1087 "mulx 24(%[x]), %%rax, %%r10\n\t"
1088 "adox %%rcx, %%rax\n\t"
1089 "adcx %%rax, %%r12\n\t"
1090 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1091 "adox %%r10, %%rax\n\t"
1092 "adcx %%rax, %%r13\n\t"
1093 "mulx 40(%[x]), %%rax, %%r10\n\t"
1094 "adox %%rcx, %%rax\n\t"
1095 "adcx %%rax, %%r14\n\t"
1096 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1097 "adox %%r10, %%rax\n\t"
1098 "adcx %%rax, %%r15\n\t"
1099 "mulx 56(%[x]), %%rax, %%r10\n\t"
1100 "adox %%rcx, %%rax\n\t"
1101 "adcx %%rax, %%r8\n\t"
1102 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1103 "adox %%r10, %%rax\n\t"
1104 "adcx %%rax, %%r9\n\t"
1105 "movl $0, %%eax\n\t"
1106 "adox %%rax, %%rcx\n\t"
1107 "adcx %%rax, %%rcx\n\t"
1108 "movq %%rcx, %%r10\n\t"
1109 "movq %%r12, 32(%[z])\n\t"
1110
1111 "movq 16(%[x]), %%rdx\n\t" // x[2] * x[3..8]
1112 "xorl %%eax, %%eax\n\t"
1113 "mulx 24(%[x]), %%rax, %%rcx\n\t"
1114 "adcx %%rax, %%r13\n\t"
1115 "movq %%r13, 40(%[z])\n\t"
1116 "mulx 32(%[x]), %%rax, %%r11\n\t"
1117 "adox %%rcx, %%rax\n\t"
1118 "adcx %%rax, %%r14\n\t"
1119 "mulx 40(%[x]), %%rax, %%rcx\n\t"
1120 "adox %%r11, %%rax\n\t"
1121 "adcx %%rax, %%r15\n\t"
1122 "mulx 48(%[x]), %%rax, %%r11\n\t"
1123 "adox %%rcx, %%rax\n\t"
1124 "adcx %%rax, %%r8\n\t"
1125 "mulx 56(%[x]), %%rax, %%rcx\n\t"
1126 "adox %%r11, %%rax\n\t"
1127 "adcx %%rax, %%r9\n\t"
1128 "mulx 64(%[x]), %%rax, %%r11\n\t"
1129 "adox %%rcx, %%rax\n\t"
1130 "adcx %%rax, %%r10\n\t"
1131 "movl $0, %%eax\n\t"
1132 "adox %%rax, %%r11\n\t"
1133 "adcx %%rax, %%r11\n\t"
1134 "movq %%r14, 48(%[z])\n\t"
1135
1136 "movq 24(%[x]), %%rdx\n\t" // x[3] * x[4..8]
1137 "xorl %%eax, %%eax\n\t"
1138 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1139 "adcx %%rax, %%r15\n\t"
1140 "movq %%r15, 56(%[z])\n\t"
1141 "mulx 40(%[x]), %%rax, %%r12\n\t"
1142 "adox %%rcx, %%rax\n\t"
1143 "adcx %%rax, %%r8\n\t"
1144 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1145 "adox %%r12, %%rax\n\t"
1146 "adcx %%rax, %%r9\n\t"
1147 "mulx 56(%[x]), %%rax, %%r12\n\t"
1148 "adox %%rcx, %%rax\n\t"
1149 "adcx %%rax, %%r10\n\t"
1150 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1151 "adox %%r12, %%rax\n\t"
1152 "adcx %%rax, %%r11\n\t"
1153 "movl $0, %%eax\n\t"
1154 "adox %%rax, %%rcx\n\t"
1155 "adcx %%rax, %%rcx\n\t"
1156 "movq %%rcx, %%r12\n\t"
1157 "movq %%r8, 64(%[z])\n\t"
1158
1159 "movq 32(%[x]), %%rdx\n\t" // x[4] * x[5..8]
1160 "xorl %%eax, %%eax\n\t"
1161 "mulx 40(%[x]), %%rax, %%rcx\n\t"
1162 "adcx %%rax, %%r9\n\t"
1163 "movq %%r9, 72(%[z])\n\t"
1164 "mulx 48(%[x]), %%rax, %%r13\n\t"
1165 "adox %%rcx, %%rax\n\t"
1166 "adcx %%rax, %%r10\n\t"
1167 "mulx 56(%[x]), %%rax, %%rcx\n\t"
1168 "adox %%r13, %%rax\n\t"
1169 "adcx %%rax, %%r11\n\t"
1170 "mulx 64(%[x]), %%rax, %%r13\n\t"
1171 "adox %%rcx, %%rax\n\t"
1172 "adcx %%rax, %%r12\n\t"
1173 "movl $0, %%eax\n\t"
1174 "adox %%rax, %%r13\n\t"
1175 "adcx %%rax, %%r13\n\t"
1176 "movq %%r10, 80(%[z])\n\t"
1177
1178 "movq 40(%[x]), %%rdx\n\t" // x[5] * x[6..8]
1179 "xorl %%eax, %%eax\n\t"
1180 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1181 "adcx %%rax, %%r11\n\t"
1182 "movq %%r11, 88(%[z])\n\t"
1183 "mulx 56(%[x]), %%rax, %%r14\n\t"
1184 "adox %%rcx, %%rax\n\t"
1185 "adcx %%rax, %%r12\n\t"
1186 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1187 "adox %%r14, %%rax\n\t"
1188 "adcx %%rax, %%r13\n\t"
1189 "movl $0, %%eax\n\t"
1190 "adox %%rax, %%rcx\n\t"
1191 "adcx %%rax, %%rcx\n\t"
1192 "movq %%rcx, %%r14\n\t"
1193 "movq %%r12, 96(%[z])\n\t"
1194
1195 "movq 48(%[x]), %%rdx\n\t" // x[6] * x[7..8]
1196 "xorl %%eax, %%eax\n\t"
1197 "mulx 56(%[x]), %%rax, %%rcx\n\t"
1198 "adcx %%rax, %%r13\n\t"
1199 "movq %%r13, 104(%[z])\n\t"
1200 "mulx 64(%[x]), %%rax, %%r15\n\t"
1201 "adox %%rcx, %%rax\n\t"
1202 "adcx %%rax, %%r14\n\t"
1203 "movl $0, %%eax\n\t"
1204 "adox %%rax, %%r15\n\t"
1205 "adcx %%rax, %%r15\n\t"
1206 "movq %%r14, 112(%[z])\n\t"
1207
1208 "movq 56(%[x]), %%rdx\n\t" // x[7] * x[8..8]
1209 "xorl %%eax, %%eax\n\t"
1210 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1211 "adcx %%rax, %%r15\n\t"
1212 "movq %%r15, 120(%[z])\n\t"
1213 "movl $0, %%eax\n\t"
1214 "adox %%rax, %%rcx\n\t"
1215 "adcx %%rax, %%rcx\n\t"
1216 "movq %%rcx, %%r8\n\t"
1217 "movq %%r8, 128(%[z])\n\t"
1218
1219 "movq $0, 136(%[z])\n\t"
1220 "xorl %%eax, %%eax\n\t" // double the cross products and add the squares
1221 "movq 0(%[x]), %%rdx\n\t"
1222 "mulx %%rdx, %%rax, %%rcx\n\t"
1223 "movq %%rax, 0(%[z])\n\t"
1224 "movq 8(%[z]), %%r8\n\t"
1225 "adcx %%r8, %%r8\n\t"
1226 "adox %%rcx, %%r8\n\t"
1227 "movq %%r8, 8(%[z])\n\t"
1228 "movq 8(%[x]), %%rdx\n\t"
1229 "mulx %%rdx, %%rax, %%rcx\n\t"
1230 "movq 16(%[z]), %%r8\n\t"
1231 "adcx %%r8, %%r8\n\t"
1232 "adox %%rax, %%r8\n\t"
1233 "movq %%r8, 16(%[z])\n\t"
1234 "movq 24(%[z]), %%r8\n\t"
1235 "adcx %%r8, %%r8\n\t"
1236 "adox %%rcx, %%r8\n\t"
1237 "movq %%r8, 24(%[z])\n\t"
1238 "movq 16(%[x]), %%rdx\n\t"
1239 "mulx %%rdx, %%rax, %%rcx\n\t"
1240 "movq 32(%[z]), %%r8\n\t"
1241 "adcx %%r8, %%r8\n\t"
1242 "adox %%rax, %%r8\n\t"
1243 "movq %%r8, 32(%[z])\n\t"
1244 "movq 40(%[z]), %%r8\n\t"
1245 "adcx %%r8, %%r8\n\t"
1246 "adox %%rcx, %%r8\n\t"
1247 "movq %%r8, 40(%[z])\n\t"
1248 "movq 24(%[x]), %%rdx\n\t"
1249 "mulx %%rdx, %%rax, %%rcx\n\t"
1250 "movq 48(%[z]), %%r8\n\t"
1251 "adcx %%r8, %%r8\n\t"
1252 "adox %%rax, %%r8\n\t"
1253 "movq %%r8, 48(%[z])\n\t"
1254 "movq 56(%[z]), %%r8\n\t"
1255 "adcx %%r8, %%r8\n\t"
1256 "adox %%rcx, %%r8\n\t"
1257 "movq %%r8, 56(%[z])\n\t"
1258 "movq 32(%[x]), %%rdx\n\t"
1259 "mulx %%rdx, %%rax, %%rcx\n\t"
1260 "movq 64(%[z]), %%r8\n\t"
1261 "adcx %%r8, %%r8\n\t"
1262 "adox %%rax, %%r8\n\t"
1263 "movq %%r8, 64(%[z])\n\t"
1264 "movq 72(%[z]), %%r8\n\t"
1265 "adcx %%r8, %%r8\n\t"
1266 "adox %%rcx, %%r8\n\t"
1267 "movq %%r8, 72(%[z])\n\t"
1268 "movq 40(%[x]), %%rdx\n\t"
1269 "mulx %%rdx, %%rax, %%rcx\n\t"
1270 "movq 80(%[z]), %%r8\n\t"
1271 "adcx %%r8, %%r8\n\t"
1272 "adox %%rax, %%r8\n\t"
1273 "movq %%r8, 80(%[z])\n\t"
1274 "movq 88(%[z]), %%r8\n\t"
1275 "adcx %%r8, %%r8\n\t"
1276 "adox %%rcx, %%r8\n\t"
1277 "movq %%r8, 88(%[z])\n\t"
1278 "movq 48(%[x]), %%rdx\n\t"
1279 "mulx %%rdx, %%rax, %%rcx\n\t"
1280 "movq 96(%[z]), %%r8\n\t"
1281 "adcx %%r8, %%r8\n\t"
1282 "adox %%rax, %%r8\n\t"
1283 "movq %%r8, 96(%[z])\n\t"
1284 "movq 104(%[z]), %%r8\n\t"
1285 "adcx %%r8, %%r8\n\t"
1286 "adox %%rcx, %%r8\n\t"
1287 "movq %%r8, 104(%[z])\n\t"
1288 "movq 56(%[x]), %%rdx\n\t"
1289 "mulx %%rdx, %%rax, %%rcx\n\t"
1290 "movq 112(%[z]), %%r8\n\t"
1291 "adcx %%r8, %%r8\n\t"
1292 "adox %%rax, %%r8\n\t"
1293 "movq %%r8, 112(%[z])\n\t"
1294 "movq 120(%[z]), %%r8\n\t"
1295 "adcx %%r8, %%r8\n\t"
1296 "adox %%rcx, %%r8\n\t"
1297 "movq %%r8, 120(%[z])\n\t"
1298 "movq 64(%[x]), %%rdx\n\t"
1299 "mulx %%rdx, %%rax, %%rcx\n\t"
1300 "movq 128(%[z]), %%r8\n\t"
1301 "adcx %%r8, %%r8\n\t"
1302 "adox %%rax, %%r8\n\t"
1303 "movq %%r8, 128(%[z])\n\t"
1304 "movq 136(%[z]), %%r8\n\t"
1305 "adcx %%r8, %%r8\n\t"
1306 "adox %%rcx, %%r8\n\t"
1307 "movq %%r8, 136(%[z])\n\t"
1308 :
1309 : [z] "r"(z), [x] "r"(x)
1310 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "cc", "memory");
1311 // clang-format on
1313}
1314
1315void bigint_mulx_mul9(word z[18], const word x[9], const word y[9]) {
1316 copy_mem(z + 9, y, 9);
1317 // clang-format off
1318 asm(
1319 "xorl %%r8d, %%r8d\n\t"
1320 "xorl %%r9d, %%r9d\n\t"
1321 "xorl %%r10d, %%r10d\n\t"
1322 "xorl %%r11d, %%r11d\n\t"
1323 "xorl %%r12d, %%r12d\n\t"
1324 "xorl %%r13d, %%r13d\n\t"
1325 "xorl %%r14d, %%r14d\n\t"
1326 "xorl %%r15d, %%r15d\n\t"
1327 "xorl %%ebx, %%ebx\n\t"
1328
1329 "movq 72(%[z]), %%rdx\n\t" // row 0: x * y[0]
1330 "xorl %%eax, %%eax\n\t"
1331 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1332 "adcx %%rax, %%r8\n\t"
1333 "movq %%r8, 0(%[z])\n\t"
1334 "mulx 8(%[x]), %%rax, %%r8\n\t"
1335 "adox %%rcx, %%rax\n\t"
1336 "adcx %%rax, %%r9\n\t"
1337 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1338 "adox %%r8, %%rax\n\t"
1339 "adcx %%rax, %%r10\n\t"
1340 "mulx 24(%[x]), %%rax, %%r8\n\t"
1341 "adox %%rcx, %%rax\n\t"
1342 "adcx %%rax, %%r11\n\t"
1343 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1344 "adox %%r8, %%rax\n\t"
1345 "adcx %%rax, %%r12\n\t"
1346 "mulx 40(%[x]), %%rax, %%r8\n\t"
1347 "adox %%rcx, %%rax\n\t"
1348 "adcx %%rax, %%r13\n\t"
1349 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1350 "adox %%r8, %%rax\n\t"
1351 "adcx %%rax, %%r14\n\t"
1352 "mulx 56(%[x]), %%rax, %%r8\n\t"
1353 "adox %%rcx, %%rax\n\t"
1354 "adcx %%rax, %%r15\n\t"
1355 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1356 "adox %%r8, %%rax\n\t"
1357 "adcx %%rax, %%rbx\n\t"
1358 "movl $0, %%eax\n\t"
1359 "adox %%rax, %%rcx\n\t"
1360 "adcx %%rax, %%rcx\n\t"
1361 "movq %%rcx, %%r8\n\t"
1362
1363 "movq 80(%[z]), %%rdx\n\t" // row 1: x * y[1]
1364 "xorl %%eax, %%eax\n\t"
1365 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1366 "adcx %%rax, %%r9\n\t"
1367 "movq %%r9, 8(%[z])\n\t"
1368 "mulx 8(%[x]), %%rax, %%r9\n\t"
1369 "adox %%rcx, %%rax\n\t"
1370 "adcx %%rax, %%r10\n\t"
1371 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1372 "adox %%r9, %%rax\n\t"
1373 "adcx %%rax, %%r11\n\t"
1374 "mulx 24(%[x]), %%rax, %%r9\n\t"
1375 "adox %%rcx, %%rax\n\t"
1376 "adcx %%rax, %%r12\n\t"
1377 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1378 "adox %%r9, %%rax\n\t"
1379 "adcx %%rax, %%r13\n\t"
1380 "mulx 40(%[x]), %%rax, %%r9\n\t"
1381 "adox %%rcx, %%rax\n\t"
1382 "adcx %%rax, %%r14\n\t"
1383 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1384 "adox %%r9, %%rax\n\t"
1385 "adcx %%rax, %%r15\n\t"
1386 "mulx 56(%[x]), %%rax, %%r9\n\t"
1387 "adox %%rcx, %%rax\n\t"
1388 "adcx %%rax, %%rbx\n\t"
1389 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1390 "adox %%r9, %%rax\n\t"
1391 "adcx %%rax, %%r8\n\t"
1392 "movl $0, %%eax\n\t"
1393 "adox %%rax, %%rcx\n\t"
1394 "adcx %%rax, %%rcx\n\t"
1395 "movq %%rcx, %%r9\n\t"
1396
1397 "movq 88(%[z]), %%rdx\n\t" // row 2: x * y[2]
1398 "xorl %%eax, %%eax\n\t"
1399 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1400 "adcx %%rax, %%r10\n\t"
1401 "movq %%r10, 16(%[z])\n\t"
1402 "mulx 8(%[x]), %%rax, %%r10\n\t"
1403 "adox %%rcx, %%rax\n\t"
1404 "adcx %%rax, %%r11\n\t"
1405 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1406 "adox %%r10, %%rax\n\t"
1407 "adcx %%rax, %%r12\n\t"
1408 "mulx 24(%[x]), %%rax, %%r10\n\t"
1409 "adox %%rcx, %%rax\n\t"
1410 "adcx %%rax, %%r13\n\t"
1411 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1412 "adox %%r10, %%rax\n\t"
1413 "adcx %%rax, %%r14\n\t"
1414 "mulx 40(%[x]), %%rax, %%r10\n\t"
1415 "adox %%rcx, %%rax\n\t"
1416 "adcx %%rax, %%r15\n\t"
1417 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1418 "adox %%r10, %%rax\n\t"
1419 "adcx %%rax, %%rbx\n\t"
1420 "mulx 56(%[x]), %%rax, %%r10\n\t"
1421 "adox %%rcx, %%rax\n\t"
1422 "adcx %%rax, %%r8\n\t"
1423 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1424 "adox %%r10, %%rax\n\t"
1425 "adcx %%rax, %%r9\n\t"
1426 "movl $0, %%eax\n\t"
1427 "adox %%rax, %%rcx\n\t"
1428 "adcx %%rax, %%rcx\n\t"
1429 "movq %%rcx, %%r10\n\t"
1430
1431 "movq 96(%[z]), %%rdx\n\t" // row 3: x * y[3]
1432 "xorl %%eax, %%eax\n\t"
1433 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1434 "adcx %%rax, %%r11\n\t"
1435 "movq %%r11, 24(%[z])\n\t"
1436 "mulx 8(%[x]), %%rax, %%r11\n\t"
1437 "adox %%rcx, %%rax\n\t"
1438 "adcx %%rax, %%r12\n\t"
1439 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1440 "adox %%r11, %%rax\n\t"
1441 "adcx %%rax, %%r13\n\t"
1442 "mulx 24(%[x]), %%rax, %%r11\n\t"
1443 "adox %%rcx, %%rax\n\t"
1444 "adcx %%rax, %%r14\n\t"
1445 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1446 "adox %%r11, %%rax\n\t"
1447 "adcx %%rax, %%r15\n\t"
1448 "mulx 40(%[x]), %%rax, %%r11\n\t"
1449 "adox %%rcx, %%rax\n\t"
1450 "adcx %%rax, %%rbx\n\t"
1451 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1452 "adox %%r11, %%rax\n\t"
1453 "adcx %%rax, %%r8\n\t"
1454 "mulx 56(%[x]), %%rax, %%r11\n\t"
1455 "adox %%rcx, %%rax\n\t"
1456 "adcx %%rax, %%r9\n\t"
1457 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1458 "adox %%r11, %%rax\n\t"
1459 "adcx %%rax, %%r10\n\t"
1460 "movl $0, %%eax\n\t"
1461 "adox %%rax, %%rcx\n\t"
1462 "adcx %%rax, %%rcx\n\t"
1463 "movq %%rcx, %%r11\n\t"
1464
1465 "movq 104(%[z]), %%rdx\n\t" // row 4: x * y[4]
1466 "xorl %%eax, %%eax\n\t"
1467 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1468 "adcx %%rax, %%r12\n\t"
1469 "movq %%r12, 32(%[z])\n\t"
1470 "mulx 8(%[x]), %%rax, %%r12\n\t"
1471 "adox %%rcx, %%rax\n\t"
1472 "adcx %%rax, %%r13\n\t"
1473 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1474 "adox %%r12, %%rax\n\t"
1475 "adcx %%rax, %%r14\n\t"
1476 "mulx 24(%[x]), %%rax, %%r12\n\t"
1477 "adox %%rcx, %%rax\n\t"
1478 "adcx %%rax, %%r15\n\t"
1479 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1480 "adox %%r12, %%rax\n\t"
1481 "adcx %%rax, %%rbx\n\t"
1482 "mulx 40(%[x]), %%rax, %%r12\n\t"
1483 "adox %%rcx, %%rax\n\t"
1484 "adcx %%rax, %%r8\n\t"
1485 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1486 "adox %%r12, %%rax\n\t"
1487 "adcx %%rax, %%r9\n\t"
1488 "mulx 56(%[x]), %%rax, %%r12\n\t"
1489 "adox %%rcx, %%rax\n\t"
1490 "adcx %%rax, %%r10\n\t"
1491 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1492 "adox %%r12, %%rax\n\t"
1493 "adcx %%rax, %%r11\n\t"
1494 "movl $0, %%eax\n\t"
1495 "adox %%rax, %%rcx\n\t"
1496 "adcx %%rax, %%rcx\n\t"
1497 "movq %%rcx, %%r12\n\t"
1498
1499 "movq 112(%[z]), %%rdx\n\t" // row 5: x * y[5]
1500 "xorl %%eax, %%eax\n\t"
1501 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1502 "adcx %%rax, %%r13\n\t"
1503 "movq %%r13, 40(%[z])\n\t"
1504 "mulx 8(%[x]), %%rax, %%r13\n\t"
1505 "adox %%rcx, %%rax\n\t"
1506 "adcx %%rax, %%r14\n\t"
1507 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1508 "adox %%r13, %%rax\n\t"
1509 "adcx %%rax, %%r15\n\t"
1510 "mulx 24(%[x]), %%rax, %%r13\n\t"
1511 "adox %%rcx, %%rax\n\t"
1512 "adcx %%rax, %%rbx\n\t"
1513 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1514 "adox %%r13, %%rax\n\t"
1515 "adcx %%rax, %%r8\n\t"
1516 "mulx 40(%[x]), %%rax, %%r13\n\t"
1517 "adox %%rcx, %%rax\n\t"
1518 "adcx %%rax, %%r9\n\t"
1519 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1520 "adox %%r13, %%rax\n\t"
1521 "adcx %%rax, %%r10\n\t"
1522 "mulx 56(%[x]), %%rax, %%r13\n\t"
1523 "adox %%rcx, %%rax\n\t"
1524 "adcx %%rax, %%r11\n\t"
1525 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1526 "adox %%r13, %%rax\n\t"
1527 "adcx %%rax, %%r12\n\t"
1528 "movl $0, %%eax\n\t"
1529 "adox %%rax, %%rcx\n\t"
1530 "adcx %%rax, %%rcx\n\t"
1531 "movq %%rcx, %%r13\n\t"
1532
1533 "movq 120(%[z]), %%rdx\n\t" // row 6: x * y[6]
1534 "xorl %%eax, %%eax\n\t"
1535 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1536 "adcx %%rax, %%r14\n\t"
1537 "movq %%r14, 48(%[z])\n\t"
1538 "mulx 8(%[x]), %%rax, %%r14\n\t"
1539 "adox %%rcx, %%rax\n\t"
1540 "adcx %%rax, %%r15\n\t"
1541 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1542 "adox %%r14, %%rax\n\t"
1543 "adcx %%rax, %%rbx\n\t"
1544 "mulx 24(%[x]), %%rax, %%r14\n\t"
1545 "adox %%rcx, %%rax\n\t"
1546 "adcx %%rax, %%r8\n\t"
1547 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1548 "adox %%r14, %%rax\n\t"
1549 "adcx %%rax, %%r9\n\t"
1550 "mulx 40(%[x]), %%rax, %%r14\n\t"
1551 "adox %%rcx, %%rax\n\t"
1552 "adcx %%rax, %%r10\n\t"
1553 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1554 "adox %%r14, %%rax\n\t"
1555 "adcx %%rax, %%r11\n\t"
1556 "mulx 56(%[x]), %%rax, %%r14\n\t"
1557 "adox %%rcx, %%rax\n\t"
1558 "adcx %%rax, %%r12\n\t"
1559 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1560 "adox %%r14, %%rax\n\t"
1561 "adcx %%rax, %%r13\n\t"
1562 "movl $0, %%eax\n\t"
1563 "adox %%rax, %%rcx\n\t"
1564 "adcx %%rax, %%rcx\n\t"
1565 "movq %%rcx, %%r14\n\t"
1566
1567 "movq 128(%[z]), %%rdx\n\t" // row 7: x * y[7]
1568 "xorl %%eax, %%eax\n\t"
1569 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1570 "adcx %%rax, %%r15\n\t"
1571 "movq %%r15, 56(%[z])\n\t"
1572 "mulx 8(%[x]), %%rax, %%r15\n\t"
1573 "adox %%rcx, %%rax\n\t"
1574 "adcx %%rax, %%rbx\n\t"
1575 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1576 "adox %%r15, %%rax\n\t"
1577 "adcx %%rax, %%r8\n\t"
1578 "mulx 24(%[x]), %%rax, %%r15\n\t"
1579 "adox %%rcx, %%rax\n\t"
1580 "adcx %%rax, %%r9\n\t"
1581 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1582 "adox %%r15, %%rax\n\t"
1583 "adcx %%rax, %%r10\n\t"
1584 "mulx 40(%[x]), %%rax, %%r15\n\t"
1585 "adox %%rcx, %%rax\n\t"
1586 "adcx %%rax, %%r11\n\t"
1587 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1588 "adox %%r15, %%rax\n\t"
1589 "adcx %%rax, %%r12\n\t"
1590 "mulx 56(%[x]), %%rax, %%r15\n\t"
1591 "adox %%rcx, %%rax\n\t"
1592 "adcx %%rax, %%r13\n\t"
1593 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1594 "adox %%r15, %%rax\n\t"
1595 "adcx %%rax, %%r14\n\t"
1596 "movl $0, %%eax\n\t"
1597 "adox %%rax, %%rcx\n\t"
1598 "adcx %%rax, %%rcx\n\t"
1599 "movq %%rcx, %%r15\n\t"
1600
1601 "movq 136(%[z]), %%rdx\n\t" // row 8: x * y[8]
1602 "xorl %%eax, %%eax\n\t"
1603 "mulx 0(%[x]), %%rax, %%rcx\n\t"
1604 "adcx %%rax, %%rbx\n\t"
1605 "movq %%rbx, 64(%[z])\n\t"
1606 "mulx 8(%[x]), %%rax, %%rbx\n\t"
1607 "adox %%rcx, %%rax\n\t"
1608 "adcx %%rax, %%r8\n\t"
1609 "mulx 16(%[x]), %%rax, %%rcx\n\t"
1610 "adox %%rbx, %%rax\n\t"
1611 "adcx %%rax, %%r9\n\t"
1612 "mulx 24(%[x]), %%rax, %%rbx\n\t"
1613 "adox %%rcx, %%rax\n\t"
1614 "adcx %%rax, %%r10\n\t"
1615 "mulx 32(%[x]), %%rax, %%rcx\n\t"
1616 "adox %%rbx, %%rax\n\t"
1617 "adcx %%rax, %%r11\n\t"
1618 "mulx 40(%[x]), %%rax, %%rbx\n\t"
1619 "adox %%rcx, %%rax\n\t"
1620 "adcx %%rax, %%r12\n\t"
1621 "mulx 48(%[x]), %%rax, %%rcx\n\t"
1622 "adox %%rbx, %%rax\n\t"
1623 "adcx %%rax, %%r13\n\t"
1624 "mulx 56(%[x]), %%rax, %%rbx\n\t"
1625 "adox %%rcx, %%rax\n\t"
1626 "adcx %%rax, %%r14\n\t"
1627 "mulx 64(%[x]), %%rax, %%rcx\n\t"
1628 "adox %%rbx, %%rax\n\t"
1629 "adcx %%rax, %%r15\n\t"
1630 "movl $0, %%eax\n\t"
1631 "adox %%rax, %%rcx\n\t"
1632 "adcx %%rax, %%rcx\n\t"
1633 "movq %%rcx, %%rbx\n\t"
1634
1635 "movq %%r8, 72(%[z])\n\t"
1636 "movq %%r9, 80(%[z])\n\t"
1637 "movq %%r10, 88(%[z])\n\t"
1638 "movq %%r11, 96(%[z])\n\t"
1639 "movq %%r12, 104(%[z])\n\t"
1640 "movq %%r13, 112(%[z])\n\t"
1641 "movq %%r14, 120(%[z])\n\t"
1642 "movq %%r15, 128(%[z])\n\t"
1643 "movq %%rbx, 136(%[z])\n\t"
1644 :
1645 : [z] "r"(z), [x] "r"(x)
1646 : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "rbx", "cc", "memory");
1647 // clang-format on
1649}
1650
1651// NOLINTEND(*-no-assembler)
1652
1653} // namespace Botan
void bigint_mulx_mul6(word z[12], const word x[6], const word y[6])
Definition mp_mulx.cpp:371
void bigint_mulx_mul9(word z[18], const word x[9], const word y[9])
Definition mp_mulx.cpp:1315
void bigint_mulx_mul8(word z[16], const word x[8], const word y[8])
Definition mp_mulx.cpp:772
constexpr void copy_mem(T *out, const T *in, size_t n)
Definition mem_ops.h:144
void bigint_mulx_sqr6(word z[12], const word x[6])
Definition mp_mulx.cpp:213
void bigint_mulx_sqr9(word z[18], const word x[9])
Definition mp_mulx.cpp:1039
void bigint_mulx_sqr4(word z[8], const word x[4])
Definition mp_mulx.cpp:28
void bigint_mulx_mul4(word z[8], const word x[4], const word y[4])
Definition mp_mulx.cpp:122
std::conditional_t< HasNative64BitRegisters, std::uint64_t, uint32_t > word
The native machine word, used as the limb type for multiprecision integers.
Definition types.h:131
void bigint_mulx_sqr8(word z[16], const word x[8])
Definition mp_mulx.cpp:538
void unpoison_asm_output(const W *ptr)
Definition mp_asmi.h:101