summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib/libcrypto/bn/asm/modexp512-x86_64.pl1393
1 files changed, 0 insertions, 1393 deletions
diff --git a/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl b/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
deleted file mode 100644
index 8645d5adcc..0000000000
--- a/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
+++ /dev/null
@@ -1,1393 +0,0 @@
1#!/usr/bin/env perl
2#
3# Copyright (c) 2010-2011 Intel Corp.
4# Author: Vinodh.Gopal@intel.com
5# Jim Guilford
6# Erdinc.Ozturk@intel.com
7# Maxim.Perminov@intel.com
8#
9# More information about algorithm used can be found at:
10# http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf
11#
12# ====================================================================
13# Copyright (c) 2011 The OpenSSL Project. All rights reserved.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions
17# are met:
18#
19# 1. Redistributions of source code must retain the above copyright
20# notice, this list of conditions and the following disclaimer.
21#
22# 2. Redistributions in binary form must reproduce the above copyright
23# notice, this list of conditions and the following disclaimer in
24# the documentation and/or other materials provided with the
25# distribution.
26#
27# 3. All advertising materials mentioning features or use of this
28# software must display the following acknowledgment:
29# "This product includes software developed by the OpenSSL Project
30# for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31#
32# 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
33# endorse or promote products derived from this software without
34# prior written permission. For written permission, please contact
35# licensing@OpenSSL.org.
36#
37# 5. Products derived from this software may not be called "OpenSSL"
38# nor may "OpenSSL" appear in their names without prior written
39# permission of the OpenSSL Project.
40#
41# 6. Redistributions of any form whatsoever must retain the following
42# acknowledgment:
43# "This product includes software developed by the OpenSSL Project
44# for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45#
46# THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
47# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
50# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
52# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
53# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
55# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
56# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
57# OF THE POSSIBILITY OF SUCH DAMAGE.
58# ====================================================================
59
60$flavour = shift;
61$output = shift;
62if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63
64$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
65( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
66( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
67die "can't locate x86_64-xlate.pl";
68
69open OUT,"| \"$^X\" $xlate $flavour $output";
70*STDOUT=*OUT;
71
72use strict;
73my $code=".text\n\n";
74my $m=0;
75
76#
77# Define x512 macros
78#
79
80#MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2
81#
82# uses rax, rdx, and args
83sub MULSTEP_512_ADD
84{
85 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_;
86 my @X=@$x; # make a copy
87$code.=<<___;
88 mov (+8*0)($SRC2), %rax
89 mul $OP # rdx:rax = %OP * [0]
90 mov ($ASRC), $X[0]
91 add %rax, $X[0]
92 adc \$0, %rdx
93 mov $X[0], $DST
94___
95for(my $i=1;$i<8;$i++) {
96$code.=<<___;
97 mov %rdx, $TMP
98
99 mov (+8*$i)($SRC2), %rax
100 mul $OP # rdx:rax = %OP * [$i]
101 mov (+8*$i)($ASRC), $X[$i]
102 add %rax, $X[$i]
103 adc \$0, %rdx
104 add $TMP, $X[$i]
105 adc \$0, %rdx
106___
107}
108$code.=<<___;
109 mov %rdx, $X[0]
110___
111}
112
113#MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp
114#
115# uses rax, rdx, and args
116sub MULSTEP_512
117{
118 my ($x, $DST, $SRC2, $OP, $TMP)=@_;
119 my @X=@$x; # make a copy
120$code.=<<___;
121 mov (+8*0)($SRC2), %rax
122 mul $OP # rdx:rax = %OP * [0]
123 add %rax, $X[0]
124 adc \$0, %rdx
125 mov $X[0], $DST
126___
127for(my $i=1;$i<8;$i++) {
128$code.=<<___;
129 mov %rdx, $TMP
130
131 mov (+8*$i)($SRC2), %rax
132 mul $OP # rdx:rax = %OP * [$i]
133 add %rax, $X[$i]
134 adc \$0, %rdx
135 add $TMP, $X[$i]
136 adc \$0, %rdx
137___
138}
139$code.=<<___;
140 mov %rdx, $X[0]
141___
142}
143
144#
145# Swizzle Macros
146#
147
148# macro to copy data from flat space to swizzled table
149#MACRO swizzle pDst, pSrc, tmp1, tmp2
150# pDst and pSrc are modified
151sub swizzle
152{
153 my ($pDst, $pSrc, $cnt, $d0)=@_;
154$code.=<<___;
155 mov \$8, $cnt
156loop_$m:
157 mov ($pSrc), $d0
158 mov $d0#w, ($pDst)
159 shr \$16, $d0
160 mov $d0#w, (+64*1)($pDst)
161 shr \$16, $d0
162 mov $d0#w, (+64*2)($pDst)
163 shr \$16, $d0
164 mov $d0#w, (+64*3)($pDst)
165 lea 8($pSrc), $pSrc
166 lea 64*4($pDst), $pDst
167 dec $cnt
168 jnz loop_$m
169___
170
171 $m++;
172}
173
174# macro to copy data from swizzled table to flat space
175#MACRO unswizzle pDst, pSrc, tmp*3
176sub unswizzle
177{
178 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_;
179$code.=<<___;
180 mov \$4, $cnt
181loop_$m:
182 movzxw (+64*3+256*0)($pSrc), $d0
183 movzxw (+64*3+256*1)($pSrc), $d1
184 shl \$16, $d0
185 shl \$16, $d1
186 mov (+64*2+256*0)($pSrc), $d0#w
187 mov (+64*2+256*1)($pSrc), $d1#w
188 shl \$16, $d0
189 shl \$16, $d1
190 mov (+64*1+256*0)($pSrc), $d0#w
191 mov (+64*1+256*1)($pSrc), $d1#w
192 shl \$16, $d0
193 shl \$16, $d1
194 mov (+64*0+256*0)($pSrc), $d0#w
195 mov (+64*0+256*1)($pSrc), $d1#w
196 mov $d0, (+8*0)($pDst)
197 mov $d1, (+8*1)($pDst)
198 lea 256*2($pSrc), $pSrc
199 lea 8*2($pDst), $pDst
200 sub \$1, $cnt
201 jnz loop_$m
202___
203
204 $m++;
205}
206
207#
208# Data Structures
209#
210
211# Reduce Data
212#
213#
214# Offset Value
215# 0C0 Carries
216# 0B8 X2[10]
217# 0B0 X2[9]
218# 0A8 X2[8]
219# 0A0 X2[7]
220# 098 X2[6]
221# 090 X2[5]
222# 088 X2[4]
223# 080 X2[3]
224# 078 X2[2]
225# 070 X2[1]
226# 068 X2[0]
227# 060 X1[12] P[10]
228# 058 X1[11] P[9] Z[8]
229# 050 X1[10] P[8] Z[7]
230# 048 X1[9] P[7] Z[6]
231# 040 X1[8] P[6] Z[5]
232# 038 X1[7] P[5] Z[4]
233# 030 X1[6] P[4] Z[3]
234# 028 X1[5] P[3] Z[2]
235# 020 X1[4] P[2] Z[1]
236# 018 X1[3] P[1] Z[0]
237# 010 X1[2] P[0] Y[2]
238# 008 X1[1] Q[1] Y[1]
239# 000 X1[0] Q[0] Y[0]
240
241my $X1_offset = 0; # 13 qwords
242my $X2_offset = $X1_offset + 13*8; # 11 qwords
243my $Carries_offset = $X2_offset + 11*8; # 1 qword
244my $Q_offset = 0; # 2 qwords
245my $P_offset = $Q_offset + 2*8; # 11 qwords
246my $Y_offset = 0; # 3 qwords
247my $Z_offset = $Y_offset + 3*8; # 9 qwords
248
249my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords)
250
251#
252# Stack Frame
253#
254#
255# offset value
256# ... <old stack contents>
257# ...
258# 280 Garray
259
260# 278 tmp16[15]
261# ... ...
262# 200 tmp16[0]
263
264# 1F8 tmp[7]
265# ... ...
266# 1C0 tmp[0]
267
268# 1B8 GT[7]
269# ... ...
270# 180 GT[0]
271
272# 178 Reduce Data
273# ... ...
274# 0B8 Reduce Data
275# 0B0 reserved
276# 0A8 reserved
277# 0A0 reserved
278# 098 reserved
279# 090 reserved
280# 088 reduce result addr
281# 080 exp[8]
282
283# ...
284# 048 exp[1]
285# 040 exp[0]
286
287# 038 reserved
288# 030 loop_idx
289# 028 pg
290# 020 i
291# 018 pData ; arg 4
292# 010 pG ; arg 2
293# 008 pResult ; arg 1
294# 000 rsp ; stack pointer before subtract
295
296my $rsp_offset = 0;
297my $pResult_offset = 8*1 + $rsp_offset;
298my $pG_offset = 8*1 + $pResult_offset;
299my $pData_offset = 8*1 + $pG_offset;
300my $i_offset = 8*1 + $pData_offset;
301my $pg_offset = 8*1 + $i_offset;
302my $loop_idx_offset = 8*1 + $pg_offset;
303my $reserved1_offset = 8*1 + $loop_idx_offset;
304my $exp_offset = 8*1 + $reserved1_offset;
305my $red_result_addr_offset= 8*9 + $exp_offset;
306my $reserved2_offset = 8*1 + $red_result_addr_offset;
307my $Reduce_Data_offset = 8*5 + $reserved2_offset;
308my $GT_offset = $Red_Data_Size + $Reduce_Data_offset;
309my $tmp_offset = 8*8 + $GT_offset;
310my $tmp16_offset = 8*8 + $tmp_offset;
311my $garray_offset = 8*16 + $tmp16_offset;
312my $mem_size = 8*8*32 + $garray_offset;
313
314#
315# Offsets within Reduce Data
316#
317#
318# struct MODF_2FOLD_MONT_512_C1_DATA {
319# UINT64 t[8][8];
320# UINT64 m[8];
321# UINT64 m1[8]; /* 2^768 % m */
322# UINT64 m2[8]; /* 2^640 % m */
323# UINT64 k1[2]; /* (- 1/m) % 2^128 */
324# };
325
326my $T = 0;
327my $M = 512; # = 8 * 8 * 8
328my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */
329my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */
330my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */
331
332#
333# FUNCTIONS
334#
335
336{{{
337#
338# MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords)
339# and add 512-bits (8 qwords)
340# to get 640 bits (10 qwords)
341# Input: 128-bit mul source: [rdi+8*1], rbp
342# 512-bit mul source: [rsi+8*n]
343# 512-bit add source: r15, r14, ..., r9, r8
344# Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0]
345# Clobbers all regs except: rcx, rsi, rdi
346$code.=<<___;
347.type MULADD_128x512,\@abi-omnipotent
348.align 16
349MULADD_128x512:
350 _CET_ENDBR
351___
352 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx");
353$code.=<<___;
354 mov (+8*1)(%rdi), %rbp
355___
356 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx");
357$code.=<<___;
358 ret
359.size MULADD_128x512,.-MULADD_128x512
360___
361}}}
362
363{{{
364#MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0
365#
366# Inputs: pDst: Destination (768 bits, 12 qwords)
367# pA: Multiplicand (1024 bits, 16 qwords)
368# pB: Multiplicand (512 bits, 8 qwords)
369# Dst = Ah * B + Al
370# where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits)
371# Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0]
372# Uses registers: arguments, RAX, RDX
373sub MULADD_256x512
374{
375 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_;
376$code.=<<___;
377 mov (+8*12)($pA), $OP
378___
379 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP);
380 push(@$X,shift(@$X));
381
382$code.=<<___;
383 mov (+8*13)($pA), $OP
384___
385 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP);
386 push(@$X,shift(@$X));
387
388$code.=<<___;
389 mov (+8*14)($pA), $OP
390___
391 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP);
392 push(@$X,shift(@$X));
393
394$code.=<<___;
395 mov (+8*15)($pA), $OP
396___
397 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP);
398 push(@$X,shift(@$X));
399}
400
401#
402# mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */
403# UINT64 *m, /* 512 bits, 8 qwords */
404# MODF_2FOLD_MONT_512_C1_DATA *data,
405# UINT64 *r) /* 512 bits, 8 qwords */
406# Input: x (number to be reduced): tmp16 (Implicit)
407# m (modulus): [pM] (Implicit)
408# data (reduce data): [pData] (Implicit)
409# Output: r (result): Address in [red_res_addr]
410# result also in: r9, r8, r15, r14, r13, r12, r11, r10
411
412my @X=map("%r$_",(8..15));
413
414$code.=<<___;
415.type mont_reduce,\@abi-omnipotent
416.align 16
417mont_reduce:
418 _CET_ENDBR
419___
420
421my $STACK_DEPTH = 8;
422 #
423 # X1 = Xh * M1 + Xl
424$code.=<<___;
425 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords
426 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords
427 add \$$M1, %rsi
428 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords
429
430___
431
432 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times
433 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0]
434
435$code.=<<___;
436 xor %rax, %rax
437 # X1 += xl
438 add (+8*8)(%rcx), $X[4]
439 adc (+8*9)(%rcx), $X[5]
440 adc (+8*10)(%rcx), $X[6]
441 adc (+8*11)(%rcx), $X[7]
442 adc \$0, %rax
443 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0]
444
445 #
446 # check for carry ;; carry stored in rax
447 mov $X[4], (+8*8)(%rdi) # rdi points to X1
448 mov $X[5], (+8*9)(%rdi)
449 mov $X[6], %rbp
450 mov $X[7], (+8*11)(%rdi)
451
452 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
453
454 mov (+8*0)(%rdi), $X[4]
455 mov (+8*1)(%rdi), $X[5]
456 mov (+8*2)(%rdi), $X[6]
457 mov (+8*3)(%rdi), $X[7]
458
459 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8
460 # rdi -> X1
461 # rsi -> M1
462
463 #
464 # X2 = Xh * M2 + Xl
465 # do first part (X2 = Xh * M2)
466 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords
467 # Xh is actually { [rdi+8*1], rbp }
468 add \$`$M2-$M1`, %rsi # rsi -> M2
469 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords
470___
471 unshift(@X,pop(@X)); unshift(@X,pop(@X));
472$code.=<<___;
473
474 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
475 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
476 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax
477
478 # X2 += Xl
479 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl
480 adc (+8*9-8*10)(%rdi), $X[7]
481 mov $X[6], (+8*8)(%rcx)
482 mov $X[7], (+8*9)(%rcx)
483
484 adc %rax, %rax
485 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
486
487 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords
488 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords
489
490 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half)
491 # B1:B0 = rsi[1:0] = K1[1:0]
492 # A1:A0 = rcx[1:0] = X2[1:0]
493 # Result = rdi[1],rbp = Q[1],rbp
494 mov (%rsi), %r8 # B0
495 mov (+8*1)(%rsi), %rbx # B1
496
497 mov (%rcx), %rax # A0
498 mul %r8 # B0
499 mov %rax, %rbp
500 mov %rdx, %r9
501
502 mov (+8*1)(%rcx), %rax # A1
503 mul %r8 # B0
504 add %rax, %r9
505
506 mov (%rcx), %rax # A0
507 mul %rbx # B1
508 add %rax, %r9
509
510 mov %r9, (+8*1)(%rdi)
511 # end MUL_128x128t128
512
513 sub \$`$K1-$M`, %rsi
514
515 mov (%rcx), $X[6]
516 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0]
517
518 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
519 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
520
521 # load first half of m to rdx, rdi, rbx, rax
522 # moved this here for efficiency
523 mov (+8*0)(%rsi), %rax
524 mov (+8*1)(%rsi), %rbx
525 mov (+8*2)(%rsi), %rdi
526 mov (+8*3)(%rsi), %rdx
527
528 # continue with reduction
529 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp
530
531 add (+8*8)(%rcx), $X[6]
532 adc (+8*9)(%rcx), $X[7]
533
534 #accumulate the final carry to rbp
535 adc %rbp, %rbp
536
537 # Add in overflow corrections: R = (X2>>128) += T[overflow]
538 # R = {r9, r8, r15, r14, ..., r10}
539 shl \$3, %rbp
540 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T)
541 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out
542
543 # rsi will be used to generate a mask after the addition
544 xor %rsi, %rsi
545
546 add (+8*8*0)(%rbp), $X[0]
547 adc (+8*8*1)(%rbp), $X[1]
548 adc (+8*8*2)(%rbp), $X[2]
549 adc (+8*8*3)(%rbp), $X[3]
550 adc (+8*8*4)(%rbp), $X[4]
551 adc (+8*8*5)(%rbp), $X[5]
552 adc (+8*8*6)(%rbp), $X[6]
553 adc (+8*8*7)(%rbp), $X[7]
554
555 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF
556 # if carry is clear: rsi = 0x0000000000000000
557 sbb \$0, %rsi
558
559 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
560 and %rsi, %rax
561 and %rsi, %rbx
562 and %rsi, %rdi
563 and %rsi, %rdx
564
565 mov \$1, %rbp
566 sub %rax, $X[0]
567 sbb %rbx, $X[1]
568 sbb %rdi, $X[2]
569 sbb %rdx, $X[3]
570
571 # if there is a borrow: rbp = 0
572 # if there is no borrow: rbp = 1
573 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m
574 sbb \$0, %rbp
575
576 #load second half of m to rdx, rdi, rbx, rax
577
578 add \$$M, %rcx
579 mov (+8*4)(%rcx), %rax
580 mov (+8*5)(%rcx), %rbx
581 mov (+8*6)(%rcx), %rdi
582 mov (+8*7)(%rcx), %rdx
583
584 # use the rsi mask as before
585 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
586 and %rsi, %rax
587 and %rsi, %rbx
588 and %rsi, %rdi
589 and %rsi, %rdx
590
591 # if rbp = 0, there was a borrow before, it is moved to the carry flag
592 # if rbp = 1, there was not a borrow before, carry flag is cleared
593 sub \$1, %rbp
594
595 sbb %rax, $X[4]
596 sbb %rbx, $X[5]
597 sbb %rdi, $X[6]
598 sbb %rdx, $X[7]
599
600 # write R back to memory
601
602 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi
603 mov $X[0], (+8*0)(%rsi)
604 mov $X[1], (+8*1)(%rsi)
605 mov $X[2], (+8*2)(%rsi)
606 mov $X[3], (+8*3)(%rsi)
607 mov $X[4], (+8*4)(%rsi)
608 mov $X[5], (+8*5)(%rsi)
609 mov $X[6], (+8*6)(%rsi)
610 mov $X[7], (+8*7)(%rsi)
611
612 ret
613.size mont_reduce,.-mont_reduce
614___
615}}}
616
617{{{
618#MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2
619#
620# Inputs: pDst: Destination (1024 bits, 16 qwords)
621# pA: Multiplicand (512 bits, 8 qwords)
622# pB: Multiplicand (512 bits, 8 qwords)
623# Uses registers rax, rdx, args
624# B operand in [pB] and also in x7...x0
625sub MUL_512x512
626{
627 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_;
628 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
629 my @X=@$x; # make a copy
630
631$code.=<<___;
632 mov (+8*0)($pA), $OP
633
634 mov $X[0], %rax
635 mul $OP # rdx:rax = %OP * [0]
636 mov %rax, (+$pDst_o+8*0)($pDst)
637 mov %rdx, $X[0]
638___
639for(my $i=1;$i<8;$i++) {
640$code.=<<___;
641 mov $X[$i], %rax
642 mul $OP # rdx:rax = %OP * [$i]
643 add %rax, $X[$i-1]
644 adc \$0, %rdx
645 mov %rdx, $X[$i]
646___
647}
648
649for(my $i=1;$i<8;$i++) {
650$code.=<<___;
651 mov (+8*$i)($pA), $OP
652___
653
654 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP);
655 push(@X,shift(@X));
656}
657
658$code.=<<___;
659 mov $X[0], (+$pDst_o+8*8)($pDst)
660 mov $X[1], (+$pDst_o+8*9)($pDst)
661 mov $X[2], (+$pDst_o+8*10)($pDst)
662 mov $X[3], (+$pDst_o+8*11)($pDst)
663 mov $X[4], (+$pDst_o+8*12)($pDst)
664 mov $X[5], (+$pDst_o+8*13)($pDst)
665 mov $X[6], (+$pDst_o+8*14)($pDst)
666 mov $X[7], (+$pDst_o+8*15)($pDst)
667___
668}
669
670#
671# mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits)
672# Input: src1: Address of source 1: rdi
673# src2: Address of source 2: rsi
674# Output: dst: Address of destination: [red_res_addr]
675# src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10
676# Temp: Clobbers [tmp16], all registers
677$code.=<<___;
678.type mont_mul_a3b,\@abi-omnipotent
679.align 16
680mont_mul_a3b:
681 _CET_ENDBR
682 #
683 # multiply tmp = src1 * src2
684 # For multiply: dst = rcx, src1 = rdi, src2 = rsi
685 # stack depth is extra 8 from call
686___
687 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx");
688$code.=<<___;
689 #
690 # Dst = tmp % m
691 # Call reduce(tmp, m, data, dst)
692
693 # tail recursion optimization: jmp to mont_reduce and return from there
694 jmp mont_reduce
695 # call mont_reduce
696 # ret
697.size mont_mul_a3b,.-mont_mul_a3b
698___
699}}}
700
701{{{
702#SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4
703#
704# Input in memory [pA] and also in x7...x0
705# Uses all argument registers plus rax and rdx
706#
707# This version computes all of the off-diagonal terms into memory,
708# and then it adds in the diagonal terms
709
710sub SQR_512
711{
712 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_;
713 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
714 my @X=@$x; # make a copy
715$code.=<<___;
716 # ------------------
717 # first pass 01...07
718 # ------------------
719 mov $X[0], $A
720
721 mov $X[1],%rax
722 mul $A
723 mov %rax, (+$pDst_o+8*1)($pDst)
724___
725for(my $i=2;$i<8;$i++) {
726$code.=<<___;
727 mov %rdx, $X[$i-2]
728 mov $X[$i],%rax
729 mul $A
730 add %rax, $X[$i-2]
731 adc \$0, %rdx
732___
733}
734$code.=<<___;
735 mov %rdx, $x7
736
737 mov $X[0], (+$pDst_o+8*2)($pDst)
738
739 # ------------------
740 # second pass 12...17
741 # ------------------
742
743 mov (+8*1)($pA), $A
744
745 mov (+8*2)($pA),%rax
746 mul $A
747 add %rax, $X[1]
748 adc \$0, %rdx
749 mov $X[1], (+$pDst_o+8*3)($pDst)
750
751 mov %rdx, $X[0]
752 mov (+8*3)($pA),%rax
753 mul $A
754 add %rax, $X[2]
755 adc \$0, %rdx
756 add $X[0], $X[2]
757 adc \$0, %rdx
758 mov $X[2], (+$pDst_o+8*4)($pDst)
759
760 mov %rdx, $X[0]
761 mov (+8*4)($pA),%rax
762 mul $A
763 add %rax, $X[3]
764 adc \$0, %rdx
765 add $X[0], $X[3]
766 adc \$0, %rdx
767
768 mov %rdx, $X[0]
769 mov (+8*5)($pA),%rax
770 mul $A
771 add %rax, $X[4]
772 adc \$0, %rdx
773 add $X[0], $X[4]
774 adc \$0, %rdx
775
776 mov %rdx, $X[0]
777 mov $X[6],%rax
778 mul $A
779 add %rax, $X[5]
780 adc \$0, %rdx
781 add $X[0], $X[5]
782 adc \$0, %rdx
783
784 mov %rdx, $X[0]
785 mov $X[7],%rax
786 mul $A
787 add %rax, $x7
788 adc \$0, %rdx
789 add $X[0], $x7
790 adc \$0, %rdx
791
792 mov %rdx, $X[1]
793
794 # ------------------
795 # third pass 23...27
796 # ------------------
797 mov (+8*2)($pA), $A
798
799 mov (+8*3)($pA),%rax
800 mul $A
801 add %rax, $X[3]
802 adc \$0, %rdx
803 mov $X[3], (+$pDst_o+8*5)($pDst)
804
805 mov %rdx, $X[0]
806 mov (+8*4)($pA),%rax
807 mul $A
808 add %rax, $X[4]
809 adc \$0, %rdx
810 add $X[0], $X[4]
811 adc \$0, %rdx
812 mov $X[4], (+$pDst_o+8*6)($pDst)
813
814 mov %rdx, $X[0]
815 mov (+8*5)($pA),%rax
816 mul $A
817 add %rax, $X[5]
818 adc \$0, %rdx
819 add $X[0], $X[5]
820 adc \$0, %rdx
821
822 mov %rdx, $X[0]
823 mov $X[6],%rax
824 mul $A
825 add %rax, $x7
826 adc \$0, %rdx
827 add $X[0], $x7
828 adc \$0, %rdx
829
830 mov %rdx, $X[0]
831 mov $X[7],%rax
832 mul $A
833 add %rax, $X[1]
834 adc \$0, %rdx
835 add $X[0], $X[1]
836 adc \$0, %rdx
837
838 mov %rdx, $X[2]
839
840 # ------------------
841 # fourth pass 34...37
842 # ------------------
843
844 mov (+8*3)($pA), $A
845
846 mov (+8*4)($pA),%rax
847 mul $A
848 add %rax, $X[5]
849 adc \$0, %rdx
850 mov $X[5], (+$pDst_o+8*7)($pDst)
851
852 mov %rdx, $X[0]
853 mov (+8*5)($pA),%rax
854 mul $A
855 add %rax, $x7
856 adc \$0, %rdx
857 add $X[0], $x7
858 adc \$0, %rdx
859 mov $x7, (+$pDst_o+8*8)($pDst)
860
861 mov %rdx, $X[0]
862 mov $X[6],%rax
863 mul $A
864 add %rax, $X[1]
865 adc \$0, %rdx
866 add $X[0], $X[1]
867 adc \$0, %rdx
868
869 mov %rdx, $X[0]
870 mov $X[7],%rax
871 mul $A
872 add %rax, $X[2]
873 adc \$0, %rdx
874 add $X[0], $X[2]
875 adc \$0, %rdx
876
877 mov %rdx, $X[5]
878
879 # ------------------
880 # fifth pass 45...47
881 # ------------------
882 mov (+8*4)($pA), $A
883
884 mov (+8*5)($pA),%rax
885 mul $A
886 add %rax, $X[1]
887 adc \$0, %rdx
888 mov $X[1], (+$pDst_o+8*9)($pDst)
889
890 mov %rdx, $X[0]
891 mov $X[6],%rax
892 mul $A
893 add %rax, $X[2]
894 adc \$0, %rdx
895 add $X[0], $X[2]
896 adc \$0, %rdx
897 mov $X[2], (+$pDst_o+8*10)($pDst)
898
899 mov %rdx, $X[0]
900 mov $X[7],%rax
901 mul $A
902 add %rax, $X[5]
903 adc \$0, %rdx
904 add $X[0], $X[5]
905 adc \$0, %rdx
906
907 mov %rdx, $X[1]
908
909 # ------------------
910 # sixth pass 56...57
911 # ------------------
912 mov (+8*5)($pA), $A
913
914 mov $X[6],%rax
915 mul $A
916 add %rax, $X[5]
917 adc \$0, %rdx
918 mov $X[5], (+$pDst_o+8*11)($pDst)
919
920 mov %rdx, $X[0]
921 mov $X[7],%rax
922 mul $A
923 add %rax, $X[1]
924 adc \$0, %rdx
925 add $X[0], $X[1]
926 adc \$0, %rdx
927 mov $X[1], (+$pDst_o+8*12)($pDst)
928
929 mov %rdx, $X[2]
930
931 # ------------------
932 # seventh pass 67
933 # ------------------
934 mov $X[6], $A
935
936 mov $X[7],%rax
937 mul $A
938 add %rax, $X[2]
939 adc \$0, %rdx
940 mov $X[2], (+$pDst_o+8*13)($pDst)
941
942 mov %rdx, (+$pDst_o+8*14)($pDst)
943
944 # start finalize (add in squares, and double off-terms)
945 mov (+$pDst_o+8*1)($pDst), $X[0]
946 mov (+$pDst_o+8*2)($pDst), $X[1]
947 mov (+$pDst_o+8*3)($pDst), $X[2]
948 mov (+$pDst_o+8*4)($pDst), $X[3]
949 mov (+$pDst_o+8*5)($pDst), $X[4]
950 mov (+$pDst_o+8*6)($pDst), $X[5]
951
952 mov (+8*3)($pA), %rax
953 mul %rax
954 mov %rax, $x6
955 mov %rdx, $X[6]
956
957 add $X[0], $X[0]
958 adc $X[1], $X[1]
959 adc $X[2], $X[2]
960 adc $X[3], $X[3]
961 adc $X[4], $X[4]
962 adc $X[5], $X[5]
963 adc \$0, $X[6]
964
965 mov (+8*0)($pA), %rax
966 mul %rax
967 mov %rax, (+$pDst_o+8*0)($pDst)
968 mov %rdx, $A
969
970 mov (+8*1)($pA), %rax
971 mul %rax
972
973 add $A, $X[0]
974 adc %rax, $X[1]
975 adc \$0, %rdx
976
977 mov %rdx, $A
978 mov $X[0], (+$pDst_o+8*1)($pDst)
979 mov $X[1], (+$pDst_o+8*2)($pDst)
980
981 mov (+8*2)($pA), %rax
982 mul %rax
983
984 add $A, $X[2]
985 adc %rax, $X[3]
986 adc \$0, %rdx
987
988 mov %rdx, $A
989
990 mov $X[2], (+$pDst_o+8*3)($pDst)
991 mov $X[3], (+$pDst_o+8*4)($pDst)
992
993 xor $tmp, $tmp
994 add $A, $X[4]
995 adc $x6, $X[5]
996 adc \$0, $tmp
997
998 mov $X[4], (+$pDst_o+8*5)($pDst)
999 mov $X[5], (+$pDst_o+8*6)($pDst)
1000
1001 # %%tmp has 0/1 in column 7
1002 # %%A6 has a full value in column 7
1003
1004 mov (+$pDst_o+8*7)($pDst), $X[0]
1005 mov (+$pDst_o+8*8)($pDst), $X[1]
1006 mov (+$pDst_o+8*9)($pDst), $X[2]
1007 mov (+$pDst_o+8*10)($pDst), $X[3]
1008 mov (+$pDst_o+8*11)($pDst), $X[4]
1009 mov (+$pDst_o+8*12)($pDst), $X[5]
1010 mov (+$pDst_o+8*13)($pDst), $x6
1011 mov (+$pDst_o+8*14)($pDst), $x7
1012
1013 mov $X[7], %rax
1014 mul %rax
1015 mov %rax, $X[7]
1016 mov %rdx, $A
1017
1018 add $X[0], $X[0]
1019 adc $X[1], $X[1]
1020 adc $X[2], $X[2]
1021 adc $X[3], $X[3]
1022 adc $X[4], $X[4]
1023 adc $X[5], $X[5]
1024 adc $x6, $x6
1025 adc $x7, $x7
1026 adc \$0, $A
1027
1028 add $tmp, $X[0]
1029
1030 mov (+8*4)($pA), %rax
1031 mul %rax
1032
1033 add $X[6], $X[0]
1034 adc %rax, $X[1]
1035 adc \$0, %rdx
1036
1037 mov %rdx, $tmp
1038
1039 mov $X[0], (+$pDst_o+8*7)($pDst)
1040 mov $X[1], (+$pDst_o+8*8)($pDst)
1041
1042 mov (+8*5)($pA), %rax
1043 mul %rax
1044
1045 add $tmp, $X[2]
1046 adc %rax, $X[3]
1047 adc \$0, %rdx
1048
1049 mov %rdx, $tmp
1050
1051 mov $X[2], (+$pDst_o+8*9)($pDst)
1052 mov $X[3], (+$pDst_o+8*10)($pDst)
1053
1054 mov (+8*6)($pA), %rax
1055 mul %rax
1056
1057 add $tmp, $X[4]
1058 adc %rax, $X[5]
1059 adc \$0, %rdx
1060
1061 mov $X[4], (+$pDst_o+8*11)($pDst)
1062 mov $X[5], (+$pDst_o+8*12)($pDst)
1063
1064 add %rdx, $x6
1065 adc $X[7], $x7
1066 adc \$0, $A
1067
1068 mov $x6, (+$pDst_o+8*13)($pDst)
1069 mov $x7, (+$pDst_o+8*14)($pDst)
1070 mov $A, (+$pDst_o+8*15)($pDst)
1071___
1072}
1073
1074#
1075# sqr_reduce: subroutine to compute Result = reduce(Result * Result)
1076#
1077# input and result also in: r9, r8, r15, r14, r13, r12, r11, r10
1078#
1079$code.=<<___;
1080.type sqr_reduce,\@abi-omnipotent
1081.align 16
1082sqr_reduce:
1083 _CET_ENDBR
1084 mov (+$pResult_offset+8)(%rsp), %rcx
1085___
1086 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi");
1087$code.=<<___;
1088 # tail recursion optimization: jmp to mont_reduce and return from there
1089 jmp mont_reduce
1090 # call mont_reduce
1091 # ret
1092.size sqr_reduce,.-sqr_reduce
1093___
1094}}}
1095
1096#
1097# MAIN FUNCTION
1098#
1099
1100#mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */
1101# UINT64 *g, /* 512 bits, 8 qwords */
1102# UINT64 *exp, /* 512 bits, 8 qwords */
1103# struct mod_ctx_512 *data)
1104
1105# window size = 5
1106# table size = 2^5 = 32
1107#table_entries equ 32
1108#table_size equ table_entries * 8
1109$code.=<<___;
1110.globl mod_exp_512
1111.type mod_exp_512,\@function,4
1112mod_exp_512:
1113 _CET_ENDBR
1114 push %rbp
1115 push %rbx
1116 push %r12
1117 push %r13
1118 push %r14
1119 push %r15
1120
1121 # adjust stack down and then align it with cache boundary
1122 mov %rsp, %r8
1123 sub \$$mem_size, %rsp
1124 and \$-64, %rsp
1125
1126 # store previous stack pointer and arguments
1127 mov %r8, (+$rsp_offset)(%rsp)
1128 mov %rdi, (+$pResult_offset)(%rsp)
1129 mov %rsi, (+$pG_offset)(%rsp)
1130 mov %rcx, (+$pData_offset)(%rsp)
1131.Lbody:
1132 # transform g into montgomery space
1133 # GT = reduce(g * C2) = reduce(g * (2^256))
1134 # reduce expects to have the input in [tmp16]
1135 pxor %xmm4, %xmm4
1136 movdqu (+16*0)(%rsi), %xmm0
1137 movdqu (+16*1)(%rsi), %xmm1
1138 movdqu (+16*2)(%rsi), %xmm2
1139 movdqu (+16*3)(%rsi), %xmm3
1140 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp)
1141 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp)
1142 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1143 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1144 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp)
1145 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp)
1146 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp)
1147 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp)
1148
1149 # load pExp before rdx gets blown away
1150 movdqu (+16*0)(%rdx), %xmm0
1151 movdqu (+16*1)(%rdx), %xmm1
1152 movdqu (+16*2)(%rdx), %xmm2
1153 movdqu (+16*3)(%rdx), %xmm3
1154
1155 lea (+$GT_offset)(%rsp), %rbx
1156 mov %rbx, (+$red_result_addr_offset)(%rsp)
1157 call mont_reduce
1158
1159 # Initialize tmp = C
1160 lea (+$tmp_offset)(%rsp), %rcx
1161 xor %rax, %rax
1162 mov %rax, (+8*0)(%rcx)
1163 mov %rax, (+8*1)(%rcx)
1164 mov %rax, (+8*3)(%rcx)
1165 mov %rax, (+8*4)(%rcx)
1166 mov %rax, (+8*5)(%rcx)
1167 mov %rax, (+8*6)(%rcx)
1168 mov %rax, (+8*7)(%rcx)
1169 mov %rax, (+$exp_offset+8*8)(%rsp)
1170 movq \$1, (+8*2)(%rcx)
1171
1172 lea (+$garray_offset)(%rsp), %rbp
1173 mov %rcx, %rsi # pTmp
1174 mov %rbp, %rdi # Garray[][0]
1175___
1176
1177 &swizzle("%rdi", "%rcx", "%rax", "%rbx");
1178
1179 # for (rax = 31; rax != 0; rax--) {
1180 # tmp = reduce(tmp * G)
1181 # swizzle(pg, tmp);
1182 # pg += 2; }
1183$code.=<<___;
1184 mov \$31, %rax
1185 mov %rax, (+$i_offset)(%rsp)
1186 mov %rbp, (+$pg_offset)(%rsp)
1187 # rsi -> pTmp
1188 mov %rsi, (+$red_result_addr_offset)(%rsp)
1189 mov (+8*0)(%rsi), %r10
1190 mov (+8*1)(%rsi), %r11
1191 mov (+8*2)(%rsi), %r12
1192 mov (+8*3)(%rsi), %r13
1193 mov (+8*4)(%rsi), %r14
1194 mov (+8*5)(%rsi), %r15
1195 mov (+8*6)(%rsi), %r8
1196 mov (+8*7)(%rsi), %r9
1197init_loop:
1198 lea (+$GT_offset)(%rsp), %rdi
1199 call mont_mul_a3b
1200 lea (+$tmp_offset)(%rsp), %rsi
1201 mov (+$pg_offset)(%rsp), %rbp
1202 add \$2, %rbp
1203 mov %rbp, (+$pg_offset)(%rsp)
1204 mov %rsi, %rcx # rcx = rsi = addr of tmp
1205___
1206
1207 &swizzle("%rbp", "%rcx", "%rax", "%rbx");
1208$code.=<<___;
1209 mov (+$i_offset)(%rsp), %rax
1210 sub \$1, %rax
1211 mov %rax, (+$i_offset)(%rsp)
1212 jne init_loop
1213
1214 #
1215 # Copy exponent onto stack
1216 movdqa %xmm0, (+$exp_offset+16*0)(%rsp)
1217 movdqa %xmm1, (+$exp_offset+16*1)(%rsp)
1218 movdqa %xmm2, (+$exp_offset+16*2)(%rsp)
1219 movdqa %xmm3, (+$exp_offset+16*3)(%rsp)
1220
1221
1222 #
1223 # Do exponentiation
1224 # Initialize result to G[exp{511:507}]
1225 mov (+$exp_offset+62)(%rsp), %eax
1226 mov %rax, %rdx
1227 shr \$11, %rax
1228 and \$0x07FF, %edx
1229 mov %edx, (+$exp_offset+62)(%rsp)
1230 lea (+$garray_offset)(%rsp,%rax,2), %rsi
1231 mov (+$pResult_offset)(%rsp), %rdx
1232___
1233
1234 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1235
1236 #
1237 # Loop variables
1238 # rcx = [loop_idx] = index: 510-5 to 0 by 5
1239$code.=<<___;
1240 movq \$505, (+$loop_idx_offset)(%rsp)
1241
1242 mov (+$pResult_offset)(%rsp), %rcx
1243 mov %rcx, (+$red_result_addr_offset)(%rsp)
1244 mov (+8*0)(%rcx), %r10
1245 mov (+8*1)(%rcx), %r11
1246 mov (+8*2)(%rcx), %r12
1247 mov (+8*3)(%rcx), %r13
1248 mov (+8*4)(%rcx), %r14
1249 mov (+8*5)(%rcx), %r15
1250 mov (+8*6)(%rcx), %r8
1251 mov (+8*7)(%rcx), %r9
1252 jmp sqr_2
1253
1254main_loop_a3b:
1255 call sqr_reduce
1256 call sqr_reduce
1257 call sqr_reduce
1258sqr_2:
1259 call sqr_reduce
1260 call sqr_reduce
1261
1262 #
1263 # Do multiply, first look up proper value in Garray
1264 mov (+$loop_idx_offset)(%rsp), %rcx # bit index
1265 mov %rcx, %rax
1266 shr \$4, %rax # rax is word pointer
1267 mov (+$exp_offset)(%rsp,%rax,2), %edx
1268 and \$15, %rcx
1269 shrq %cl, %rdx
1270 and \$0x1F, %rdx
1271
1272 lea (+$garray_offset)(%rsp,%rdx,2), %rsi
1273 lea (+$tmp_offset)(%rsp), %rdx
1274 mov %rdx, %rdi
1275___
1276
1277 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1278 # rdi = tmp = pG
1279
1280 #
1281 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData)
1282 # result result pG M Data
1283$code.=<<___;
1284 mov (+$pResult_offset)(%rsp), %rsi
1285 call mont_mul_a3b
1286
1287 #
1288 # finish loop
1289 mov (+$loop_idx_offset)(%rsp), %rcx
1290 sub \$5, %rcx
1291 mov %rcx, (+$loop_idx_offset)(%rsp)
1292 jge main_loop_a3b
1293
1294 #
1295
1296end_main_loop_a3b:
1297 # transform result out of Montgomery space
1298 # result = reduce(result)
1299 mov (+$pResult_offset)(%rsp), %rdx
1300 pxor %xmm4, %xmm4
1301 movdqu (+16*0)(%rdx), %xmm0
1302 movdqu (+16*1)(%rdx), %xmm1
1303 movdqu (+16*2)(%rdx), %xmm2
1304 movdqu (+16*3)(%rdx), %xmm3
1305 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp)
1306 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp)
1307 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1308 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1309 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp)
1310 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp)
1311 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp)
1312 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp)
1313 call mont_reduce
1314
1315 # If result > m, subtract m
1316 # load result into r15:r8
1317 mov (+$pResult_offset)(%rsp), %rax
1318 mov (+8*0)(%rax), %r8
1319 mov (+8*1)(%rax), %r9
1320 mov (+8*2)(%rax), %r10
1321 mov (+8*3)(%rax), %r11
1322 mov (+8*4)(%rax), %r12
1323 mov (+8*5)(%rax), %r13
1324 mov (+8*6)(%rax), %r14
1325 mov (+8*7)(%rax), %r15
1326
1327 # subtract m
1328 mov (+$pData_offset)(%rsp), %rbx
1329 add \$$M, %rbx
1330
1331 sub (+8*0)(%rbx), %r8
1332 sbb (+8*1)(%rbx), %r9
1333 sbb (+8*2)(%rbx), %r10
1334 sbb (+8*3)(%rbx), %r11
1335 sbb (+8*4)(%rbx), %r12
1336 sbb (+8*5)(%rbx), %r13
1337 sbb (+8*6)(%rbx), %r14
1338 sbb (+8*7)(%rbx), %r15
1339
1340 # if Carry is clear, replace result with difference
1341 mov (+8*0)(%rax), %rsi
1342 mov (+8*1)(%rax), %rdi
1343 mov (+8*2)(%rax), %rcx
1344 mov (+8*3)(%rax), %rdx
1345 cmovnc %r8, %rsi
1346 cmovnc %r9, %rdi
1347 cmovnc %r10, %rcx
1348 cmovnc %r11, %rdx
1349 mov %rsi, (+8*0)(%rax)
1350 mov %rdi, (+8*1)(%rax)
1351 mov %rcx, (+8*2)(%rax)
1352 mov %rdx, (+8*3)(%rax)
1353
1354 mov (+8*4)(%rax), %rsi
1355 mov (+8*5)(%rax), %rdi
1356 mov (+8*6)(%rax), %rcx
1357 mov (+8*7)(%rax), %rdx
1358 cmovnc %r12, %rsi
1359 cmovnc %r13, %rdi
1360 cmovnc %r14, %rcx
1361 cmovnc %r15, %rdx
1362 mov %rsi, (+8*4)(%rax)
1363 mov %rdi, (+8*5)(%rax)
1364 mov %rcx, (+8*6)(%rax)
1365 mov %rdx, (+8*7)(%rax)
1366
1367 mov (+$rsp_offset)(%rsp), %rsi
1368 mov 0(%rsi),%r15
1369 mov 8(%rsi),%r14
1370 mov 16(%rsi),%r13
1371 mov 24(%rsi),%r12
1372 mov 32(%rsi),%rbx
1373 mov 40(%rsi),%rbp
1374 lea 48(%rsi),%rsp
1375.Lepilogue:
1376 ret
1377.size mod_exp_512, . - mod_exp_512
1378___
1379
1380sub reg_part {
1381my ($reg,$conv)=@_;
1382 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
1383 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
1384 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
1385 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
1386 return $reg;
1387}
1388
1389$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
1390$code =~ s/\`([^\`]*)\`/eval $1/gem;
1391$code =~ s/(\(\+[^)]+\))/eval $1/gem;
1392print $code;
1393close STDOUT;
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-08-04 06:59:10 +0000