summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/libcrypto/bn/asm/modexp512-x86_64.pl')
-rw-r--r--src/lib/libcrypto/bn/asm/modexp512-x86_64.pl1496
1 files changed, 0 insertions, 1496 deletions
diff --git a/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl b/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
deleted file mode 100644
index 54aeb01921..0000000000
--- a/src/lib/libcrypto/bn/asm/modexp512-x86_64.pl
+++ /dev/null
@@ -1,1496 +0,0 @@
1#!/usr/bin/env perl
2#
3# Copyright (c) 2010-2011 Intel Corp.
4# Author: Vinodh.Gopal@intel.com
5# Jim Guilford
6# Erdinc.Ozturk@intel.com
7# Maxim.Perminov@intel.com
8#
9# More information about algorithm used can be found at:
10# http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf
11#
12# ====================================================================
13# Copyright (c) 2011 The OpenSSL Project. All rights reserved.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions
17# are met:
18#
19# 1. Redistributions of source code must retain the above copyright
20# notice, this list of conditions and the following disclaimer.
21#
22# 2. Redistributions in binary form must reproduce the above copyright
23# notice, this list of conditions and the following disclaimer in
24# the documentation and/or other materials provided with the
25# distribution.
26#
27# 3. All advertising materials mentioning features or use of this
28# software must display the following acknowledgment:
29# "This product includes software developed by the OpenSSL Project
30# for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31#
32# 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
33# endorse or promote products derived from this software without
34# prior written permission. For written permission, please contact
35# licensing@OpenSSL.org.
36#
37# 5. Products derived from this software may not be called "OpenSSL"
38# nor may "OpenSSL" appear in their names without prior written
39# permission of the OpenSSL Project.
40#
41# 6. Redistributions of any form whatsoever must retain the following
42# acknowledgment:
43# "This product includes software developed by the OpenSSL Project
44# for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45#
46# THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
47# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
50# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
52# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
53# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
55# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
56# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
57# OF THE POSSIBILITY OF SUCH DAMAGE.
58# ====================================================================
59
60$flavour = shift;
61$output = shift;
62if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63
64my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
65
66$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
68( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
69die "can't locate x86_64-xlate.pl";
70
71open STDOUT,"| $^X $xlate $flavour $output";
72
73use strict;
74my $code=".text\n\n";
75my $m=0;
76
77#
78# Define x512 macros
79#
80
81#MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2
82#
83# uses rax, rdx, and args
84sub MULSTEP_512_ADD
85{
86 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_;
87 my @X=@$x; # make a copy
88$code.=<<___;
89 mov (+8*0)($SRC2), %rax
90 mul $OP # rdx:rax = %OP * [0]
91 mov ($ASRC), $X[0]
92 add %rax, $X[0]
93 adc \$0, %rdx
94 mov $X[0], $DST
95___
96for(my $i=1;$i<8;$i++) {
97$code.=<<___;
98 mov %rdx, $TMP
99
100 mov (+8*$i)($SRC2), %rax
101 mul $OP # rdx:rax = %OP * [$i]
102 mov (+8*$i)($ASRC), $X[$i]
103 add %rax, $X[$i]
104 adc \$0, %rdx
105 add $TMP, $X[$i]
106 adc \$0, %rdx
107___
108}
109$code.=<<___;
110 mov %rdx, $X[0]
111___
112}
113
114#MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp
115#
116# uses rax, rdx, and args
117sub MULSTEP_512
118{
119 my ($x, $DST, $SRC2, $OP, $TMP)=@_;
120 my @X=@$x; # make a copy
121$code.=<<___;
122 mov (+8*0)($SRC2), %rax
123 mul $OP # rdx:rax = %OP * [0]
124 add %rax, $X[0]
125 adc \$0, %rdx
126 mov $X[0], $DST
127___
128for(my $i=1;$i<8;$i++) {
129$code.=<<___;
130 mov %rdx, $TMP
131
132 mov (+8*$i)($SRC2), %rax
133 mul $OP # rdx:rax = %OP * [$i]
134 add %rax, $X[$i]
135 adc \$0, %rdx
136 add $TMP, $X[$i]
137 adc \$0, %rdx
138___
139}
140$code.=<<___;
141 mov %rdx, $X[0]
142___
143}
144
145#
146# Swizzle Macros
147#
148
149# macro to copy data from flat space to swizzled table
150#MACRO swizzle pDst, pSrc, tmp1, tmp2
151# pDst and pSrc are modified
152sub swizzle
153{
154 my ($pDst, $pSrc, $cnt, $d0)=@_;
155$code.=<<___;
156 mov \$8, $cnt
157loop_$m:
158 mov ($pSrc), $d0
159 mov $d0#w, ($pDst)
160 shr \$16, $d0
161 mov $d0#w, (+64*1)($pDst)
162 shr \$16, $d0
163 mov $d0#w, (+64*2)($pDst)
164 shr \$16, $d0
165 mov $d0#w, (+64*3)($pDst)
166 lea 8($pSrc), $pSrc
167 lea 64*4($pDst), $pDst
168 dec $cnt
169 jnz loop_$m
170___
171
172 $m++;
173}
174
175# macro to copy data from swizzled table to flat space
176#MACRO unswizzle pDst, pSrc, tmp*3
177sub unswizzle
178{
179 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_;
180$code.=<<___;
181 mov \$4, $cnt
182loop_$m:
183 movzxw (+64*3+256*0)($pSrc), $d0
184 movzxw (+64*3+256*1)($pSrc), $d1
185 shl \$16, $d0
186 shl \$16, $d1
187 mov (+64*2+256*0)($pSrc), $d0#w
188 mov (+64*2+256*1)($pSrc), $d1#w
189 shl \$16, $d0
190 shl \$16, $d1
191 mov (+64*1+256*0)($pSrc), $d0#w
192 mov (+64*1+256*1)($pSrc), $d1#w
193 shl \$16, $d0
194 shl \$16, $d1
195 mov (+64*0+256*0)($pSrc), $d0#w
196 mov (+64*0+256*1)($pSrc), $d1#w
197 mov $d0, (+8*0)($pDst)
198 mov $d1, (+8*1)($pDst)
199 lea 256*2($pSrc), $pSrc
200 lea 8*2($pDst), $pDst
201 sub \$1, $cnt
202 jnz loop_$m
203___
204
205 $m++;
206}
207
208#
209# Data Structures
210#
211
212# Reduce Data
213#
214#
215# Offset Value
216# 0C0 Carries
217# 0B8 X2[10]
218# 0B0 X2[9]
219# 0A8 X2[8]
220# 0A0 X2[7]
221# 098 X2[6]
222# 090 X2[5]
223# 088 X2[4]
224# 080 X2[3]
225# 078 X2[2]
226# 070 X2[1]
227# 068 X2[0]
228# 060 X1[12] P[10]
229# 058 X1[11] P[9] Z[8]
230# 050 X1[10] P[8] Z[7]
231# 048 X1[9] P[7] Z[6]
232# 040 X1[8] P[6] Z[5]
233# 038 X1[7] P[5] Z[4]
234# 030 X1[6] P[4] Z[3]
235# 028 X1[5] P[3] Z[2]
236# 020 X1[4] P[2] Z[1]
237# 018 X1[3] P[1] Z[0]
238# 010 X1[2] P[0] Y[2]
239# 008 X1[1] Q[1] Y[1]
240# 000 X1[0] Q[0] Y[0]
241
242my $X1_offset = 0; # 13 qwords
243my $X2_offset = $X1_offset + 13*8; # 11 qwords
244my $Carries_offset = $X2_offset + 11*8; # 1 qword
245my $Q_offset = 0; # 2 qwords
246my $P_offset = $Q_offset + 2*8; # 11 qwords
247my $Y_offset = 0; # 3 qwords
248my $Z_offset = $Y_offset + 3*8; # 9 qwords
249
250my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords)
251
252#
253# Stack Frame
254#
255#
256# offset value
257# ... <old stack contents>
258# ...
259# 280 Garray
260
261# 278 tmp16[15]
262# ... ...
263# 200 tmp16[0]
264
265# 1F8 tmp[7]
266# ... ...
267# 1C0 tmp[0]
268
269# 1B8 GT[7]
270# ... ...
271# 180 GT[0]
272
273# 178 Reduce Data
274# ... ...
275# 0B8 Reduce Data
276# 0B0 reserved
277# 0A8 reserved
278# 0A0 reserved
279# 098 reserved
280# 090 reserved
281# 088 reduce result addr
282# 080 exp[8]
283
284# ...
285# 048 exp[1]
286# 040 exp[0]
287
288# 038 reserved
289# 030 loop_idx
290# 028 pg
291# 020 i
292# 018 pData ; arg 4
293# 010 pG ; arg 2
294# 008 pResult ; arg 1
295# 000 rsp ; stack pointer before subtract
296
297my $rsp_offset = 0;
298my $pResult_offset = 8*1 + $rsp_offset;
299my $pG_offset = 8*1 + $pResult_offset;
300my $pData_offset = 8*1 + $pG_offset;
301my $i_offset = 8*1 + $pData_offset;
302my $pg_offset = 8*1 + $i_offset;
303my $loop_idx_offset = 8*1 + $pg_offset;
304my $reserved1_offset = 8*1 + $loop_idx_offset;
305my $exp_offset = 8*1 + $reserved1_offset;
306my $red_result_addr_offset= 8*9 + $exp_offset;
307my $reserved2_offset = 8*1 + $red_result_addr_offset;
308my $Reduce_Data_offset = 8*5 + $reserved2_offset;
309my $GT_offset = $Red_Data_Size + $Reduce_Data_offset;
310my $tmp_offset = 8*8 + $GT_offset;
311my $tmp16_offset = 8*8 + $tmp_offset;
312my $garray_offset = 8*16 + $tmp16_offset;
313my $mem_size = 8*8*32 + $garray_offset;
314
315#
316# Offsets within Reduce Data
317#
318#
319# struct MODF_2FOLD_MONT_512_C1_DATA {
320# UINT64 t[8][8];
321# UINT64 m[8];
322# UINT64 m1[8]; /* 2^768 % m */
323# UINT64 m2[8]; /* 2^640 % m */
324# UINT64 k1[2]; /* (- 1/m) % 2^128 */
325# };
326
327my $T = 0;
328my $M = 512; # = 8 * 8 * 8
329my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */
330my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */
331my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */
332
333#
334# FUNCTIONS
335#
336
337{{{
338#
339# MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords)
340# and add 512-bits (8 qwords)
341# to get 640 bits (10 qwords)
342# Input: 128-bit mul source: [rdi+8*1], rbp
343# 512-bit mul source: [rsi+8*n]
344# 512-bit add source: r15, r14, ..., r9, r8
345# Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0]
346# Clobbers all regs except: rcx, rsi, rdi
347$code.=<<___;
348.type MULADD_128x512,\@abi-omnipotent
349.align 16
350MULADD_128x512:
351___
352 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx");
353$code.=<<___;
354 mov (+8*1)(%rdi), %rbp
355___
356 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx");
357$code.=<<___;
358 ret
359.size MULADD_128x512,.-MULADD_128x512
360___
361}}}
362
363{{{
364#MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0
365#
366# Inputs: pDst: Destination (768 bits, 12 qwords)
367# pA: Multiplicand (1024 bits, 16 qwords)
368# pB: Multiplicand (512 bits, 8 qwords)
369# Dst = Ah * B + Al
370# where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits)
371# Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0]
372# Uses registers: arguments, RAX, RDX
373sub MULADD_256x512
374{
375 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_;
376$code.=<<___;
377 mov (+8*12)($pA), $OP
378___
379 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP);
380 push(@$X,shift(@$X));
381
382$code.=<<___;
383 mov (+8*13)($pA), $OP
384___
385 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP);
386 push(@$X,shift(@$X));
387
388$code.=<<___;
389 mov (+8*14)($pA), $OP
390___
391 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP);
392 push(@$X,shift(@$X));
393
394$code.=<<___;
395 mov (+8*15)($pA), $OP
396___
397 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP);
398 push(@$X,shift(@$X));
399}
400
401#
402# mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */
403# UINT64 *m, /* 512 bits, 8 qwords */
404# MODF_2FOLD_MONT_512_C1_DATA *data,
405# UINT64 *r) /* 512 bits, 8 qwords */
406# Input: x (number to be reduced): tmp16 (Implicit)
407# m (modulus): [pM] (Implicit)
408# data (reduce data): [pData] (Implicit)
409# Output: r (result): Address in [red_res_addr]
410# result also in: r9, r8, r15, r14, r13, r12, r11, r10
411
412my @X=map("%r$_",(8..15));
413
414$code.=<<___;
415.type mont_reduce,\@abi-omnipotent
416.align 16
417mont_reduce:
418___
419
420my $STACK_DEPTH = 8;
421 #
422 # X1 = Xh * M1 + Xl
423$code.=<<___;
424 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords
425 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords
426 add \$$M1, %rsi
427 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords
428
429___
430
431 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times
432 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0]
433
434$code.=<<___;
435 xor %rax, %rax
436 # X1 += xl
437 add (+8*8)(%rcx), $X[4]
438 adc (+8*9)(%rcx), $X[5]
439 adc (+8*10)(%rcx), $X[6]
440 adc (+8*11)(%rcx), $X[7]
441 adc \$0, %rax
442 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0]
443
444 #
445 # check for carry ;; carry stored in rax
446 mov $X[4], (+8*8)(%rdi) # rdi points to X1
447 mov $X[5], (+8*9)(%rdi)
448 mov $X[6], %rbp
449 mov $X[7], (+8*11)(%rdi)
450
451 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
452
453 mov (+8*0)(%rdi), $X[4]
454 mov (+8*1)(%rdi), $X[5]
455 mov (+8*2)(%rdi), $X[6]
456 mov (+8*3)(%rdi), $X[7]
457
458 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8
459 # rdi -> X1
460 # rsi -> M1
461
462 #
463 # X2 = Xh * M2 + Xl
464 # do first part (X2 = Xh * M2)
465 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords
466 # Xh is actually { [rdi+8*1], rbp }
467 add \$`$M2-$M1`, %rsi # rsi -> M2
468 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords
469___
470 unshift(@X,pop(@X)); unshift(@X,pop(@X));
471$code.=<<___;
472
473 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
474 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
475 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax
476
477 # X2 += Xl
478 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl
479 adc (+8*9-8*10)(%rdi), $X[7]
480 mov $X[6], (+8*8)(%rcx)
481 mov $X[7], (+8*9)(%rcx)
482
483 adc %rax, %rax
484 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
485
486 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords
487 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords
488
489 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half)
490 # B1:B0 = rsi[1:0] = K1[1:0]
491 # A1:A0 = rcx[1:0] = X2[1:0]
492 # Result = rdi[1],rbp = Q[1],rbp
493 mov (%rsi), %r8 # B0
494 mov (+8*1)(%rsi), %rbx # B1
495
496 mov (%rcx), %rax # A0
497 mul %r8 # B0
498 mov %rax, %rbp
499 mov %rdx, %r9
500
501 mov (+8*1)(%rcx), %rax # A1
502 mul %r8 # B0
503 add %rax, %r9
504
505 mov (%rcx), %rax # A0
506 mul %rbx # B1
507 add %rax, %r9
508
509 mov %r9, (+8*1)(%rdi)
510 # end MUL_128x128t128
511
512 sub \$`$K1-$M`, %rsi
513
514 mov (%rcx), $X[6]
515 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0]
516
517 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
518 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
519
520 # load first half of m to rdx, rdi, rbx, rax
521 # moved this here for efficiency
522 mov (+8*0)(%rsi), %rax
523 mov (+8*1)(%rsi), %rbx
524 mov (+8*2)(%rsi), %rdi
525 mov (+8*3)(%rsi), %rdx
526
527 # continue with reduction
528 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp
529
530 add (+8*8)(%rcx), $X[6]
531 adc (+8*9)(%rcx), $X[7]
532
533 #accumulate the final carry to rbp
534 adc %rbp, %rbp
535
536 # Add in overflow corrections: R = (X2>>128) += T[overflow]
537 # R = {r9, r8, r15, r14, ..., r10}
538 shl \$3, %rbp
539 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T)
540 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out
541
542 # rsi will be used to generate a mask after the addition
543 xor %rsi, %rsi
544
545 add (+8*8*0)(%rbp), $X[0]
546 adc (+8*8*1)(%rbp), $X[1]
547 adc (+8*8*2)(%rbp), $X[2]
548 adc (+8*8*3)(%rbp), $X[3]
549 adc (+8*8*4)(%rbp), $X[4]
550 adc (+8*8*5)(%rbp), $X[5]
551 adc (+8*8*6)(%rbp), $X[6]
552 adc (+8*8*7)(%rbp), $X[7]
553
554 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF
555 # if carry is clear: rsi = 0x0000000000000000
556 sbb \$0, %rsi
557
558 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
559 and %rsi, %rax
560 and %rsi, %rbx
561 and %rsi, %rdi
562 and %rsi, %rdx
563
564 mov \$1, %rbp
565 sub %rax, $X[0]
566 sbb %rbx, $X[1]
567 sbb %rdi, $X[2]
568 sbb %rdx, $X[3]
569
570 # if there is a borrow: rbp = 0
571 # if there is no borrow: rbp = 1
572 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m
573 sbb \$0, %rbp
574
575 #load second half of m to rdx, rdi, rbx, rax
576
577 add \$$M, %rcx
578 mov (+8*4)(%rcx), %rax
579 mov (+8*5)(%rcx), %rbx
580 mov (+8*6)(%rcx), %rdi
581 mov (+8*7)(%rcx), %rdx
582
583 # use the rsi mask as before
584 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
585 and %rsi, %rax
586 and %rsi, %rbx
587 and %rsi, %rdi
588 and %rsi, %rdx
589
590 # if rbp = 0, there was a borrow before, it is moved to the carry flag
591 # if rbp = 1, there was not a borrow before, carry flag is cleared
592 sub \$1, %rbp
593
594 sbb %rax, $X[4]
595 sbb %rbx, $X[5]
596 sbb %rdi, $X[6]
597 sbb %rdx, $X[7]
598
599 # write R back to memory
600
601 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi
602 mov $X[0], (+8*0)(%rsi)
603 mov $X[1], (+8*1)(%rsi)
604 mov $X[2], (+8*2)(%rsi)
605 mov $X[3], (+8*3)(%rsi)
606 mov $X[4], (+8*4)(%rsi)
607 mov $X[5], (+8*5)(%rsi)
608 mov $X[6], (+8*6)(%rsi)
609 mov $X[7], (+8*7)(%rsi)
610
611 ret
612.size mont_reduce,.-mont_reduce
613___
614}}}
615
616{{{
617#MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2
618#
619# Inputs: pDst: Destination (1024 bits, 16 qwords)
620# pA: Multiplicand (512 bits, 8 qwords)
621# pB: Multiplicand (512 bits, 8 qwords)
622# Uses registers rax, rdx, args
623# B operand in [pB] and also in x7...x0
624sub MUL_512x512
625{
626 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_;
627 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
628 my @X=@$x; # make a copy
629
630$code.=<<___;
631 mov (+8*0)($pA), $OP
632
633 mov $X[0], %rax
634 mul $OP # rdx:rax = %OP * [0]
635 mov %rax, (+$pDst_o+8*0)($pDst)
636 mov %rdx, $X[0]
637___
638for(my $i=1;$i<8;$i++) {
639$code.=<<___;
640 mov $X[$i], %rax
641 mul $OP # rdx:rax = %OP * [$i]
642 add %rax, $X[$i-1]
643 adc \$0, %rdx
644 mov %rdx, $X[$i]
645___
646}
647
648for(my $i=1;$i<8;$i++) {
649$code.=<<___;
650 mov (+8*$i)($pA), $OP
651___
652
653 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP);
654 push(@X,shift(@X));
655}
656
657$code.=<<___;
658 mov $X[0], (+$pDst_o+8*8)($pDst)
659 mov $X[1], (+$pDst_o+8*9)($pDst)
660 mov $X[2], (+$pDst_o+8*10)($pDst)
661 mov $X[3], (+$pDst_o+8*11)($pDst)
662 mov $X[4], (+$pDst_o+8*12)($pDst)
663 mov $X[5], (+$pDst_o+8*13)($pDst)
664 mov $X[6], (+$pDst_o+8*14)($pDst)
665 mov $X[7], (+$pDst_o+8*15)($pDst)
666___
667}
668
669#
670# mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits)
671# Input: src1: Address of source 1: rdi
672# src2: Address of source 2: rsi
673# Output: dst: Address of destination: [red_res_addr]
674# src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10
675# Temp: Clobbers [tmp16], all registers
676$code.=<<___;
677.type mont_mul_a3b,\@abi-omnipotent
678.align 16
679mont_mul_a3b:
680 #
681 # multiply tmp = src1 * src2
682 # For multiply: dst = rcx, src1 = rdi, src2 = rsi
683 # stack depth is extra 8 from call
684___
685 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx");
686$code.=<<___;
687 #
688 # Dst = tmp % m
689 # Call reduce(tmp, m, data, dst)
690
691 # tail recursion optimization: jmp to mont_reduce and return from there
692 jmp mont_reduce
693 # call mont_reduce
694 # ret
695.size mont_mul_a3b,.-mont_mul_a3b
696___
697}}}
698
699{{{
700#SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4
701#
702# Input in memory [pA] and also in x7...x0
703# Uses all argument registers plus rax and rdx
704#
705# This version computes all of the off-diagonal terms into memory,
706# and then it adds in the diagonal terms
707
708sub SQR_512
709{
710 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_;
711 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
712 my @X=@$x; # make a copy
713$code.=<<___;
714 # ------------------
715 # first pass 01...07
716 # ------------------
717 mov $X[0], $A
718
719 mov $X[1],%rax
720 mul $A
721 mov %rax, (+$pDst_o+8*1)($pDst)
722___
723for(my $i=2;$i<8;$i++) {
724$code.=<<___;
725 mov %rdx, $X[$i-2]
726 mov $X[$i],%rax
727 mul $A
728 add %rax, $X[$i-2]
729 adc \$0, %rdx
730___
731}
732$code.=<<___;
733 mov %rdx, $x7
734
735 mov $X[0], (+$pDst_o+8*2)($pDst)
736
737 # ------------------
738 # second pass 12...17
739 # ------------------
740
741 mov (+8*1)($pA), $A
742
743 mov (+8*2)($pA),%rax
744 mul $A
745 add %rax, $X[1]
746 adc \$0, %rdx
747 mov $X[1], (+$pDst_o+8*3)($pDst)
748
749 mov %rdx, $X[0]
750 mov (+8*3)($pA),%rax
751 mul $A
752 add %rax, $X[2]
753 adc \$0, %rdx
754 add $X[0], $X[2]
755 adc \$0, %rdx
756 mov $X[2], (+$pDst_o+8*4)($pDst)
757
758 mov %rdx, $X[0]
759 mov (+8*4)($pA),%rax
760 mul $A
761 add %rax, $X[3]
762 adc \$0, %rdx
763 add $X[0], $X[3]
764 adc \$0, %rdx
765
766 mov %rdx, $X[0]
767 mov (+8*5)($pA),%rax
768 mul $A
769 add %rax, $X[4]
770 adc \$0, %rdx
771 add $X[0], $X[4]
772 adc \$0, %rdx
773
774 mov %rdx, $X[0]
775 mov $X[6],%rax
776 mul $A
777 add %rax, $X[5]
778 adc \$0, %rdx
779 add $X[0], $X[5]
780 adc \$0, %rdx
781
782 mov %rdx, $X[0]
783 mov $X[7],%rax
784 mul $A
785 add %rax, $x7
786 adc \$0, %rdx
787 add $X[0], $x7
788 adc \$0, %rdx
789
790 mov %rdx, $X[1]
791
792 # ------------------
793 # third pass 23...27
794 # ------------------
795 mov (+8*2)($pA), $A
796
797 mov (+8*3)($pA),%rax
798 mul $A
799 add %rax, $X[3]
800 adc \$0, %rdx
801 mov $X[3], (+$pDst_o+8*5)($pDst)
802
803 mov %rdx, $X[0]
804 mov (+8*4)($pA),%rax
805 mul $A
806 add %rax, $X[4]
807 adc \$0, %rdx
808 add $X[0], $X[4]
809 adc \$0, %rdx
810 mov $X[4], (+$pDst_o+8*6)($pDst)
811
812 mov %rdx, $X[0]
813 mov (+8*5)($pA),%rax
814 mul $A
815 add %rax, $X[5]
816 adc \$0, %rdx
817 add $X[0], $X[5]
818 adc \$0, %rdx
819
820 mov %rdx, $X[0]
821 mov $X[6],%rax
822 mul $A
823 add %rax, $x7
824 adc \$0, %rdx
825 add $X[0], $x7
826 adc \$0, %rdx
827
828 mov %rdx, $X[0]
829 mov $X[7],%rax
830 mul $A
831 add %rax, $X[1]
832 adc \$0, %rdx
833 add $X[0], $X[1]
834 adc \$0, %rdx
835
836 mov %rdx, $X[2]
837
838 # ------------------
839 # fourth pass 34...37
840 # ------------------
841
842 mov (+8*3)($pA), $A
843
844 mov (+8*4)($pA),%rax
845 mul $A
846 add %rax, $X[5]
847 adc \$0, %rdx
848 mov $X[5], (+$pDst_o+8*7)($pDst)
849
850 mov %rdx, $X[0]
851 mov (+8*5)($pA),%rax
852 mul $A
853 add %rax, $x7
854 adc \$0, %rdx
855 add $X[0], $x7
856 adc \$0, %rdx
857 mov $x7, (+$pDst_o+8*8)($pDst)
858
859 mov %rdx, $X[0]
860 mov $X[6],%rax
861 mul $A
862 add %rax, $X[1]
863 adc \$0, %rdx
864 add $X[0], $X[1]
865 adc \$0, %rdx
866
867 mov %rdx, $X[0]
868 mov $X[7],%rax
869 mul $A
870 add %rax, $X[2]
871 adc \$0, %rdx
872 add $X[0], $X[2]
873 adc \$0, %rdx
874
875 mov %rdx, $X[5]
876
877 # ------------------
878 # fifth pass 45...47
879 # ------------------
880 mov (+8*4)($pA), $A
881
882 mov (+8*5)($pA),%rax
883 mul $A
884 add %rax, $X[1]
885 adc \$0, %rdx
886 mov $X[1], (+$pDst_o+8*9)($pDst)
887
888 mov %rdx, $X[0]
889 mov $X[6],%rax
890 mul $A
891 add %rax, $X[2]
892 adc \$0, %rdx
893 add $X[0], $X[2]
894 adc \$0, %rdx
895 mov $X[2], (+$pDst_o+8*10)($pDst)
896
897 mov %rdx, $X[0]
898 mov $X[7],%rax
899 mul $A
900 add %rax, $X[5]
901 adc \$0, %rdx
902 add $X[0], $X[5]
903 adc \$0, %rdx
904
905 mov %rdx, $X[1]
906
907 # ------------------
908 # sixth pass 56...57
909 # ------------------
910 mov (+8*5)($pA), $A
911
912 mov $X[6],%rax
913 mul $A
914 add %rax, $X[5]
915 adc \$0, %rdx
916 mov $X[5], (+$pDst_o+8*11)($pDst)
917
918 mov %rdx, $X[0]
919 mov $X[7],%rax
920 mul $A
921 add %rax, $X[1]
922 adc \$0, %rdx
923 add $X[0], $X[1]
924 adc \$0, %rdx
925 mov $X[1], (+$pDst_o+8*12)($pDst)
926
927 mov %rdx, $X[2]
928
929 # ------------------
930 # seventh pass 67
931 # ------------------
932 mov $X[6], $A
933
934 mov $X[7],%rax
935 mul $A
936 add %rax, $X[2]
937 adc \$0, %rdx
938 mov $X[2], (+$pDst_o+8*13)($pDst)
939
940 mov %rdx, (+$pDst_o+8*14)($pDst)
941
942 # start finalize (add in squares, and double off-terms)
943 mov (+$pDst_o+8*1)($pDst), $X[0]
944 mov (+$pDst_o+8*2)($pDst), $X[1]
945 mov (+$pDst_o+8*3)($pDst), $X[2]
946 mov (+$pDst_o+8*4)($pDst), $X[3]
947 mov (+$pDst_o+8*5)($pDst), $X[4]
948 mov (+$pDst_o+8*6)($pDst), $X[5]
949
950 mov (+8*3)($pA), %rax
951 mul %rax
952 mov %rax, $x6
953 mov %rdx, $X[6]
954
955 add $X[0], $X[0]
956 adc $X[1], $X[1]
957 adc $X[2], $X[2]
958 adc $X[3], $X[3]
959 adc $X[4], $X[4]
960 adc $X[5], $X[5]
961 adc \$0, $X[6]
962
963 mov (+8*0)($pA), %rax
964 mul %rax
965 mov %rax, (+$pDst_o+8*0)($pDst)
966 mov %rdx, $A
967
968 mov (+8*1)($pA), %rax
969 mul %rax
970
971 add $A, $X[0]
972 adc %rax, $X[1]
973 adc \$0, %rdx
974
975 mov %rdx, $A
976 mov $X[0], (+$pDst_o+8*1)($pDst)
977 mov $X[1], (+$pDst_o+8*2)($pDst)
978
979 mov (+8*2)($pA), %rax
980 mul %rax
981
982 add $A, $X[2]
983 adc %rax, $X[3]
984 adc \$0, %rdx
985
986 mov %rdx, $A
987
988 mov $X[2], (+$pDst_o+8*3)($pDst)
989 mov $X[3], (+$pDst_o+8*4)($pDst)
990
991 xor $tmp, $tmp
992 add $A, $X[4]
993 adc $x6, $X[5]
994 adc \$0, $tmp
995
996 mov $X[4], (+$pDst_o+8*5)($pDst)
997 mov $X[5], (+$pDst_o+8*6)($pDst)
998
999 # %%tmp has 0/1 in column 7
1000 # %%A6 has a full value in column 7
1001
1002 mov (+$pDst_o+8*7)($pDst), $X[0]
1003 mov (+$pDst_o+8*8)($pDst), $X[1]
1004 mov (+$pDst_o+8*9)($pDst), $X[2]
1005 mov (+$pDst_o+8*10)($pDst), $X[3]
1006 mov (+$pDst_o+8*11)($pDst), $X[4]
1007 mov (+$pDst_o+8*12)($pDst), $X[5]
1008 mov (+$pDst_o+8*13)($pDst), $x6
1009 mov (+$pDst_o+8*14)($pDst), $x7
1010
1011 mov $X[7], %rax
1012 mul %rax
1013 mov %rax, $X[7]
1014 mov %rdx, $A
1015
1016 add $X[0], $X[0]
1017 adc $X[1], $X[1]
1018 adc $X[2], $X[2]
1019 adc $X[3], $X[3]
1020 adc $X[4], $X[4]
1021 adc $X[5], $X[5]
1022 adc $x6, $x6
1023 adc $x7, $x7
1024 adc \$0, $A
1025
1026 add $tmp, $X[0]
1027
1028 mov (+8*4)($pA), %rax
1029 mul %rax
1030
1031 add $X[6], $X[0]
1032 adc %rax, $X[1]
1033 adc \$0, %rdx
1034
1035 mov %rdx, $tmp
1036
1037 mov $X[0], (+$pDst_o+8*7)($pDst)
1038 mov $X[1], (+$pDst_o+8*8)($pDst)
1039
1040 mov (+8*5)($pA), %rax
1041 mul %rax
1042
1043 add $tmp, $X[2]
1044 adc %rax, $X[3]
1045 adc \$0, %rdx
1046
1047 mov %rdx, $tmp
1048
1049 mov $X[2], (+$pDst_o+8*9)($pDst)
1050 mov $X[3], (+$pDst_o+8*10)($pDst)
1051
1052 mov (+8*6)($pA), %rax
1053 mul %rax
1054
1055 add $tmp, $X[4]
1056 adc %rax, $X[5]
1057 adc \$0, %rdx
1058
1059 mov $X[4], (+$pDst_o+8*11)($pDst)
1060 mov $X[5], (+$pDst_o+8*12)($pDst)
1061
1062 add %rdx, $x6
1063 adc $X[7], $x7
1064 adc \$0, $A
1065
1066 mov $x6, (+$pDst_o+8*13)($pDst)
1067 mov $x7, (+$pDst_o+8*14)($pDst)
1068 mov $A, (+$pDst_o+8*15)($pDst)
1069___
1070}
1071
1072#
1073# sqr_reduce: subroutine to compute Result = reduce(Result * Result)
1074#
1075# input and result also in: r9, r8, r15, r14, r13, r12, r11, r10
1076#
1077$code.=<<___;
1078.type sqr_reduce,\@abi-omnipotent
1079.align 16
1080sqr_reduce:
1081 mov (+$pResult_offset+8)(%rsp), %rcx
1082___
1083 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi");
1084$code.=<<___;
1085 # tail recursion optimization: jmp to mont_reduce and return from there
1086 jmp mont_reduce
1087 # call mont_reduce
1088 # ret
1089.size sqr_reduce,.-sqr_reduce
1090___
1091}}}
1092
1093#
1094# MAIN FUNCTION
1095#
1096
1097#mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */
1098# UINT64 *g, /* 512 bits, 8 qwords */
1099# UINT64 *exp, /* 512 bits, 8 qwords */
1100# struct mod_ctx_512 *data)
1101
1102# window size = 5
1103# table size = 2^5 = 32
1104#table_entries equ 32
1105#table_size equ table_entries * 8
1106$code.=<<___;
1107.globl mod_exp_512
1108.type mod_exp_512,\@function,4
1109mod_exp_512:
1110 push %rbp
1111 push %rbx
1112 push %r12
1113 push %r13
1114 push %r14
1115 push %r15
1116
1117 # adjust stack down and then align it with cache boundary
1118 mov %rsp, %r8
1119 sub \$$mem_size, %rsp
1120 and \$-64, %rsp
1121
1122 # store previous stack pointer and arguments
1123 mov %r8, (+$rsp_offset)(%rsp)
1124 mov %rdi, (+$pResult_offset)(%rsp)
1125 mov %rsi, (+$pG_offset)(%rsp)
1126 mov %rcx, (+$pData_offset)(%rsp)
1127.Lbody:
1128 # transform g into montgomery space
1129 # GT = reduce(g * C2) = reduce(g * (2^256))
1130 # reduce expects to have the input in [tmp16]
1131 pxor %xmm4, %xmm4
1132 movdqu (+16*0)(%rsi), %xmm0
1133 movdqu (+16*1)(%rsi), %xmm1
1134 movdqu (+16*2)(%rsi), %xmm2
1135 movdqu (+16*3)(%rsi), %xmm3
1136 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp)
1137 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp)
1138 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1139 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1140 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp)
1141 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp)
1142 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp)
1143 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp)
1144
1145 # load pExp before rdx gets blown away
1146 movdqu (+16*0)(%rdx), %xmm0
1147 movdqu (+16*1)(%rdx), %xmm1
1148 movdqu (+16*2)(%rdx), %xmm2
1149 movdqu (+16*3)(%rdx), %xmm3
1150
1151 lea (+$GT_offset)(%rsp), %rbx
1152 mov %rbx, (+$red_result_addr_offset)(%rsp)
1153 call mont_reduce
1154
1155 # Initialize tmp = C
1156 lea (+$tmp_offset)(%rsp), %rcx
1157 xor %rax, %rax
1158 mov %rax, (+8*0)(%rcx)
1159 mov %rax, (+8*1)(%rcx)
1160 mov %rax, (+8*3)(%rcx)
1161 mov %rax, (+8*4)(%rcx)
1162 mov %rax, (+8*5)(%rcx)
1163 mov %rax, (+8*6)(%rcx)
1164 mov %rax, (+8*7)(%rcx)
1165 mov %rax, (+$exp_offset+8*8)(%rsp)
1166 movq \$1, (+8*2)(%rcx)
1167
1168 lea (+$garray_offset)(%rsp), %rbp
1169 mov %rcx, %rsi # pTmp
1170 mov %rbp, %rdi # Garray[][0]
1171___
1172
1173 &swizzle("%rdi", "%rcx", "%rax", "%rbx");
1174
1175 # for (rax = 31; rax != 0; rax--) {
1176 # tmp = reduce(tmp * G)
1177 # swizzle(pg, tmp);
1178 # pg += 2; }
1179$code.=<<___;
1180 mov \$31, %rax
1181 mov %rax, (+$i_offset)(%rsp)
1182 mov %rbp, (+$pg_offset)(%rsp)
1183 # rsi -> pTmp
1184 mov %rsi, (+$red_result_addr_offset)(%rsp)
1185 mov (+8*0)(%rsi), %r10
1186 mov (+8*1)(%rsi), %r11
1187 mov (+8*2)(%rsi), %r12
1188 mov (+8*3)(%rsi), %r13
1189 mov (+8*4)(%rsi), %r14
1190 mov (+8*5)(%rsi), %r15
1191 mov (+8*6)(%rsi), %r8
1192 mov (+8*7)(%rsi), %r9
1193init_loop:
1194 lea (+$GT_offset)(%rsp), %rdi
1195 call mont_mul_a3b
1196 lea (+$tmp_offset)(%rsp), %rsi
1197 mov (+$pg_offset)(%rsp), %rbp
1198 add \$2, %rbp
1199 mov %rbp, (+$pg_offset)(%rsp)
1200 mov %rsi, %rcx # rcx = rsi = addr of tmp
1201___
1202
1203 &swizzle("%rbp", "%rcx", "%rax", "%rbx");
1204$code.=<<___;
1205 mov (+$i_offset)(%rsp), %rax
1206 sub \$1, %rax
1207 mov %rax, (+$i_offset)(%rsp)
1208 jne init_loop
1209
1210 #
1211 # Copy exponent onto stack
1212 movdqa %xmm0, (+$exp_offset+16*0)(%rsp)
1213 movdqa %xmm1, (+$exp_offset+16*1)(%rsp)
1214 movdqa %xmm2, (+$exp_offset+16*2)(%rsp)
1215 movdqa %xmm3, (+$exp_offset+16*3)(%rsp)
1216
1217
1218 #
1219 # Do exponentiation
1220 # Initialize result to G[exp{511:507}]
1221 mov (+$exp_offset+62)(%rsp), %eax
1222 mov %rax, %rdx
1223 shr \$11, %rax
1224 and \$0x07FF, %edx
1225 mov %edx, (+$exp_offset+62)(%rsp)
1226 lea (+$garray_offset)(%rsp,%rax,2), %rsi
1227 mov (+$pResult_offset)(%rsp), %rdx
1228___
1229
1230 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1231
1232 #
1233 # Loop variables
1234 # rcx = [loop_idx] = index: 510-5 to 0 by 5
1235$code.=<<___;
1236 movq \$505, (+$loop_idx_offset)(%rsp)
1237
1238 mov (+$pResult_offset)(%rsp), %rcx
1239 mov %rcx, (+$red_result_addr_offset)(%rsp)
1240 mov (+8*0)(%rcx), %r10
1241 mov (+8*1)(%rcx), %r11
1242 mov (+8*2)(%rcx), %r12
1243 mov (+8*3)(%rcx), %r13
1244 mov (+8*4)(%rcx), %r14
1245 mov (+8*5)(%rcx), %r15
1246 mov (+8*6)(%rcx), %r8
1247 mov (+8*7)(%rcx), %r9
1248 jmp sqr_2
1249
1250main_loop_a3b:
1251 call sqr_reduce
1252 call sqr_reduce
1253 call sqr_reduce
1254sqr_2:
1255 call sqr_reduce
1256 call sqr_reduce
1257
1258 #
1259 # Do multiply, first look up proper value in Garray
1260 mov (+$loop_idx_offset)(%rsp), %rcx # bit index
1261 mov %rcx, %rax
1262 shr \$4, %rax # rax is word pointer
1263 mov (+$exp_offset)(%rsp,%rax,2), %edx
1264 and \$15, %rcx
1265 shrq %cl, %rdx
1266 and \$0x1F, %rdx
1267
1268 lea (+$garray_offset)(%rsp,%rdx,2), %rsi
1269 lea (+$tmp_offset)(%rsp), %rdx
1270 mov %rdx, %rdi
1271___
1272
1273 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1274 # rdi = tmp = pG
1275
1276 #
1277 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData)
1278 # result result pG M Data
1279$code.=<<___;
1280 mov (+$pResult_offset)(%rsp), %rsi
1281 call mont_mul_a3b
1282
1283 #
1284 # finish loop
1285 mov (+$loop_idx_offset)(%rsp), %rcx
1286 sub \$5, %rcx
1287 mov %rcx, (+$loop_idx_offset)(%rsp)
1288 jge main_loop_a3b
1289
1290 #
1291
1292end_main_loop_a3b:
1293 # transform result out of Montgomery space
1294 # result = reduce(result)
1295 mov (+$pResult_offset)(%rsp), %rdx
1296 pxor %xmm4, %xmm4
1297 movdqu (+16*0)(%rdx), %xmm0
1298 movdqu (+16*1)(%rdx), %xmm1
1299 movdqu (+16*2)(%rdx), %xmm2
1300 movdqu (+16*3)(%rdx), %xmm3
1301 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp)
1302 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp)
1303 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1304 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1305 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp)
1306 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp)
1307 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp)
1308 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp)
1309 call mont_reduce
1310
1311 # If result > m, subract m
1312 # load result into r15:r8
1313 mov (+$pResult_offset)(%rsp), %rax
1314 mov (+8*0)(%rax), %r8
1315 mov (+8*1)(%rax), %r9
1316 mov (+8*2)(%rax), %r10
1317 mov (+8*3)(%rax), %r11
1318 mov (+8*4)(%rax), %r12
1319 mov (+8*5)(%rax), %r13
1320 mov (+8*6)(%rax), %r14
1321 mov (+8*7)(%rax), %r15
1322
1323 # subtract m
1324 mov (+$pData_offset)(%rsp), %rbx
1325 add \$$M, %rbx
1326
1327 sub (+8*0)(%rbx), %r8
1328 sbb (+8*1)(%rbx), %r9
1329 sbb (+8*2)(%rbx), %r10
1330 sbb (+8*3)(%rbx), %r11
1331 sbb (+8*4)(%rbx), %r12
1332 sbb (+8*5)(%rbx), %r13
1333 sbb (+8*6)(%rbx), %r14
1334 sbb (+8*7)(%rbx), %r15
1335
1336 # if Carry is clear, replace result with difference
1337 mov (+8*0)(%rax), %rsi
1338 mov (+8*1)(%rax), %rdi
1339 mov (+8*2)(%rax), %rcx
1340 mov (+8*3)(%rax), %rdx
1341 cmovnc %r8, %rsi
1342 cmovnc %r9, %rdi
1343 cmovnc %r10, %rcx
1344 cmovnc %r11, %rdx
1345 mov %rsi, (+8*0)(%rax)
1346 mov %rdi, (+8*1)(%rax)
1347 mov %rcx, (+8*2)(%rax)
1348 mov %rdx, (+8*3)(%rax)
1349
1350 mov (+8*4)(%rax), %rsi
1351 mov (+8*5)(%rax), %rdi
1352 mov (+8*6)(%rax), %rcx
1353 mov (+8*7)(%rax), %rdx
1354 cmovnc %r12, %rsi
1355 cmovnc %r13, %rdi
1356 cmovnc %r14, %rcx
1357 cmovnc %r15, %rdx
1358 mov %rsi, (+8*4)(%rax)
1359 mov %rdi, (+8*5)(%rax)
1360 mov %rcx, (+8*6)(%rax)
1361 mov %rdx, (+8*7)(%rax)
1362
1363 mov (+$rsp_offset)(%rsp), %rsi
1364 mov 0(%rsi),%r15
1365 mov 8(%rsi),%r14
1366 mov 16(%rsi),%r13
1367 mov 24(%rsi),%r12
1368 mov 32(%rsi),%rbx
1369 mov 40(%rsi),%rbp
1370 lea 48(%rsi),%rsp
1371.Lepilogue:
1372 ret
1373.size mod_exp_512, . - mod_exp_512
1374___
1375
1376if ($win64) {
1377# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1378# CONTEXT *context,DISPATCHER_CONTEXT *disp)
1379my $rec="%rcx";
1380my $frame="%rdx";
1381my $context="%r8";
1382my $disp="%r9";
1383
1384$code.=<<___;
1385.extern __imp_RtlVirtualUnwind
1386.type mod_exp_512_se_handler,\@abi-omnipotent
1387.align 16
1388mod_exp_512_se_handler:
1389 push %rsi
1390 push %rdi
1391 push %rbx
1392 push %rbp
1393 push %r12
1394 push %r13
1395 push %r14
1396 push %r15
1397 pushfq
1398 sub \$64,%rsp
1399
1400 mov 120($context),%rax # pull context->Rax
1401 mov 248($context),%rbx # pull context->Rip
1402
1403 lea .Lbody(%rip),%r10
1404 cmp %r10,%rbx # context->Rip<prologue label
1405 jb .Lin_prologue
1406
1407 mov 152($context),%rax # pull context->Rsp
1408
1409 lea .Lepilogue(%rip),%r10
1410 cmp %r10,%rbx # context->Rip>=epilogue label
1411 jae .Lin_prologue
1412
1413 mov $rsp_offset(%rax),%rax # pull saved Rsp
1414
1415 mov 32(%rax),%rbx
1416 mov 40(%rax),%rbp
1417 mov 24(%rax),%r12
1418 mov 16(%rax),%r13
1419 mov 8(%rax),%r14
1420 mov 0(%rax),%r15
1421 lea 48(%rax),%rax
1422 mov %rbx,144($context) # restore context->Rbx
1423 mov %rbp,160($context) # restore context->Rbp
1424 mov %r12,216($context) # restore context->R12
1425 mov %r13,224($context) # restore context->R13
1426 mov %r14,232($context) # restore context->R14
1427 mov %r15,240($context) # restore context->R15
1428
1429.Lin_prologue:
1430 mov 8(%rax),%rdi
1431 mov 16(%rax),%rsi
1432 mov %rax,152($context) # restore context->Rsp
1433 mov %rsi,168($context) # restore context->Rsi
1434 mov %rdi,176($context) # restore context->Rdi
1435
1436 mov 40($disp),%rdi # disp->ContextRecord
1437 mov $context,%rsi # context
1438 mov \$154,%ecx # sizeof(CONTEXT)
1439 .long 0xa548f3fc # cld; rep movsq
1440
1441 mov $disp,%rsi
1442 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
1443 mov 8(%rsi),%rdx # arg2, disp->ImageBase
1444 mov 0(%rsi),%r8 # arg3, disp->ControlPc
1445 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
1446 mov 40(%rsi),%r10 # disp->ContextRecord
1447 lea 56(%rsi),%r11 # &disp->HandlerData
1448 lea 24(%rsi),%r12 # &disp->EstablisherFrame
1449 mov %r10,32(%rsp) # arg5
1450 mov %r11,40(%rsp) # arg6
1451 mov %r12,48(%rsp) # arg7
1452 mov %rcx,56(%rsp) # arg8, (NULL)
1453 call *__imp_RtlVirtualUnwind(%rip)
1454
1455 mov \$1,%eax # ExceptionContinueSearch
1456 add \$64,%rsp
1457 popfq
1458 pop %r15
1459 pop %r14
1460 pop %r13
1461 pop %r12
1462 pop %rbp
1463 pop %rbx
1464 pop %rdi
1465 pop %rsi
1466 ret
1467.size mod_exp_512_se_handler,.-mod_exp_512_se_handler
1468
1469.section .pdata
1470.align 4
1471 .rva .LSEH_begin_mod_exp_512
1472 .rva .LSEH_end_mod_exp_512
1473 .rva .LSEH_info_mod_exp_512
1474
1475.section .xdata
1476.align 8
1477.LSEH_info_mod_exp_512:
1478 .byte 9,0,0,0
1479 .rva mod_exp_512_se_handler
1480___
1481}
1482
1483sub reg_part {
1484my ($reg,$conv)=@_;
1485 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
1486 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
1487 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
1488 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
1489 return $reg;
1490}
1491
1492$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
1493$code =~ s/\`([^\`]*)\`/eval $1/gem;
1494$code =~ s/(\(\+[^)]+\))/eval $1/gem;
1495print $code;
1496close STDOUT;
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2026-01-04 23:36:56 +0000