summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/rc4/asm
diff options
context:
space:
mode:
authordjm <>2010-10-01 22:59:01 +0000
committerdjm <>2010-10-01 22:59:01 +0000
commitfe047d8b632246cb2db3234a0a4f32e5c318857b (patch)
tree939b752540947d33507b3acc48d76a8bfb7c3dc3 /src/lib/libcrypto/rc4/asm
parent2ea67f4aa254b09ded62e6e14fc893bbe6381579 (diff)
downloadopenbsd-fe047d8b632246cb2db3234a0a4f32e5c318857b.tar.gz
openbsd-fe047d8b632246cb2db3234a0a4f32e5c318857b.tar.bz2
openbsd-fe047d8b632246cb2db3234a0a4f32e5c318857b.zip
resolve conflicts, fix local changes
Diffstat (limited to 'src/lib/libcrypto/rc4/asm')
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-586.pl448
-rwxr-xr-xsrc/lib/libcrypto/rc4/asm/rc4-amd64.pl227
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-ia64.S159
-rwxr-xr-xsrc/lib/libcrypto/rc4/asm/rc4-x86_64.pl177
4 files changed, 402 insertions, 609 deletions
diff --git a/src/lib/libcrypto/rc4/asm/rc4-586.pl b/src/lib/libcrypto/rc4/asm/rc4-586.pl
index ef7eee766c..38a44a70ef 100644
--- a/src/lib/libcrypto/rc4/asm/rc4-586.pl
+++ b/src/lib/libcrypto/rc4/asm/rc4-586.pl
@@ -1,14 +1,21 @@
1#!/usr/local/bin/perl 1#!/usr/bin/env perl
2
3# ====================================================================
4# [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
2 9
3# At some point it became apparent that the original SSLeay RC4 10# At some point it became apparent that the original SSLeay RC4
4# assembler implementation performs suboptimaly on latest IA-32 11# assembler implementation performs suboptimally on latest IA-32
5# microarchitectures. After re-tuning performance has changed as 12# microarchitectures. After re-tuning performance has changed as
6# following: 13# following:
7# 14#
8# Pentium +0% 15# Pentium -10%
9# Pentium III +17% 16# Pentium III +12%
10# AMD +52%(*) 17# AMD +50%(*)
11# P4 +180%(**) 18# P4 +250%(**)
12# 19#
13# (*) This number is actually a trade-off:-) It's possible to 20# (*) This number is actually a trade-off:-) It's possible to
14# achieve +72%, but at the cost of -48% off PIII performance. 21# achieve +72%, but at the cost of -48% off PIII performance.
@@ -17,214 +24,247 @@
17# For reference! This code delivers ~80% of rc4-amd64.pl 24# For reference! This code delivers ~80% of rc4-amd64.pl
18# performance on the same Opteron machine. 25# performance on the same Opteron machine.
19# (**) This number requires compressed key schedule set up by 26# (**) This number requires compressed key schedule set up by
20# RC4_set_key and therefore doesn't apply to 0.9.7 [option for 27# RC4_set_key [see commentary below for further details].
21# compressed key schedule is implemented in 0.9.8 and later,
22# see commentary section in rc4_skey.c for further details].
23# 28#
24# <appro@fy.chalmers.se> 29# <appro@fy.chalmers.se>
25 30
26push(@INC,"perlasm","../../perlasm"); 31$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
32push(@INC,"${dir}","${dir}../../perlasm");
27require "x86asm.pl"; 33require "x86asm.pl";
28 34
29&asm_init($ARGV[0],"rc4-586.pl"); 35&asm_init($ARGV[0],"rc4-586.pl");
30 36
31$x="eax"; 37$xx="eax";
32$y="ebx"; 38$yy="ebx";
33$tx="ecx"; 39$tx="ecx";
34$ty="edx"; 40$ty="edx";
35$in="esi"; 41$inp="esi";
36$out="edi"; 42$out="ebp";
37$d="ebp"; 43$dat="edi";
38 44
39&RC4("RC4"); 45sub RC4_loop {
40 46 my $i=shift;
41&asm_finish(); 47 my $func = ($i==0)?*mov:*or;
42 48
43sub RC4_loop 49 &add (&LB($yy),&LB($tx));
44 { 50 &mov ($ty,&DWP(0,$dat,$yy,4));
45 local($n,$p,$char)=@_; 51 &mov (&DWP(0,$dat,$yy,4),$tx);
46 52 &mov (&DWP(0,$dat,$xx,4),$ty);
47 &comment("Round $n"); 53 &add ($ty,$tx);
48 54 &inc (&LB($xx));
49 if ($char) 55 &and ($ty,0xff);
50 { 56 &ror ($out,8) if ($i!=0);
51 if ($p >= 0) 57 if ($i<3) {
52 { 58 &mov ($tx,&DWP(0,$dat,$xx,4));
53 &mov($ty, &swtmp(2)); 59 } else {
54 &cmp($ty, $in); 60 &mov ($tx,&wparam(3)); # reload [re-biased] out
55 &jbe(&label("finished"));
56 &inc($in);
57 }
58 else
59 {
60 &add($ty, 8);
61 &inc($in);
62 &cmp($ty, $in);
63 &jb(&label("finished"));
64 &mov(&swtmp(2), $ty);
65 }
66 }
67 # Moved out
68 # &mov( $tx, &DWP(0,$d,$x,4)) if $p < 0;
69
70 &add( &LB($y), &LB($tx));
71 &mov( $ty, &DWP(0,$d,$y,4));
72 # XXX
73 &mov( &DWP(0,$d,$x,4),$ty);
74 &add( $ty, $tx);
75 &mov( &DWP(0,$d,$y,4),$tx);
76 &and( $ty, 0xff);
77 &inc( &LB($x)); # NEXT ROUND
78 &mov( $tx, &DWP(0,$d,$x,4)) if $p < 1; # NEXT ROUND
79 &mov( $ty, &DWP(0,$d,$ty,4));
80
81 if (!$char)
82 {
83 #moved up into last round
84 if ($p >= 1)
85 {
86 &add( $out, 8)
87 }
88 &movb( &BP($n,"esp","",0), &LB($ty));
89 }
90 else
91 {
92 # Note in+=8 has occured
93 &movb( &HB($ty), &BP(-1,$in,"",0));
94 # XXX
95 &xorb(&LB($ty), &HB($ty));
96 # XXX
97 &movb(&BP($n,$out,"",0),&LB($ty));
98 }
99 } 61 }
100 62 &$func ($out,&DWP(0,$dat,$ty,4));
101 63}
102sub RC4 64
103 { 65# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
104 local($name)=@_; 66&function_begin("RC4");
105 67 &mov ($dat,&wparam(0)); # load key schedule pointer
106 &function_begin_B($name,""); 68 &mov ($ty, &wparam(1)); # load len
107 69 &mov ($inp,&wparam(2)); # load inp
108 &mov($ty,&wparam(1)); # len 70 &mov ($out,&wparam(3)); # load out
109 &cmp($ty,0); 71
110 &jne(&label("proceed")); 72 &xor ($xx,$xx); # avoid partial register stalls
111 &ret(); 73 &xor ($yy,$yy);
112 &set_label("proceed"); 74
113 75 &cmp ($ty,0); # safety net
114 &comment(""); 76 &je (&label("abort"));
115 77
116 &push("ebp"); 78 &mov (&LB($xx),&BP(0,$dat)); # load key->x
117 &push("ebx"); 79 &mov (&LB($yy),&BP(4,$dat)); # load key->y
118 &push("esi"); 80 &add ($dat,8);
119 &xor( $x, $x); # avoid partial register stalls 81
120 &push("edi"); 82 &lea ($tx,&DWP(0,$inp,$ty));
121 &xor( $y, $y); # avoid partial register stalls 83 &sub ($out,$inp); # re-bias out
122 &mov( $d, &wparam(0)); # key 84 &mov (&wparam(1),$tx); # save input+len
123 &mov( $in, &wparam(2)); 85
124 86 &inc (&LB($xx));
125 &movb( &LB($x), &BP(0,$d,"",1)); 87
126 &movb( &LB($y), &BP(4,$d,"",1)); 88 # detect compressed key schedule...
127 89 &cmp (&DWP(256,$dat),-1);
128 &mov( $out, &wparam(3)); 90 &je (&label("RC4_CHAR"));
129 &inc( &LB($x)); 91
130 92 &mov ($tx,&DWP(0,$dat,$xx,4));
131 &stack_push(3); # 3 temp variables 93
132 &add( $d, 8); 94 &and ($ty,-4); # how many 4-byte chunks?
133 95 &jz (&label("loop1"));
134 # detect compressed schedule, see commentary section in rc4_skey.c... 96
135 # in 0.9.7 context ~50 bytes below RC4_CHAR label remain redundant, 97 &lea ($ty,&DWP(-4,$inp,$ty));
136 # as compressed key schedule is set up in 0.9.8 and later. 98 &mov (&wparam(2),$ty); # save input+(len/4)*4-4
137 &cmp(&DWP(256,$d),-1); 99 &mov (&wparam(3),$out); # $out as accumulator in this loop
138 &je(&label("RC4_CHAR")); 100
139 101 &set_label("loop4",16);
140 &lea( $ty, &DWP(-8,$ty,$in)); 102 for ($i=0;$i<4;$i++) { RC4_loop($i); }
141 103 &ror ($out,8);
142 # check for 0 length input 104 &xor ($out,&DWP(0,$inp));
143 105 &cmp ($inp,&wparam(2)); # compare to input+(len/4)*4-4
144 &mov( &swtmp(2), $ty); # this is now address to exit at 106 &mov (&DWP(0,$tx,$inp),$out);# $tx holds re-biased out here
145 &mov( $tx, &DWP(0,$d,$x,4)); 107 &lea ($inp,&DWP(4,$inp));
146 108 &mov ($tx,&DWP(0,$dat,$xx,4));
147 &cmp( $ty, $in); 109 &jb (&label("loop4"));
148 &jb( &label("end")); # less than 8 bytes 110
149 111 &cmp ($inp,&wparam(1)); # compare to input+len
150 &set_label("start"); 112 &je (&label("done"));
151 113 &mov ($out,&wparam(3)); # restore $out
152 # filling DELAY SLOT 114
153 &add( $in, 8); 115 &set_label("loop1",16);
154 116 &add (&LB($yy),&LB($tx));
155 &RC4_loop(0,-1,0); 117 &mov ($ty,&DWP(0,$dat,$yy,4));
156 &RC4_loop(1,0,0); 118 &mov (&DWP(0,$dat,$yy,4),$tx);
157 &RC4_loop(2,0,0); 119 &mov (&DWP(0,$dat,$xx,4),$ty);
158 &RC4_loop(3,0,0); 120 &add ($ty,$tx);
159 &RC4_loop(4,0,0); 121 &inc (&LB($xx));
160 &RC4_loop(5,0,0); 122 &and ($ty,0xff);
161 &RC4_loop(6,0,0); 123 &mov ($ty,&DWP(0,$dat,$ty,4));
162 &RC4_loop(7,1,0); 124 &xor (&LB($ty),&BP(0,$inp));
163 125 &lea ($inp,&DWP(1,$inp));
164 &comment("apply the cipher text"); 126 &mov ($tx,&DWP(0,$dat,$xx,4));
165 # xor the cipher data with input 127 &cmp ($inp,&wparam(1)); # compare to input+len
166 128 &mov (&BP(-1,$out,$inp),&LB($ty));
167 #&add( $out, 8); #moved up into last round 129 &jb (&label("loop1"));
168 130
169 &mov( $tx, &swtmp(0)); 131 &jmp (&label("done"));
170 &mov( $ty, &DWP(-8,$in,"",0)); 132
171 &xor( $tx, $ty); 133# this is essentially Intel P4 specific codepath...
172 &mov( $ty, &DWP(-4,$in,"",0)); 134&set_label("RC4_CHAR",16);
173 &mov( &DWP(-8,$out,"",0), $tx); 135 &movz ($tx,&BP(0,$dat,$xx));
174 &mov( $tx, &swtmp(1));
175 &xor( $tx, $ty);
176 &mov( $ty, &swtmp(2)); # load end ptr;
177 &mov( &DWP(-4,$out,"",0), $tx);
178 &mov( $tx, &DWP(0,$d,$x,4));
179 &cmp($in, $ty);
180 &jbe(&label("start"));
181
182 &set_label("end");
183
184 # There is quite a bit of extra crap in RC4_loop() for this
185 # first round
186 &RC4_loop(0,-1,1);
187 &RC4_loop(1,0,1);
188 &RC4_loop(2,0,1);
189 &RC4_loop(3,0,1);
190 &RC4_loop(4,0,1);
191 &RC4_loop(5,0,1);
192 &RC4_loop(6,1,1);
193
194 &jmp(&label("finished"));
195
196 &align(16);
197 # this is essentially Intel P4 specific codepath, see rc4_skey.c,
198 # and is engaged in 0.9.8 and later context...
199 &set_label("RC4_CHAR");
200
201 &lea ($ty,&DWP(0,$in,$ty));
202 &mov (&swtmp(2),$ty);
203 &movz ($tx,&BP(0,$d,$x));
204
205 # strangely enough unrolled loop performs over 20% slower... 136 # strangely enough unrolled loop performs over 20% slower...
206 &set_label("RC4_CHAR_loop"); 137 &set_label("cloop1");
207 &add (&LB($y),&LB($tx)); 138 &add (&LB($yy),&LB($tx));
208 &movz ($ty,&BP(0,$d,$y)); 139 &movz ($ty,&BP(0,$dat,$yy));
209 &movb (&BP(0,$d,$y),&LB($tx)); 140 &mov (&BP(0,$dat,$yy),&LB($tx));
210 &movb (&BP(0,$d,$x),&LB($ty)); 141 &mov (&BP(0,$dat,$xx),&LB($ty));
211 &add (&LB($ty),&LB($tx)); 142 &add (&LB($ty),&LB($tx));
212 &movz ($ty,&BP(0,$d,$ty)); 143 &movz ($ty,&BP(0,$dat,$ty));
213 &add (&LB($x),1); 144 &add (&LB($xx),1);
214 &xorb (&LB($ty),&BP(0,$in)); 145 &xor (&LB($ty),&BP(0,$inp));
215 &lea ($in,&DWP(1,$in)); 146 &lea ($inp,&DWP(1,$inp));
216 &movz ($tx,&BP(0,$d,$x)); 147 &movz ($tx,&BP(0,$dat,$xx));
217 &cmp ($in,&swtmp(2)); 148 &cmp ($inp,&wparam(1));
218 &movb (&BP(0,$out),&LB($ty)); 149 &mov (&BP(-1,$out,$inp),&LB($ty));
219 &lea ($out,&DWP(1,$out)); 150 &jb (&label("cloop1"));
220 &jb (&label("RC4_CHAR_loop")); 151
221 152&set_label("done");
222 &set_label("finished"); 153 &dec (&LB($xx));
223 &dec( $x); 154 &mov (&BP(-4,$dat),&LB($yy)); # save key->y
224 &stack_pop(3); 155 &mov (&BP(-8,$dat),&LB($xx)); # save key->x
225 &movb( &BP(-4,$d,"",0),&LB($y)); 156&set_label("abort");
226 &movb( &BP(-8,$d,"",0),&LB($x)); 157&function_end("RC4");
227 158
228 &function_end($name); 159########################################################################
229 } 160
161$inp="esi";
162$out="edi";
163$idi="ebp";
164$ido="ecx";
165$idx="edx";
166
167&external_label("OPENSSL_ia32cap_P");
168
169# void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data);
170&function_begin("RC4_set_key");
171 &mov ($out,&wparam(0)); # load key
172 &mov ($idi,&wparam(1)); # load len
173 &mov ($inp,&wparam(2)); # load data
174 &picmeup($idx,"OPENSSL_ia32cap_P");
175
176 &lea ($out,&DWP(2*4,$out)); # &key->data
177 &lea ($inp,&DWP(0,$inp,$idi)); # $inp to point at the end
178 &neg ($idi);
179 &xor ("eax","eax");
180 &mov (&DWP(-4,$out),$idi); # borrow key->y
181
182 &bt (&DWP(0,$idx),20); # check for bit#20
183 &jc (&label("c1stloop"));
184
185&set_label("w1stloop",16);
186 &mov (&DWP(0,$out,"eax",4),"eax"); # key->data[i]=i;
187 &add (&LB("eax"),1); # i++;
188 &jnc (&label("w1stloop"));
189
190 &xor ($ido,$ido);
191 &xor ($idx,$idx);
192
193&set_label("w2ndloop",16);
194 &mov ("eax",&DWP(0,$out,$ido,4));
195 &add (&LB($idx),&BP(0,$inp,$idi));
196 &add (&LB($idx),&LB("eax"));
197 &add ($idi,1);
198 &mov ("ebx",&DWP(0,$out,$idx,4));
199 &jnz (&label("wnowrap"));
200 &mov ($idi,&DWP(-4,$out));
201 &set_label("wnowrap");
202 &mov (&DWP(0,$out,$idx,4),"eax");
203 &mov (&DWP(0,$out,$ido,4),"ebx");
204 &add (&LB($ido),1);
205 &jnc (&label("w2ndloop"));
206&jmp (&label("exit"));
207
208# Unlike all other x86 [and x86_64] implementations, Intel P4 core
209# [including EM64T] was found to perform poorly with above "32-bit" key
210# schedule, a.k.a. RC4_INT. Performance improvement for IA-32 hand-coded
211# assembler turned out to be 3.5x if re-coded for compressed 8-bit one,
212# a.k.a. RC4_CHAR! It's however inappropriate to just switch to 8-bit
213# schedule for x86[_64], because non-P4 implementations suffer from
214# significant performance losses then, e.g. PIII exhibits >2x
215# deterioration, and so does Opteron. In order to assure optimal
216# all-round performance, we detect P4 at run-time and set up compressed
217# key schedule, which is recognized by RC4 procedure.
218
219&set_label("c1stloop",16);
220 &mov (&BP(0,$out,"eax"),&LB("eax")); # key->data[i]=i;
221 &add (&LB("eax"),1); # i++;
222 &jnc (&label("c1stloop"));
223
224 &xor ($ido,$ido);
225 &xor ($idx,$idx);
226 &xor ("ebx","ebx");
227
228&set_label("c2ndloop",16);
229 &mov (&LB("eax"),&BP(0,$out,$ido));
230 &add (&LB($idx),&BP(0,$inp,$idi));
231 &add (&LB($idx),&LB("eax"));
232 &add ($idi,1);
233 &mov (&LB("ebx"),&BP(0,$out,$idx));
234 &jnz (&label("cnowrap"));
235 &mov ($idi,&DWP(-4,$out));
236 &set_label("cnowrap");
237 &mov (&BP(0,$out,$idx),&LB("eax"));
238 &mov (&BP(0,$out,$ido),&LB("ebx"));
239 &add (&LB($ido),1);
240 &jnc (&label("c2ndloop"));
241
242 &mov (&DWP(256,$out),-1); # mark schedule as compressed
243
244&set_label("exit");
245 &xor ("eax","eax");
246 &mov (&DWP(-8,$out),"eax"); # key->x=0;
247 &mov (&DWP(-4,$out),"eax"); # key->y=0;
248&function_end("RC4_set_key");
249
250# const char *RC4_options(void);
251&function_begin_B("RC4_options");
252 &call (&label("pic_point"));
253&set_label("pic_point");
254 &blindpop("eax");
255 &lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax"));
256 &picmeup("edx","OPENSSL_ia32cap_P");
257 &bt (&DWP(0,"edx"),20);
258 &jnc (&label("skip"));
259 &add ("eax",12);
260 &set_label("skip");
261 &ret ();
262&set_label("opts",64);
263&asciz ("rc4(4x,int)");
264&asciz ("rc4(1x,char)");
265&asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>");
266&align (64);
267&function_end_B("RC4_options");
268
269&asm_finish();
230 270
diff --git a/src/lib/libcrypto/rc4/asm/rc4-amd64.pl b/src/lib/libcrypto/rc4/asm/rc4-amd64.pl
deleted file mode 100755
index 9e0da8af99..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-amd64.pl
+++ /dev/null
@@ -1,227 +0,0 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. Rights for redistribution and usage in source and binary
6# forms are granted according to the OpenSSL license.
7# ====================================================================
8#
9# 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
10# "hand-coded assembler"] doesn't stand for the whole improvement
11# coefficient. It turned out that eliminating RC4_CHAR from config
12# line results in ~40% improvement (yes, even for C implementation).
13# Presumably it has everything to do with AMD cache architecture and
14# RAW or whatever penalties. Once again! The module *requires* config
15# line *without* RC4_CHAR! As for coding "secret," I bet on partial
16# register arithmetics. For example instead of 'inc %r8; and $255,%r8'
17# I simply 'inc %r8b'. Even though optimization manual discourages
18# to operate on partial registers, it turned out to be the best bet.
19# At least for AMD... How IA32E would perform remains to be seen...
20
21# As was shown by Marc Bevand reordering of couple of load operations
22# results in even higher performance gain of 3.3x:-) At least on
23# Opteron... For reference, 1x in this case is RC4_CHAR C-code
24# compiled with gcc 3.3.2, which performs at ~54MBps per 1GHz clock.
25# Latter means that if you want to *estimate* what to expect from
26# *your* CPU, then multiply 54 by 3.3 and clock frequency in GHz.
27
28# Intel P4 EM64T core was found to run the AMD64 code really slow...
29# The only way to achieve comparable performance on P4 is to keep
30# RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
31# compose blended code, which would perform even within 30% marginal
32# on either AMD and Intel platforms, I implement both cases. See
33# rc4_skey.c for further details... This applies to 0.9.8 and later.
34# In 0.9.7 context RC4_CHAR codepath is never engaged and ~70 bytes
35# of code remain redundant.
36
37$output=shift;
38
39$win64a=1 if ($output =~ /win64a.[s|asm]/);
40
41open STDOUT,">$output" || die "can't open $output: $!";
42
43if (defined($win64a)) {
44 $dat="%rcx"; # arg1
45 $len="%rdx"; # arg2
46 $inp="%rsi"; # r8, arg3 moves here
47 $out="%rdi"; # r9, arg4 moves here
48} else {
49 $dat="%rdi"; # arg1
50 $len="%rsi"; # arg2
51 $inp="%rdx"; # arg3
52 $out="%rcx"; # arg4
53}
54
55$XX="%r10";
56$TX="%r8";
57$YY="%r11";
58$TY="%r9";
59
60sub PTR() {
61 my $ret=shift;
62 if (defined($win64a)) {
63 $ret =~ s/\[([\S]+)\+([\S]+)\]/[$2+$1]/g; # [%rN+%rM*4]->[%rM*4+%rN]
64 $ret =~ s/:([^\[]+)\[([^\]]+)\]/:[$2+$1]/g; # :off[ea]->:[ea+off]
65 } else {
66 $ret =~ s/[\+\*]/,/g; # [%rN+%rM*4]->[%rN,%rM,4]
67 $ret =~ s/\[([^\]]+)\]/($1)/g; # [%rN]->(%rN)
68 }
69 $ret;
70}
71
72$code=<<___ if (!defined($win64a));
73.text
74
75.globl RC4
76.type RC4,\@function
77.align 16
78RC4: or $len,$len
79 jne .Lentry
80 repret
81.Lentry:
82___
83$code=<<___ if (defined($win64a));
84_TEXT SEGMENT
85PUBLIC RC4
86ALIGN 16
87RC4 PROC
88 or $len,$len
89 jne .Lentry
90 repret
91.Lentry:
92 push %rdi
93 push %rsi
94 sub \$40,%rsp
95 mov %r8,$inp
96 mov %r9,$out
97___
98$code.=<<___;
99 add \$8,$dat
100 movl `&PTR("DWORD:-8[$dat]")`,$XX#d
101 movl `&PTR("DWORD:-4[$dat]")`,$YY#d
102 cmpl \$-1,`&PTR("DWORD:256[$dat]")`
103 je .LRC4_CHAR
104 test \$-8,$len
105 jz .Lloop1
106.align 16
107.Lloop8:
108 inc $XX#b
109 movl `&PTR("DWORD:[$dat+$XX*4]")`,$TX#d
110 add $TX#b,$YY#b
111 movl `&PTR("DWORD:[$dat+$YY*4]")`,$TY#d
112 movl $TX#d,`&PTR("DWORD:[$dat+$YY*4]")`
113 movl $TY#d,`&PTR("DWORD:[$dat+$XX*4]")`
114 add $TX#b,$TY#b
115 inc $XX#b
116 movl `&PTR("DWORD:[$dat+$XX*4]")`,$TX#d
117 movb `&PTR("BYTE:[$dat+$TY*4]")`,%al
118___
119for ($i=1;$i<=6;$i++) {
120$code.=<<___;
121 add $TX#b,$YY#b
122 ror \$8,%rax
123 movl `&PTR("DWORD:[$dat+$YY*4]")`,$TY#d
124 movl $TX#d,`&PTR("DWORD:[$dat+$YY*4]")`
125 movl $TY#d,`&PTR("DWORD:[$dat+$XX*4]")`
126 add $TX#b,$TY#b
127 inc $XX#b
128 movl `&PTR("DWORD:[$dat+$XX*4]")`,$TX#d
129 movb `&PTR("BYTE:[$dat+$TY*4]")`,%al
130___
131}
132$code.=<<___;
133 add $TX#b,$YY#b
134 ror \$8,%rax
135 movl `&PTR("DWORD:[$dat+$YY*4]")`,$TY#d
136 movl $TX#d,`&PTR("DWORD:[$dat+$YY*4]")`
137 movl $TY#d,`&PTR("DWORD:[$dat+$XX*4]")`
138 sub \$8,$len
139 add $TY#b,$TX#b
140 movb `&PTR("BYTE:[$dat+$TX*4]")`,%al
141 ror \$8,%rax
142 add \$8,$inp
143 add \$8,$out
144
145 xor `&PTR("QWORD:-8[$inp]")`,%rax
146 mov %rax,`&PTR("QWORD:-8[$out]")`
147
148 test \$-8,$len
149 jnz .Lloop8
150 cmp \$0,$len
151 jne .Lloop1
152.Lexit:
153 movl $XX#d,`&PTR("DWORD:-8[$dat]")`
154 movl $YY#d,`&PTR("DWORD:-4[$dat]")`
155___
156$code.=<<___ if (defined($win64a));
157 add \$40,%rsp
158 pop %rsi
159 pop %rdi
160___
161$code.=<<___;
162 repret
163.align 16
164.Lloop1:
165 movzb `&PTR("BYTE:[$inp]")`,%eax
166 inc $XX#b
167 movl `&PTR("DWORD:[$dat+$XX*4]")`,$TX#d
168 add $TX#b,$YY#b
169 movl `&PTR("DWORD:[$dat+$YY*4]")`,$TY#d
170 movl $TX#d,`&PTR("DWORD:[$dat+$YY*4]")`
171 movl $TY#d,`&PTR("DWORD:[$dat+$XX*4]")`
172 add $TY#b,$TX#b
173 movl `&PTR("DWORD:[$dat+$TX*4]")`,$TY#d
174 xor $TY,%rax
175 inc $inp
176 movb %al,`&PTR("BYTE:[$out]")`
177 inc $out
178 dec $len
179 jnz .Lloop1
180 jmp .Lexit
181
182.align 16
183.LRC4_CHAR:
184 inc $XX#b
185 movzb `&PTR("BYTE:[$dat+$XX]")`,$TX#d
186 add $TX#b,$YY#b
187 movzb `&PTR("BYTE:[$dat+$YY]")`,$TY#d
188 movb $TX#b,`&PTR("BYTE:[$dat+$YY]")`
189 movb $TY#b,`&PTR("BYTE:[$dat+$XX]")`
190 add $TX#b,$TY#b
191 movzb `&PTR("BYTE:[$dat+$TY]")`,$TY#d
192 xorb `&PTR("BYTE:[$inp]")`,$TY#b
193 movb $TY#b,`&PTR("BYTE:[$out]")`
194 inc $inp
195 inc $out
196 dec $len
197 jnz .LRC4_CHAR
198 jmp .Lexit
199___
200$code.=<<___ if (defined($win64a));
201RC4 ENDP
202_TEXT ENDS
203END
204___
205$code.=<<___ if (!defined($win64a));
206.size RC4,.-RC4
207___
208
209$code =~ s/#([bwd])/$1/gm;
210$code =~ s/\`([^\`]*)\`/eval $1/gem;
211
212if (defined($win64a)) {
213 $code =~ s/\.align/ALIGN/gm;
214 $code =~ s/[\$%]//gm;
215 $code =~ s/\.L/\$L/gm;
216 $code =~ s/([\w]+)([\s]+)([\S]+),([\S]+)/$1$2$4,$3/gm;
217 $code =~ s/([QD]*WORD|BYTE):/$1 PTR/gm;
218 $code =~ s/mov[bwlq]/mov/gm;
219 $code =~ s/movzb/movzx/gm;
220 $code =~ s/repret/DB\t0F3h,0C3h/gm;
221 $code =~ s/cmpl/cmp/gm;
222 $code =~ s/xorb/xor/gm;
223} else {
224 $code =~ s/([QD]*WORD|BYTE)://gm;
225 $code =~ s/repret/.byte\t0xF3,0xC3/gm;
226}
227print $code;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-ia64.S b/src/lib/libcrypto/rc4/asm/rc4-ia64.S
deleted file mode 100644
index 8210c47d04..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-ia64.S
+++ /dev/null
@@ -1,159 +0,0 @@
1// ====================================================================
2// Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
3// project.
4//
5// Rights for redistribution and usage in source and binary forms are
6// granted according to the OpenSSL license. Warranty of any kind is
7// disclaimed.
8// ====================================================================
9
10.ident "rc4-ia64.S, Version 2.0"
11.ident "IA-64 ISA artwork by Andy Polyakov <appro@fy.chalmers.se>"
12
13// What's wrong with compiler generated code? Because of the nature of
14// C language, compiler doesn't [dare to] reorder load and stores. But
15// being memory-bound, RC4 should benefit from reorder [on in-order-
16// execution core such as IA-64]. But what can we reorder? At the very
17// least we can safely reorder references to key schedule in respect
18// to input and output streams. Secondly, from the first [close] glance
19// it appeared that it's possible to pull up some references to
20// elements of the key schedule itself. Original rationale ["prior
21// loads are not safe only for "degenerated" key schedule, when some
22// elements equal to the same value"] was kind of sloppy. I should have
23// formulated as it really was: if we assume that pulling up reference
24// to key[x+1] is not safe, then it would mean that key schedule would
25// "degenerate," which is never the case. The problem is that this
26// holds true in respect to references to key[x], but not to key[y].
27// Legitimate "collisions" do occur within every 256^2 bytes window.
28// Fortunately there're enough free instruction slots to keep prior
29// reference to key[x+1], detect "collision" and compensate for it.
30// All this without sacrificing a single clock cycle:-) Throughput is
31// ~210MBps on 900MHz CPU, which is is >3x faster than gcc generated
32// code and +30% - if compared to HP-UX C. Unrolling loop below should
33// give >30% on top of that...
34
35.text
36.explicit
37
38#if defined(_HPUX_SOURCE) && !defined(_LP64)
39# define ADDP addp4
40#else
41# define ADDP add
42#endif
43
44#ifndef SZ
45#define SZ 4 // this is set to sizeof(RC4_INT)
46#endif
47// SZ==4 seems to be optimal. At least SZ==8 is not any faster, not for
48// assembler implementation, while SZ==1 code is ~30% slower.
49#if SZ==1 // RC4_INT is unsigned char
50# define LDKEY ld1
51# define STKEY st1
52# define OFF 0
53#elif SZ==4 // RC4_INT is unsigned int
54# define LDKEY ld4
55# define STKEY st4
56# define OFF 2
57#elif SZ==8 // RC4_INT is unsigned long
58# define LDKEY ld8
59# define STKEY st8
60# define OFF 3
61#endif
62
63out=r8; // [expanded] output pointer
64inp=r9; // [expanded] output pointer
65prsave=r10;
66key=r28; // [expanded] pointer to RC4_KEY
67ksch=r29; // (key->data+255)[&~(sizeof(key->data)-1)]
68xx=r30;
69yy=r31;
70
71// void RC4(RC4_KEY *key,size_t len,const void *inp,void *out);
72.global RC4#
73.proc RC4#
74.align 32
75.skip 16
76RC4:
77 .prologue
78 .save ar.pfs,r2
79{ .mii; alloc r2=ar.pfs,4,12,0,16
80 .save pr,prsave
81 mov prsave=pr
82 ADDP key=0,in0 };;
83{ .mib; cmp.eq p6,p0=0,in1 // len==0?
84 .save ar.lc,r3
85 mov r3=ar.lc
86(p6) br.ret.spnt.many b0 };; // emergency exit
87
88 .body
89 .rotr dat[4],key_x[4],tx[2],rnd[2],key_y[2],ty[1];
90
91{ .mib; LDKEY xx=[key],SZ // load key->x
92 add in1=-1,in1 // adjust len for loop counter
93 nop.b 0 }
94{ .mib; ADDP inp=0,in2
95 ADDP out=0,in3
96 brp.loop.imp .Ltop,.Lexit-16 };;
97{ .mmi; LDKEY yy=[key] // load key->y
98 add ksch=SZ,key
99 mov ar.lc=in1 }
100{ .mmi; mov key_y[1]=r0 // guarantee inequality
101 // in first iteration
102 add xx=1,xx
103 mov pr.rot=1<<16 };;
104{ .mii; nop.m 0
105 dep key_x[1]=xx,r0,OFF,8
106 mov ar.ec=3 };; // note that epilogue counter
107 // is off by 1. I compensate
108 // for this at exit...
109.Ltop:
110// The loop is scheduled for 4*(n+2) spin-rate on Itanium 2, which
111// theoretically gives asymptotic performance of clock frequency
112// divided by 4 bytes per seconds, or 400MBps on 1.6GHz CPU. This is
113// for sizeof(RC4_INT)==4. For smaller RC4_INT STKEY inadvertently
114// splits the last bundle and you end up with 5*n spin-rate:-(
115// Originally the loop was scheduled for 3*n and relied on key
116// schedule to be aligned at 256*sizeof(RC4_INT) boundary. But
117// *(out++)=dat, which maps to st1, had same effect [inadvertent
118// bundle split] and holded the loop back. Rescheduling for 4*n
119// made it possible to eliminate dependence on specific alignment
120// and allow OpenSSH keep "abusing" our API. Reaching for 3*n would
121// require unrolling, sticking to variable shift instruction for
122// collecting output [to avoid starvation for integer shifter] and
123// copying of key schedule to controlled place in stack [so that
124// deposit instruction can serve as substitute for whole
125// key->data+((x&255)<<log2(sizeof(key->data[0])))]...
126{ .mmi; (p19) st1 [out]=dat[3],1 // *(out++)=dat
127 (p16) add xx=1,xx // x++
128 (p18) dep rnd[1]=rnd[1],r0,OFF,8 } // ((tx+ty)&255)<<OFF
129{ .mmi; (p16) add key_x[1]=ksch,key_x[1] // &key[xx&255]
130 (p17) add key_y[1]=ksch,key_y[1] };; // &key[yy&255]
131{ .mmi; (p16) LDKEY tx[0]=[key_x[1]] // tx=key[xx]
132 (p17) LDKEY ty[0]=[key_y[1]] // ty=key[yy]
133 (p16) dep key_x[0]=xx,r0,OFF,8 } // (xx&255)<<OFF
134{ .mmi; (p18) add rnd[1]=ksch,rnd[1] // &key[(tx+ty)&255]
135 (p16) cmp.ne.unc p20,p21=key_x[1],key_y[1] };;
136{ .mmi; (p18) LDKEY rnd[1]=[rnd[1]] // rnd=key[(tx+ty)&255]
137 (p16) ld1 dat[0]=[inp],1 } // dat=*(inp++)
138.pred.rel "mutex",p20,p21
139{ .mmi; (p21) add yy=yy,tx[1] // (p16)
140 (p20) add yy=yy,tx[0] // (p16) y+=tx
141 (p21) mov tx[0]=tx[1] };; // (p16)
142{ .mmi; (p17) STKEY [key_y[1]]=tx[1] // key[yy]=tx
143 (p17) STKEY [key_x[2]]=ty[0] // key[xx]=ty
144 (p16) dep key_y[0]=yy,r0,OFF,8 } // &key[yy&255]
145{ .mmb; (p17) add rnd[0]=tx[1],ty[0] // tx+=ty
146 (p18) xor dat[2]=dat[2],rnd[1] // dat^=rnd
147 br.ctop.sptk .Ltop };;
148.Lexit:
149{ .mib; STKEY [key]=yy,-SZ // save key->y
150 mov pr=prsave,0x1ffff
151 nop.b 0 }
152{ .mib; st1 [out]=dat[3],1 // compensate for truncated
153 // epilogue counter
154 add xx=-1,xx
155 nop.b 0 };;
156{ .mib; STKEY [key]=xx // save key->x
157 mov ar.lc=r3
158 br.ret.sptk.many b0 };;
159.endp RC4#
diff --git a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
index 3a54623495..544386bf53 100755
--- a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
+++ b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
@@ -58,14 +58,18 @@
58# fit for Core2 and therefore the code was modified to skip cloop8 on 58# fit for Core2 and therefore the code was modified to skip cloop8 on
59# this CPU. 59# this CPU.
60 60
61$output=shift; 61$flavour = shift;
62$output = shift;
63if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
64
65$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
62 66
63$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 67$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
64( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 68( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
65( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 69( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
66die "can't locate x86_64-xlate.pl"; 70die "can't locate x86_64-xlate.pl";
67 71
68open STDOUT,"| $^X $xlate $output"; 72open STDOUT,"| $^X $xlate $flavour $output";
69 73
70$dat="%rdi"; # arg1 74$dat="%rdi"; # arg1
71$len="%rsi"; # arg2 75$len="%rsi"; # arg2
@@ -87,8 +91,10 @@ RC4: or $len,$len
87 jne .Lentry 91 jne .Lentry
88 ret 92 ret
89.Lentry: 93.Lentry:
94 push %rbx
90 push %r12 95 push %r12
91 push %r13 96 push %r13
97.Lprologue:
92 98
93 add \$8,$dat 99 add \$8,$dat
94 movl -8($dat),$XX[0]#d 100 movl -8($dat),$XX[0]#d
@@ -133,16 +139,8 @@ $code.=<<___;
133 jnz .Lloop8 139 jnz .Lloop8
134 cmp \$0,$len 140 cmp \$0,$len
135 jne .Lloop1 141 jne .Lloop1
136___ 142 jmp .Lexit
137$code.=<<___;
138.Lexit:
139 sub \$1,$XX[0]#b
140 movl $XX[0]#d,-8($dat)
141 movl $YY#d,-4($dat)
142 143
143 pop %r13
144 pop %r12
145 ret
146.align 16 144.align 16
147.Lloop1: 145.Lloop1:
148 add $TX[0]#b,$YY#b 146 add $TX[0]#b,$YY#b
@@ -167,9 +165,8 @@ $code.=<<___;
167 movzb ($dat,$XX[0]),$TX[0]#d 165 movzb ($dat,$XX[0]),$TX[0]#d
168 test \$-8,$len 166 test \$-8,$len
169 jz .Lcloop1 167 jz .Lcloop1
170 cmp \$0,260($dat) 168 cmpl \$0,260($dat)
171 jnz .Lcloop1 169 jnz .Lcloop1
172 push %rbx
173 jmp .Lcloop8 170 jmp .Lcloop8
174.align 16 171.align 16
175.Lcloop8: 172.Lcloop8:
@@ -224,7 +221,6 @@ $code.=<<___;
224 221
225 test \$-8,$len 222 test \$-8,$len
226 jnz .Lcloop8 223 jnz .Lcloop8
227 pop %rbx
228 cmp \$0,$len 224 cmp \$0,$len
229 jne .Lcloop1 225 jne .Lcloop1
230 jmp .Lexit 226 jmp .Lexit
@@ -249,6 +245,19 @@ $code.=<<___;
249 sub \$1,$len 245 sub \$1,$len
250 jnz .Lcloop1 246 jnz .Lcloop1
251 jmp .Lexit 247 jmp .Lexit
248
249.align 16
250.Lexit:
251 sub \$1,$XX[0]#b
252 movl $XX[0]#d,-8($dat)
253 movl $YY#d,-4($dat)
254
255 mov (%rsp),%r13
256 mov 8(%rsp),%r12
257 mov 16(%rsp),%rbx
258 add \$24,%rsp
259.Lepilogue:
260 ret
252.size RC4,.-RC4 261.size RC4,.-RC4
253___ 262___
254 263
@@ -269,6 +278,7 @@ RC4_set_key:
269 xor $ido,$ido 278 xor $ido,$ido
270 xor %r10,%r10 279 xor %r10,%r10
271 xor %r11,%r11 280 xor %r11,%r11
281
272 mov PIC_GOT(OPENSSL_ia32cap_P),$idx#d 282 mov PIC_GOT(OPENSSL_ia32cap_P),$idx#d
273 bt \$20,$idx#d 283 bt \$20,$idx#d
274 jnc .Lw1stloop 284 jnc .Lw1stloop
@@ -332,11 +342,10 @@ RC4_set_key:
332.size RC4_set_key,.-RC4_set_key 342.size RC4_set_key,.-RC4_set_key
333 343
334.globl RC4_options 344.globl RC4_options
335.type RC4_options,\@function,0 345.type RC4_options,\@abi-omnipotent
336.align 16 346.align 16
337RC4_options: 347RC4_options:
338 .picmeup %rax 348 lea .Lopts(%rip),%rax
339 lea .Lopts-.(%rax),%rax
340 mov PIC_GOT(OPENSSL_ia32cap_P),%edx 349 mov PIC_GOT(OPENSSL_ia32cap_P),%edx
341 bt \$20,%edx 350 bt \$20,%edx
342 jnc .Ldone 351 jnc .Ldone
@@ -356,9 +365,139 @@ RC4_options:
356.size RC4_options,.-RC4_options 365.size RC4_options,.-RC4_options
357___ 366___
358 367
359$code =~ s/#([bwd])/$1/gm; 368# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
369# CONTEXT *context,DISPATCHER_CONTEXT *disp)
370if ($win64) {
371$rec="%rcx";
372$frame="%rdx";
373$context="%r8";
374$disp="%r9";
375
376$code.=<<___;
377.extern __imp_RtlVirtualUnwind
378.type stream_se_handler,\@abi-omnipotent
379.align 16
380stream_se_handler:
381 push %rsi
382 push %rdi
383 push %rbx
384 push %rbp
385 push %r12
386 push %r13
387 push %r14
388 push %r15
389 pushfq
390 sub \$64,%rsp
391
392 mov 120($context),%rax # pull context->Rax
393 mov 248($context),%rbx # pull context->Rip
394
395 lea .Lprologue(%rip),%r10
396 cmp %r10,%rbx # context->Rip<prologue label
397 jb .Lin_prologue
398
399 mov 152($context),%rax # pull context->Rsp
400
401 lea .Lepilogue(%rip),%r10
402 cmp %r10,%rbx # context->Rip>=epilogue label
403 jae .Lin_prologue
404
405 lea 24(%rax),%rax
406
407 mov -8(%rax),%rbx
408 mov -16(%rax),%r12
409 mov -24(%rax),%r13
410 mov %rbx,144($context) # restore context->Rbx
411 mov %r12,216($context) # restore context->R12
412 mov %r13,224($context) # restore context->R13
413
414.Lin_prologue:
415 mov 8(%rax),%rdi
416 mov 16(%rax),%rsi
417 mov %rax,152($context) # restore context->Rsp
418 mov %rsi,168($context) # restore context->Rsi
419 mov %rdi,176($context) # restore context->Rdi
420
421 jmp .Lcommon_seh_exit
422.size stream_se_handler,.-stream_se_handler
423
424.type key_se_handler,\@abi-omnipotent
425.align 16
426key_se_handler:
427 push %rsi
428 push %rdi
429 push %rbx
430 push %rbp
431 push %r12
432 push %r13
433 push %r14
434 push %r15
435 pushfq
436 sub \$64,%rsp
437
438 mov 152($context),%rax # pull context->Rsp
439 mov 8(%rax),%rdi
440 mov 16(%rax),%rsi
441 mov %rsi,168($context) # restore context->Rsi
442 mov %rdi,176($context) # restore context->Rdi
360 443
361$code =~ s/RC4_set_key/private_RC4_set_key/g if ($ENV{FIPSCANLIB} ne ""); 444.Lcommon_seh_exit:
445
446 mov 40($disp),%rdi # disp->ContextRecord
447 mov $context,%rsi # context
448 mov \$154,%ecx # sizeof(CONTEXT)
449 .long 0xa548f3fc # cld; rep movsq
450
451 mov $disp,%rsi
452 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
453 mov 8(%rsi),%rdx # arg2, disp->ImageBase
454 mov 0(%rsi),%r8 # arg3, disp->ControlPc
455 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
456 mov 40(%rsi),%r10 # disp->ContextRecord
457 lea 56(%rsi),%r11 # &disp->HandlerData
458 lea 24(%rsi),%r12 # &disp->EstablisherFrame
459 mov %r10,32(%rsp) # arg5
460 mov %r11,40(%rsp) # arg6
461 mov %r12,48(%rsp) # arg7
462 mov %rcx,56(%rsp) # arg8, (NULL)
463 call *__imp_RtlVirtualUnwind(%rip)
464
465 mov \$1,%eax # ExceptionContinueSearch
466 add \$64,%rsp
467 popfq
468 pop %r15
469 pop %r14
470 pop %r13
471 pop %r12
472 pop %rbp
473 pop %rbx
474 pop %rdi
475 pop %rsi
476 ret
477.size key_se_handler,.-key_se_handler
478
479.section .pdata
480.align 4
481 .rva .LSEH_begin_RC4
482 .rva .LSEH_end_RC4
483 .rva .LSEH_info_RC4
484
485 .rva .LSEH_begin_RC4_set_key
486 .rva .LSEH_end_RC4_set_key
487 .rva .LSEH_info_RC4_set_key
488
489.section .xdata
490.align 8
491.LSEH_info_RC4:
492 .byte 9,0,0,0
493 .rva stream_se_handler
494.LSEH_info_RC4_set_key:
495 .byte 9,0,0,0
496 .rva key_se_handler
497___
498}
499
500$code =~ s/#([bwd])/$1/gm;
362 501
363print $code; 502print $code;
364 503
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-08-15 08:20:18 +0000