diff options
author | djm <> | 2012-10-13 21:23:50 +0000 |
---|---|---|
committer | djm <> | 2012-10-13 21:23:50 +0000 |
commit | 228cae30b117c2493f69ad3c195341cd6ec8d430 (patch) | |
tree | 29ff00b10d52c0978077c4fd83c33b065bade73e /src/lib/libcrypto/rc4 | |
parent | 731838c66b52c0ae5888333005b74115a620aa96 (diff) | |
download | openbsd-228cae30b117c2493f69ad3c195341cd6ec8d430.tar.gz openbsd-228cae30b117c2493f69ad3c195341cd6ec8d430.tar.bz2 openbsd-228cae30b117c2493f69ad3c195341cd6ec8d430.zip |
import OpenSSL-1.0.1c
Diffstat (limited to 'src/lib/libcrypto/rc4')
-rw-r--r-- | src/lib/libcrypto/rc4/asm/rc4-586.pl | 162 | ||||
-rw-r--r-- | src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl | 631 | ||||
-rw-r--r-- | src/lib/libcrypto/rc4/asm/rc4-parisc.pl | 313 | ||||
-rw-r--r-- | src/lib/libcrypto/rc4/asm/rc4-s390x.pl | 47 | ||||
-rwxr-xr-x | src/lib/libcrypto/rc4/asm/rc4-x86_64.pl | 290 | ||||
-rw-r--r-- | src/lib/libcrypto/rc4/rc4.h | 1 | ||||
-rw-r--r-- | src/lib/libcrypto/rc4/rc4_skey.c | 36 |
7 files changed, 1366 insertions, 114 deletions
diff --git a/src/lib/libcrypto/rc4/asm/rc4-586.pl b/src/lib/libcrypto/rc4/asm/rc4-586.pl index 38a44a70ef..5c9ac6ad28 100644 --- a/src/lib/libcrypto/rc4/asm/rc4-586.pl +++ b/src/lib/libcrypto/rc4/asm/rc4-586.pl | |||
@@ -28,6 +28,34 @@ | |||
28 | # | 28 | # |
29 | # <appro@fy.chalmers.se> | 29 | # <appro@fy.chalmers.se> |
30 | 30 | ||
31 | # May 2011 | ||
32 | # | ||
33 | # Optimize for Core2 and Westmere [and incidentally Opteron]. Current | ||
34 | # performance in cycles per processed byte (less is better) and | ||
35 | # improvement relative to previous version of this module is: | ||
36 | # | ||
37 | # Pentium 10.2 # original numbers | ||
38 | # Pentium III 7.8(*) | ||
39 | # Intel P4 7.5 | ||
40 | # | ||
41 | # Opteron 6.1/+20% # new MMX numbers | ||
42 | # Core2 5.3/+67%(**) | ||
43 | # Westmere 5.1/+94%(**) | ||
44 | # Sandy Bridge 5.0/+8% | ||
45 | # Atom 12.6/+6% | ||
46 | # | ||
47 | # (*) PIII can actually deliver 6.6 cycles per byte with MMX code, | ||
48 | # but this specific code performs poorly on Core2. And vice | ||
49 | # versa, below MMX/SSE code delivering 5.8/7.1 on Core2 performs | ||
50 | # poorly on PIII, at 8.0/14.5:-( As PIII is not a "hot" CPU | ||
51 | # [anymore], I chose to discard PIII-specific code path and opt | ||
52 | # for original IALU-only code, which is why MMX/SSE code path | ||
53 | # is guarded by SSE2 bit (see below), not MMX/SSE. | ||
54 | # (**) Performance vs. block size on Core2 and Westmere had a maximum | ||
55 | # at ... 64 bytes block size. And it was quite a maximum, 40-60% | ||
56 | # in comparison to largest 8KB block size. Above improvement | ||
57 | # coefficients are for the largest block size. | ||
58 | |||
31 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | 59 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
32 | push(@INC,"${dir}","${dir}../../perlasm"); | 60 | push(@INC,"${dir}","${dir}../../perlasm"); |
33 | require "x86asm.pl"; | 61 | require "x86asm.pl"; |
@@ -62,6 +90,68 @@ sub RC4_loop { | |||
62 | &$func ($out,&DWP(0,$dat,$ty,4)); | 90 | &$func ($out,&DWP(0,$dat,$ty,4)); |
63 | } | 91 | } |
64 | 92 | ||
93 | if ($alt=0) { | ||
94 | # >20% faster on Atom and Sandy Bridge[!], 8% faster on Opteron, | ||
95 | # but ~40% slower on Core2 and Westmere... Attempt to add movz | ||
96 | # brings down Opteron by 25%, Atom and Sandy Bridge by 15%, yet | ||
97 | # on Core2 with movz it's almost 20% slower than below alternative | ||
98 | # code... Yes, it's a total mess... | ||
99 | my @XX=($xx,$out); | ||
100 | $RC4_loop_mmx = sub { # SSE actually... | ||
101 | my $i=shift; | ||
102 | my $j=$i<=0?0:$i>>1; | ||
103 | my $mm=$i<=0?"mm0":"mm".($i&1); | ||
104 | |||
105 | &add (&LB($yy),&LB($tx)); | ||
106 | &lea (@XX[1],&DWP(1,@XX[0])); | ||
107 | &pxor ("mm2","mm0") if ($i==0); | ||
108 | &psllq ("mm1",8) if ($i==0); | ||
109 | &and (@XX[1],0xff); | ||
110 | &pxor ("mm0","mm0") if ($i<=0); | ||
111 | &mov ($ty,&DWP(0,$dat,$yy,4)); | ||
112 | &mov (&DWP(0,$dat,$yy,4),$tx); | ||
113 | &pxor ("mm1","mm2") if ($i==0); | ||
114 | &mov (&DWP(0,$dat,$XX[0],4),$ty); | ||
115 | &add (&LB($ty),&LB($tx)); | ||
116 | &movd (@XX[0],"mm7") if ($i==0); | ||
117 | &mov ($tx,&DWP(0,$dat,@XX[1],4)); | ||
118 | &pxor ("mm1","mm1") if ($i==1); | ||
119 | &movq ("mm2",&QWP(0,$inp)) if ($i==1); | ||
120 | &movq (&QWP(-8,(@XX[0],$inp)),"mm1") if ($i==0); | ||
121 | &pinsrw ($mm,&DWP(0,$dat,$ty,4),$j); | ||
122 | |||
123 | push (@XX,shift(@XX)) if ($i>=0); | ||
124 | } | ||
125 | } else { | ||
126 | # Using pinsrw here improves performane on Intel CPUs by 2-3%, but | ||
127 | # brings down AMD by 7%... | ||
128 | $RC4_loop_mmx = sub { | ||
129 | my $i=shift; | ||
130 | |||
131 | &add (&LB($yy),&LB($tx)); | ||
132 | &psllq ("mm1",8*(($i-1)&7)) if (abs($i)!=1); | ||
133 | &mov ($ty,&DWP(0,$dat,$yy,4)); | ||
134 | &mov (&DWP(0,$dat,$yy,4),$tx); | ||
135 | &mov (&DWP(0,$dat,$xx,4),$ty); | ||
136 | &inc ($xx); | ||
137 | &add ($ty,$tx); | ||
138 | &movz ($xx,&LB($xx)); # (*) | ||
139 | &movz ($ty,&LB($ty)); # (*) | ||
140 | &pxor ("mm2",$i==1?"mm0":"mm1") if ($i>=0); | ||
141 | &movq ("mm0",&QWP(0,$inp)) if ($i<=0); | ||
142 | &movq (&QWP(-8,($out,$inp)),"mm2") if ($i==0); | ||
143 | &mov ($tx,&DWP(0,$dat,$xx,4)); | ||
144 | &movd ($i>0?"mm1":"mm2",&DWP(0,$dat,$ty,4)); | ||
145 | |||
146 | # (*) This is the key to Core2 and Westmere performance. | ||
147 | # Whithout movz out-of-order execution logic confuses | ||
148 | # itself and fails to reorder loads and stores. Problem | ||
149 | # appears to be fixed in Sandy Bridge... | ||
150 | } | ||
151 | } | ||
152 | |||
153 | &external_label("OPENSSL_ia32cap_P"); | ||
154 | |||
65 | # void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out); | 155 | # void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out); |
66 | &function_begin("RC4"); | 156 | &function_begin("RC4"); |
67 | &mov ($dat,&wparam(0)); # load key schedule pointer | 157 | &mov ($dat,&wparam(0)); # load key schedule pointer |
@@ -94,11 +184,56 @@ sub RC4_loop { | |||
94 | &and ($ty,-4); # how many 4-byte chunks? | 184 | &and ($ty,-4); # how many 4-byte chunks? |
95 | &jz (&label("loop1")); | 185 | &jz (&label("loop1")); |
96 | 186 | ||
187 | &test ($ty,-8); | ||
188 | &mov (&wparam(3),$out); # $out as accumulator in these loops | ||
189 | &jz (&label("go4loop4")); | ||
190 | |||
191 | &picmeup($out,"OPENSSL_ia32cap_P"); | ||
192 | &bt (&DWP(0,$out),26); # check SSE2 bit [could have been MMX] | ||
193 | &jnc (&label("go4loop4")); | ||
194 | |||
195 | &mov ($out,&wparam(3)) if (!$alt); | ||
196 | &movd ("mm7",&wparam(3)) if ($alt); | ||
197 | &and ($ty,-8); | ||
198 | &lea ($ty,&DWP(-8,$inp,$ty)); | ||
199 | &mov (&DWP(-4,$dat),$ty); # save input+(len/8)*8-8 | ||
200 | |||
201 | &$RC4_loop_mmx(-1); | ||
202 | &jmp(&label("loop_mmx_enter")); | ||
203 | |||
204 | &set_label("loop_mmx",16); | ||
205 | &$RC4_loop_mmx(0); | ||
206 | &set_label("loop_mmx_enter"); | ||
207 | for ($i=1;$i<8;$i++) { &$RC4_loop_mmx($i); } | ||
208 | &mov ($ty,$yy); | ||
209 | &xor ($yy,$yy); # this is second key to Core2 | ||
210 | &mov (&LB($yy),&LB($ty)); # and Westmere performance... | ||
211 | &cmp ($inp,&DWP(-4,$dat)); | ||
212 | &lea ($inp,&DWP(8,$inp)); | ||
213 | &jb (&label("loop_mmx")); | ||
214 | |||
215 | if ($alt) { | ||
216 | &movd ($out,"mm7"); | ||
217 | &pxor ("mm2","mm0"); | ||
218 | &psllq ("mm1",8); | ||
219 | &pxor ("mm1","mm2"); | ||
220 | &movq (&QWP(-8,$out,$inp),"mm1"); | ||
221 | } else { | ||
222 | &psllq ("mm1",56); | ||
223 | &pxor ("mm2","mm1"); | ||
224 | &movq (&QWP(-8,$out,$inp),"mm2"); | ||
225 | } | ||
226 | &emms (); | ||
227 | |||
228 | &cmp ($inp,&wparam(1)); # compare to input+len | ||
229 | &je (&label("done")); | ||
230 | &jmp (&label("loop1")); | ||
231 | |||
232 | &set_label("go4loop4",16); | ||
97 | &lea ($ty,&DWP(-4,$inp,$ty)); | 233 | &lea ($ty,&DWP(-4,$inp,$ty)); |
98 | &mov (&wparam(2),$ty); # save input+(len/4)*4-4 | 234 | &mov (&wparam(2),$ty); # save input+(len/4)*4-4 |
99 | &mov (&wparam(3),$out); # $out as accumulator in this loop | ||
100 | 235 | ||
101 | &set_label("loop4",16); | 236 | &set_label("loop4"); |
102 | for ($i=0;$i<4;$i++) { RC4_loop($i); } | 237 | for ($i=0;$i<4;$i++) { RC4_loop($i); } |
103 | &ror ($out,8); | 238 | &ror ($out,8); |
104 | &xor ($out,&DWP(0,$inp)); | 239 | &xor ($out,&DWP(0,$inp)); |
@@ -151,7 +286,7 @@ sub RC4_loop { | |||
151 | 286 | ||
152 | &set_label("done"); | 287 | &set_label("done"); |
153 | &dec (&LB($xx)); | 288 | &dec (&LB($xx)); |
154 | &mov (&BP(-4,$dat),&LB($yy)); # save key->y | 289 | &mov (&DWP(-4,$dat),$yy); # save key->y |
155 | &mov (&BP(-8,$dat),&LB($xx)); # save key->x | 290 | &mov (&BP(-8,$dat),&LB($xx)); # save key->x |
156 | &set_label("abort"); | 291 | &set_label("abort"); |
157 | &function_end("RC4"); | 292 | &function_end("RC4"); |
@@ -164,10 +299,8 @@ $idi="ebp"; | |||
164 | $ido="ecx"; | 299 | $ido="ecx"; |
165 | $idx="edx"; | 300 | $idx="edx"; |
166 | 301 | ||
167 | &external_label("OPENSSL_ia32cap_P"); | ||
168 | |||
169 | # void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data); | 302 | # void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data); |
170 | &function_begin("RC4_set_key"); | 303 | &function_begin("private_RC4_set_key"); |
171 | &mov ($out,&wparam(0)); # load key | 304 | &mov ($out,&wparam(0)); # load key |
172 | &mov ($idi,&wparam(1)); # load len | 305 | &mov ($idi,&wparam(1)); # load len |
173 | &mov ($inp,&wparam(2)); # load data | 306 | &mov ($inp,&wparam(2)); # load data |
@@ -245,7 +378,7 @@ $idx="edx"; | |||
245 | &xor ("eax","eax"); | 378 | &xor ("eax","eax"); |
246 | &mov (&DWP(-8,$out),"eax"); # key->x=0; | 379 | &mov (&DWP(-8,$out),"eax"); # key->x=0; |
247 | &mov (&DWP(-4,$out),"eax"); # key->y=0; | 380 | &mov (&DWP(-4,$out),"eax"); # key->y=0; |
248 | &function_end("RC4_set_key"); | 381 | &function_end("private_RC4_set_key"); |
249 | 382 | ||
250 | # const char *RC4_options(void); | 383 | # const char *RC4_options(void); |
251 | &function_begin_B("RC4_options"); | 384 | &function_begin_B("RC4_options"); |
@@ -254,14 +387,21 @@ $idx="edx"; | |||
254 | &blindpop("eax"); | 387 | &blindpop("eax"); |
255 | &lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax")); | 388 | &lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax")); |
256 | &picmeup("edx","OPENSSL_ia32cap_P"); | 389 | &picmeup("edx","OPENSSL_ia32cap_P"); |
257 | &bt (&DWP(0,"edx"),20); | 390 | &mov ("edx",&DWP(0,"edx")); |
258 | &jnc (&label("skip")); | 391 | &bt ("edx",20); |
259 | &add ("eax",12); | 392 | &jc (&label("1xchar")); |
260 | &set_label("skip"); | 393 | &bt ("edx",26); |
394 | &jnc (&label("ret")); | ||
395 | &add ("eax",25); | ||
396 | &ret (); | ||
397 | &set_label("1xchar"); | ||
398 | &add ("eax",12); | ||
399 | &set_label("ret"); | ||
261 | &ret (); | 400 | &ret (); |
262 | &set_label("opts",64); | 401 | &set_label("opts",64); |
263 | &asciz ("rc4(4x,int)"); | 402 | &asciz ("rc4(4x,int)"); |
264 | &asciz ("rc4(1x,char)"); | 403 | &asciz ("rc4(1x,char)"); |
404 | &asciz ("rc4(8x,mmx)"); | ||
265 | &asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>"); | 405 | &asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>"); |
266 | &align (64); | 406 | &align (64); |
267 | &function_end_B("RC4_options"); | 407 | &function_end_B("RC4_options"); |
diff --git a/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl new file mode 100644 index 0000000000..7f684092d4 --- /dev/null +++ b/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl | |||
@@ -0,0 +1,631 @@ | |||
1 | #!/usr/bin/env perl | ||
2 | # | ||
3 | # ==================================================================== | ||
4 | # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL | ||
5 | # project. The module is, however, dual licensed under OpenSSL and | ||
6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
8 | # ==================================================================== | ||
9 | |||
10 | # June 2011 | ||
11 | # | ||
12 | # This is RC4+MD5 "stitch" implementation. The idea, as spelled in | ||
13 | # http://download.intel.com/design/intarch/papers/323686.pdf, is that | ||
14 | # since both algorithms exhibit instruction-level parallelism, ILP, | ||
15 | # below theoretical maximum, interleaving them would allow to utilize | ||
16 | # processor resources better and achieve better performance. RC4 | ||
17 | # instruction sequence is virtually identical to rc4-x86_64.pl, which | ||
18 | # is heavily based on submission by Maxim Perminov, Maxim Locktyukhin | ||
19 | # and Jim Guilford of Intel. MD5 is fresh implementation aiming to | ||
20 | # minimize register usage, which was used as "main thread" with RC4 | ||
21 | # weaved into it, one RC4 round per one MD5 round. In addition to the | ||
22 | # stiched subroutine the script can generate standalone replacement | ||
23 | # md5_block_asm_data_order and RC4. Below are performance numbers in | ||
24 | # cycles per processed byte, less is better, for these the standalone | ||
25 | # subroutines, sum of them, and stitched one: | ||
26 | # | ||
27 | # RC4 MD5 RC4+MD5 stitch gain | ||
28 | # Opteron 6.5(*) 5.4 11.9 7.0 +70%(*) | ||
29 | # Core2 6.5 5.8 12.3 7.7 +60% | ||
30 | # Westmere 4.3 5.2 9.5 7.0 +36% | ||
31 | # Sandy Bridge 4.2 5.5 9.7 6.8 +43% | ||
32 | # Atom 9.3 6.5 15.8 11.1 +42% | ||
33 | # | ||
34 | # (*) rc4-x86_64.pl delivers 5.3 on Opteron, so real improvement | ||
35 | # is +53%... | ||
36 | |||
37 | my ($rc4,$md5)=(1,1); # what to generate? | ||
38 | my $D="#" if (!$md5); # if set to "#", MD5 is stitched into RC4(), | ||
39 | # but its result is discarded. Idea here is | ||
40 | # to be able to use 'openssl speed rc4' for | ||
41 | # benchmarking the stitched subroutine... | ||
42 | |||
43 | my $flavour = shift; | ||
44 | my $output = shift; | ||
45 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | ||
46 | |||
47 | my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | ||
48 | |||
49 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; my $xlate; | ||
50 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | ||
51 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | ||
52 | die "can't locate x86_64-xlate.pl"; | ||
53 | |||
54 | open STDOUT,"| $^X $xlate $flavour $output"; | ||
55 | |||
56 | my ($dat,$in0,$out,$ctx,$inp,$len, $func,$nargs); | ||
57 | |||
58 | if ($rc4 && !$md5) { | ||
59 | ($dat,$len,$in0,$out) = ("%rdi","%rsi","%rdx","%rcx"); | ||
60 | $func="RC4"; $nargs=4; | ||
61 | } elsif ($md5 && !$rc4) { | ||
62 | ($ctx,$inp,$len) = ("%rdi","%rsi","%rdx"); | ||
63 | $func="md5_block_asm_data_order"; $nargs=3; | ||
64 | } else { | ||
65 | ($dat,$in0,$out,$ctx,$inp,$len) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); | ||
66 | $func="rc4_md5_enc"; $nargs=6; | ||
67 | # void rc4_md5_enc( | ||
68 | # RC4_KEY *key, # | ||
69 | # const void *in0, # RC4 input | ||
70 | # void *out, # RC4 output | ||
71 | # MD5_CTX *ctx, # | ||
72 | # const void *inp, # MD5 input | ||
73 | # size_t len); # number of 64-byte blocks | ||
74 | } | ||
75 | |||
76 | my @K=( 0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee, | ||
77 | 0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501, | ||
78 | 0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be, | ||
79 | 0x6b901122,0xfd987193,0xa679438e,0x49b40821, | ||
80 | |||
81 | 0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa, | ||
82 | 0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8, | ||
83 | 0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed, | ||
84 | 0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a, | ||
85 | |||
86 | 0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c, | ||
87 | 0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70, | ||
88 | 0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05, | ||
89 | 0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665, | ||
90 | |||
91 | 0xf4292244,0x432aff97,0xab9423a7,0xfc93a039, | ||
92 | 0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1, | ||
93 | 0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1, | ||
94 | 0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391 ); | ||
95 | |||
96 | my @V=("%r8d","%r9d","%r10d","%r11d"); # MD5 registers | ||
97 | my $tmp="%r12d"; | ||
98 | |||
99 | my @XX=("%rbp","%rsi"); # RC4 registers | ||
100 | my @TX=("%rax","%rbx"); | ||
101 | my $YY="%rcx"; | ||
102 | my $TY="%rdx"; | ||
103 | |||
104 | my $MOD=32; # 16, 32 or 64 | ||
105 | |||
106 | $code.=<<___; | ||
107 | .text | ||
108 | .align 16 | ||
109 | |||
110 | .globl $func | ||
111 | .type $func,\@function,$nargs | ||
112 | $func: | ||
113 | cmp \$0,$len | ||
114 | je .Labort | ||
115 | push %rbx | ||
116 | push %rbp | ||
117 | push %r12 | ||
118 | push %r13 | ||
119 | push %r14 | ||
120 | push %r15 | ||
121 | sub \$40,%rsp | ||
122 | .Lbody: | ||
123 | ___ | ||
124 | if ($rc4) { | ||
125 | $code.=<<___; | ||
126 | $D#md5# mov $ctx,%r11 # reassign arguments | ||
127 | mov $len,%r12 | ||
128 | mov $in0,%r13 | ||
129 | mov $out,%r14 | ||
130 | $D#md5# mov $inp,%r15 | ||
131 | ___ | ||
132 | $ctx="%r11" if ($md5); # reassign arguments | ||
133 | $len="%r12"; | ||
134 | $in0="%r13"; | ||
135 | $out="%r14"; | ||
136 | $inp="%r15" if ($md5); | ||
137 | $inp=$in0 if (!$md5); | ||
138 | $code.=<<___; | ||
139 | xor $XX[0],$XX[0] | ||
140 | xor $YY,$YY | ||
141 | |||
142 | lea 8($dat),$dat | ||
143 | mov -8($dat),$XX[0]#b | ||
144 | mov -4($dat),$YY#b | ||
145 | |||
146 | inc $XX[0]#b | ||
147 | sub $in0,$out | ||
148 | movl ($dat,$XX[0],4),$TX[0]#d | ||
149 | ___ | ||
150 | $code.=<<___ if (!$md5); | ||
151 | xor $TX[1],$TX[1] | ||
152 | test \$-128,$len | ||
153 | jz .Loop1 | ||
154 | sub $XX[0],$TX[1] | ||
155 | and \$`$MOD-1`,$TX[1] | ||
156 | jz .Loop${MOD}_is_hot | ||
157 | sub $TX[1],$len | ||
158 | .Loop${MOD}_warmup: | ||
159 | add $TX[0]#b,$YY#b | ||
160 | movl ($dat,$YY,4),$TY#d | ||
161 | movl $TX[0]#d,($dat,$YY,4) | ||
162 | movl $TY#d,($dat,$XX[0],4) | ||
163 | add $TY#b,$TX[0]#b | ||
164 | inc $XX[0]#b | ||
165 | movl ($dat,$TX[0],4),$TY#d | ||
166 | movl ($dat,$XX[0],4),$TX[0]#d | ||
167 | xorb ($in0),$TY#b | ||
168 | movb $TY#b,($out,$in0) | ||
169 | lea 1($in0),$in0 | ||
170 | dec $TX[1] | ||
171 | jnz .Loop${MOD}_warmup | ||
172 | |||
173 | mov $YY,$TX[1] | ||
174 | xor $YY,$YY | ||
175 | mov $TX[1]#b,$YY#b | ||
176 | |||
177 | .Loop${MOD}_is_hot: | ||
178 | mov $len,32(%rsp) # save original $len | ||
179 | shr \$6,$len # number of 64-byte blocks | ||
180 | ___ | ||
181 | if ($D && !$md5) { # stitch in dummy MD5 | ||
182 | $md5=1; | ||
183 | $ctx="%r11"; | ||
184 | $inp="%r15"; | ||
185 | $code.=<<___; | ||
186 | mov %rsp,$ctx | ||
187 | mov $in0,$inp | ||
188 | ___ | ||
189 | } | ||
190 | } | ||
191 | $code.=<<___; | ||
192 | #rc4# add $TX[0]#b,$YY#b | ||
193 | #rc4# lea ($dat,$XX[0],4),$XX[1] | ||
194 | shl \$6,$len | ||
195 | add $inp,$len # pointer to the end of input | ||
196 | mov $len,16(%rsp) | ||
197 | |||
198 | #md5# mov $ctx,24(%rsp) # save pointer to MD5_CTX | ||
199 | #md5# mov 0*4($ctx),$V[0] # load current hash value from MD5_CTX | ||
200 | #md5# mov 1*4($ctx),$V[1] | ||
201 | #md5# mov 2*4($ctx),$V[2] | ||
202 | #md5# mov 3*4($ctx),$V[3] | ||
203 | jmp .Loop | ||
204 | |||
205 | .align 16 | ||
206 | .Loop: | ||
207 | #md5# mov $V[0],0*4(%rsp) # put aside current hash value | ||
208 | #md5# mov $V[1],1*4(%rsp) | ||
209 | #md5# mov $V[2],2*4(%rsp) | ||
210 | #md5# mov $V[3],$tmp # forward reference | ||
211 | #md5# mov $V[3],3*4(%rsp) | ||
212 | ___ | ||
213 | |||
214 | sub R0 { | ||
215 | my ($i,$a,$b,$c,$d)=@_; | ||
216 | my @rot0=(7,12,17,22); | ||
217 | my $j=$i%16; | ||
218 | my $k=$i%$MOD; | ||
219 | my $xmm="%xmm".($j&1); | ||
220 | $code.=" movdqu ($in0),%xmm2\n" if ($rc4 && $j==15); | ||
221 | $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1); | ||
222 | $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1); | ||
223 | $code.=<<___; | ||
224 | #rc4# movl ($dat,$YY,4),$TY#d | ||
225 | #md5# xor $c,$tmp | ||
226 | #rc4# movl $TX[0]#d,($dat,$YY,4) | ||
227 | #md5# and $b,$tmp | ||
228 | #md5# add 4*`$j`($inp),$a | ||
229 | #rc4# add $TY#b,$TX[0]#b | ||
230 | #rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d | ||
231 | #md5# add \$$K[$i],$a | ||
232 | #md5# xor $d,$tmp | ||
233 | #rc4# movz $TX[0]#b,$TX[0]#d | ||
234 | #rc4# movl $TY#d,4*$k($XX[1]) | ||
235 | #md5# add $tmp,$a | ||
236 | #rc4# add $TX[1]#b,$YY#b | ||
237 | #md5# rol \$$rot0[$j%4],$a | ||
238 | #md5# mov `$j==15?"$b":"$c"`,$tmp # forward reference | ||
239 | #rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n | ||
240 | #md5# add $b,$a | ||
241 | ___ | ||
242 | $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1); | ||
243 | mov $YY,$XX[1] | ||
244 | xor $YY,$YY # keyword to partial register | ||
245 | mov $XX[1]#b,$YY#b | ||
246 | lea ($dat,$XX[0],4),$XX[1] | ||
247 | ___ | ||
248 | $code.=<<___ if ($rc4 && $j==15); | ||
249 | psllq \$8,%xmm1 | ||
250 | pxor %xmm0,%xmm2 | ||
251 | pxor %xmm1,%xmm2 | ||
252 | ___ | ||
253 | } | ||
254 | sub R1 { | ||
255 | my ($i,$a,$b,$c,$d)=@_; | ||
256 | my @rot1=(5,9,14,20); | ||
257 | my $j=$i%16; | ||
258 | my $k=$i%$MOD; | ||
259 | my $xmm="%xmm".($j&1); | ||
260 | $code.=" movdqu 16($in0),%xmm3\n" if ($rc4 && $j==15); | ||
261 | $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1); | ||
262 | $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1); | ||
263 | $code.=<<___; | ||
264 | #rc4# movl ($dat,$YY,4),$TY#d | ||
265 | #md5# xor $b,$tmp | ||
266 | #rc4# movl $TX[0]#d,($dat,$YY,4) | ||
267 | #md5# and $d,$tmp | ||
268 | #md5# add 4*`((1+5*$j)%16)`($inp),$a | ||
269 | #rc4# add $TY#b,$TX[0]#b | ||
270 | #rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d | ||
271 | #md5# add \$$K[$i],$a | ||
272 | #md5# xor $c,$tmp | ||
273 | #rc4# movz $TX[0]#b,$TX[0]#d | ||
274 | #rc4# movl $TY#d,4*$k($XX[1]) | ||
275 | #md5# add $tmp,$a | ||
276 | #rc4# add $TX[1]#b,$YY#b | ||
277 | #md5# rol \$$rot1[$j%4],$a | ||
278 | #md5# mov `$j==15?"$c":"$b"`,$tmp # forward reference | ||
279 | #rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n | ||
280 | #md5# add $b,$a | ||
281 | ___ | ||
282 | $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1); | ||
283 | mov $YY,$XX[1] | ||
284 | xor $YY,$YY # keyword to partial register | ||
285 | mov $XX[1]#b,$YY#b | ||
286 | lea ($dat,$XX[0],4),$XX[1] | ||
287 | ___ | ||
288 | $code.=<<___ if ($rc4 && $j==15); | ||
289 | psllq \$8,%xmm1 | ||
290 | pxor %xmm0,%xmm3 | ||
291 | pxor %xmm1,%xmm3 | ||
292 | ___ | ||
293 | } | ||
294 | sub R2 { | ||
295 | my ($i,$a,$b,$c,$d)=@_; | ||
296 | my @rot2=(4,11,16,23); | ||
297 | my $j=$i%16; | ||
298 | my $k=$i%$MOD; | ||
299 | my $xmm="%xmm".($j&1); | ||
300 | $code.=" movdqu 32($in0),%xmm4\n" if ($rc4 && $j==15); | ||
301 | $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1); | ||
302 | $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1); | ||
303 | $code.=<<___; | ||
304 | #rc4# movl ($dat,$YY,4),$TY#d | ||
305 | #md5# xor $c,$tmp | ||
306 | #rc4# movl $TX[0]#d,($dat,$YY,4) | ||
307 | #md5# xor $b,$tmp | ||
308 | #md5# add 4*`((5+3*$j)%16)`($inp),$a | ||
309 | #rc4# add $TY#b,$TX[0]#b | ||
310 | #rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d | ||
311 | #md5# add \$$K[$i],$a | ||
312 | #rc4# movz $TX[0]#b,$TX[0]#d | ||
313 | #md5# add $tmp,$a | ||
314 | #rc4# movl $TY#d,4*$k($XX[1]) | ||
315 | #rc4# add $TX[1]#b,$YY#b | ||
316 | #md5# rol \$$rot2[$j%4],$a | ||
317 | #md5# mov `$j==15?"\\\$-1":"$c"`,$tmp # forward reference | ||
318 | #rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n | ||
319 | #md5# add $b,$a | ||
320 | ___ | ||
321 | $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1); | ||
322 | mov $YY,$XX[1] | ||
323 | xor $YY,$YY # keyword to partial register | ||
324 | mov $XX[1]#b,$YY#b | ||
325 | lea ($dat,$XX[0],4),$XX[1] | ||
326 | ___ | ||
327 | $code.=<<___ if ($rc4 && $j==15); | ||
328 | psllq \$8,%xmm1 | ||
329 | pxor %xmm0,%xmm4 | ||
330 | pxor %xmm1,%xmm4 | ||
331 | ___ | ||
332 | } | ||
333 | sub R3 { | ||
334 | my ($i,$a,$b,$c,$d)=@_; | ||
335 | my @rot3=(6,10,15,21); | ||
336 | my $j=$i%16; | ||
337 | my $k=$i%$MOD; | ||
338 | my $xmm="%xmm".($j&1); | ||
339 | $code.=" movdqu 48($in0),%xmm5\n" if ($rc4 && $j==15); | ||
340 | $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1); | ||
341 | $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1); | ||
342 | $code.=<<___; | ||
343 | #rc4# movl ($dat,$YY,4),$TY#d | ||
344 | #md5# xor $d,$tmp | ||
345 | #rc4# movl $TX[0]#d,($dat,$YY,4) | ||
346 | #md5# or $b,$tmp | ||
347 | #md5# add 4*`((7*$j)%16)`($inp),$a | ||
348 | #rc4# add $TY#b,$TX[0]#b | ||
349 | #rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d | ||
350 | #md5# add \$$K[$i],$a | ||
351 | #rc4# movz $TX[0]#b,$TX[0]#d | ||
352 | #md5# xor $c,$tmp | ||
353 | #rc4# movl $TY#d,4*$k($XX[1]) | ||
354 | #md5# add $tmp,$a | ||
355 | #rc4# add $TX[1]#b,$YY#b | ||
356 | #md5# rol \$$rot3[$j%4],$a | ||
357 | #md5# mov \$-1,$tmp # forward reference | ||
358 | #rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n | ||
359 | #md5# add $b,$a | ||
360 | ___ | ||
361 | $code.=<<___ if ($rc4 && $j==15); | ||
362 | mov $XX[0],$XX[1] | ||
363 | xor $XX[0],$XX[0] # keyword to partial register | ||
364 | mov $XX[1]#b,$XX[0]#b | ||
365 | mov $YY,$XX[1] | ||
366 | xor $YY,$YY # keyword to partial register | ||
367 | mov $XX[1]#b,$YY#b | ||
368 | lea ($dat,$XX[0],4),$XX[1] | ||
369 | psllq \$8,%xmm1 | ||
370 | pxor %xmm0,%xmm5 | ||
371 | pxor %xmm1,%xmm5 | ||
372 | ___ | ||
373 | } | ||
374 | |||
375 | my $i=0; | ||
376 | for(;$i<16;$i++) { R0($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); } | ||
377 | for(;$i<32;$i++) { R1($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); } | ||
378 | for(;$i<48;$i++) { R2($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); } | ||
379 | for(;$i<64;$i++) { R3($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); } | ||
380 | |||
381 | $code.=<<___; | ||
382 | #md5# add 0*4(%rsp),$V[0] # accumulate hash value | ||
383 | #md5# add 1*4(%rsp),$V[1] | ||
384 | #md5# add 2*4(%rsp),$V[2] | ||
385 | #md5# add 3*4(%rsp),$V[3] | ||
386 | |||
387 | #rc4# movdqu %xmm2,($out,$in0) # write RC4 output | ||
388 | #rc4# movdqu %xmm3,16($out,$in0) | ||
389 | #rc4# movdqu %xmm4,32($out,$in0) | ||
390 | #rc4# movdqu %xmm5,48($out,$in0) | ||
391 | #md5# lea 64($inp),$inp | ||
392 | #rc4# lea 64($in0),$in0 | ||
393 | cmp 16(%rsp),$inp # are we done? | ||
394 | jb .Loop | ||
395 | |||
396 | #md5# mov 24(%rsp),$len # restore pointer to MD5_CTX | ||
397 | #rc4# sub $TX[0]#b,$YY#b # correct $YY | ||
398 | #md5# mov $V[0],0*4($len) # write MD5_CTX | ||
399 | #md5# mov $V[1],1*4($len) | ||
400 | #md5# mov $V[2],2*4($len) | ||
401 | #md5# mov $V[3],3*4($len) | ||
402 | ___ | ||
403 | $code.=<<___ if ($rc4 && (!$md5 || $D)); | ||
404 | mov 32(%rsp),$len # restore original $len | ||
405 | and \$63,$len # remaining bytes | ||
406 | jnz .Loop1 | ||
407 | jmp .Ldone | ||
408 | |||
409 | .align 16 | ||
410 | .Loop1: | ||
411 | add $TX[0]#b,$YY#b | ||
412 | movl ($dat,$YY,4),$TY#d | ||
413 | movl $TX[0]#d,($dat,$YY,4) | ||
414 | movl $TY#d,($dat,$XX[0],4) | ||
415 | add $TY#b,$TX[0]#b | ||
416 | inc $XX[0]#b | ||
417 | movl ($dat,$TX[0],4),$TY#d | ||
418 | movl ($dat,$XX[0],4),$TX[0]#d | ||
419 | xorb ($in0),$TY#b | ||
420 | movb $TY#b,($out,$in0) | ||
421 | lea 1($in0),$in0 | ||
422 | dec $len | ||
423 | jnz .Loop1 | ||
424 | |||
425 | .Ldone: | ||
426 | ___ | ||
427 | $code.=<<___; | ||
428 | #rc4# sub \$1,$XX[0]#b | ||
429 | #rc4# movl $XX[0]#d,-8($dat) | ||
430 | #rc4# movl $YY#d,-4($dat) | ||
431 | |||
432 | mov 40(%rsp),%r15 | ||
433 | mov 48(%rsp),%r14 | ||
434 | mov 56(%rsp),%r13 | ||
435 | mov 64(%rsp),%r12 | ||
436 | mov 72(%rsp),%rbp | ||
437 | mov 80(%rsp),%rbx | ||
438 | lea 88(%rsp),%rsp | ||
439 | .Lepilogue: | ||
440 | .Labort: | ||
441 | ret | ||
442 | .size $func,.-$func | ||
443 | ___ | ||
444 | |||
445 | if ($rc4 && $D) { # sole purpose of this section is to provide | ||
446 | # option to use the generated module as drop-in | ||
447 | # replacement for rc4-x86_64.pl for debugging | ||
448 | # and testing purposes... | ||
449 | my ($idx,$ido)=("%r8","%r9"); | ||
450 | my ($dat,$len,$inp)=("%rdi","%rsi","%rdx"); | ||
451 | |||
452 | $code.=<<___; | ||
453 | .globl RC4_set_key | ||
454 | .type RC4_set_key,\@function,3 | ||
455 | .align 16 | ||
456 | RC4_set_key: | ||
457 | lea 8($dat),$dat | ||
458 | lea ($inp,$len),$inp | ||
459 | neg $len | ||
460 | mov $len,%rcx | ||
461 | xor %eax,%eax | ||
462 | xor $ido,$ido | ||
463 | xor %r10,%r10 | ||
464 | xor %r11,%r11 | ||
465 | jmp .Lw1stloop | ||
466 | |||
467 | .align 16 | ||
468 | .Lw1stloop: | ||
469 | mov %eax,($dat,%rax,4) | ||
470 | add \$1,%al | ||
471 | jnc .Lw1stloop | ||
472 | |||
473 | xor $ido,$ido | ||
474 | xor $idx,$idx | ||
475 | .align 16 | ||
476 | .Lw2ndloop: | ||
477 | mov ($dat,$ido,4),%r10d | ||
478 | add ($inp,$len,1),$idx#b | ||
479 | add %r10b,$idx#b | ||
480 | add \$1,$len | ||
481 | mov ($dat,$idx,4),%r11d | ||
482 | cmovz %rcx,$len | ||
483 | mov %r10d,($dat,$idx,4) | ||
484 | mov %r11d,($dat,$ido,4) | ||
485 | add \$1,$ido#b | ||
486 | jnc .Lw2ndloop | ||
487 | |||
488 | xor %eax,%eax | ||
489 | mov %eax,-8($dat) | ||
490 | mov %eax,-4($dat) | ||
491 | ret | ||
492 | .size RC4_set_key,.-RC4_set_key | ||
493 | |||
494 | .globl RC4_options | ||
495 | .type RC4_options,\@abi-omnipotent | ||
496 | .align 16 | ||
497 | RC4_options: | ||
498 | lea .Lopts(%rip),%rax | ||
499 | ret | ||
500 | .align 64 | ||
501 | .Lopts: | ||
502 | .asciz "rc4(64x,int)" | ||
503 | .align 64 | ||
504 | .size RC4_options,.-RC4_options | ||
505 | ___ | ||
506 | } | ||
507 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | ||
508 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | ||
509 | if ($win64) { | ||
510 | my $rec="%rcx"; | ||
511 | my $frame="%rdx"; | ||
512 | my $context="%r8"; | ||
513 | my $disp="%r9"; | ||
514 | |||
515 | $code.=<<___; | ||
516 | .extern __imp_RtlVirtualUnwind | ||
517 | .type se_handler,\@abi-omnipotent | ||
518 | .align 16 | ||
519 | se_handler: | ||
520 | push %rsi | ||
521 | push %rdi | ||
522 | push %rbx | ||
523 | push %rbp | ||
524 | push %r12 | ||
525 | push %r13 | ||
526 | push %r14 | ||
527 | push %r15 | ||
528 | pushfq | ||
529 | sub \$64,%rsp | ||
530 | |||
531 | mov 120($context),%rax # pull context->Rax | ||
532 | mov 248($context),%rbx # pull context->Rip | ||
533 | |||
534 | lea .Lbody(%rip),%r10 | ||
535 | cmp %r10,%rbx # context->Rip<.Lbody | ||
536 | jb .Lin_prologue | ||
537 | |||
538 | mov 152($context),%rax # pull context->Rsp | ||
539 | |||
540 | lea .Lepilogue(%rip),%r10 | ||
541 | cmp %r10,%rbx # context->Rip>=.Lepilogue | ||
542 | jae .Lin_prologue | ||
543 | |||
544 | mov 40(%rax),%r15 | ||
545 | mov 48(%rax),%r14 | ||
546 | mov 56(%rax),%r13 | ||
547 | mov 64(%rax),%r12 | ||
548 | mov 72(%rax),%rbp | ||
549 | mov 80(%rax),%rbx | ||
550 | lea 88(%rax),%rax | ||
551 | |||
552 | mov %rbx,144($context) # restore context->Rbx | ||
553 | mov %rbp,160($context) # restore context->Rbp | ||
554 | mov %r12,216($context) # restore context->R12 | ||
555 | mov %r13,224($context) # restore context->R12 | ||
556 | mov %r14,232($context) # restore context->R14 | ||
557 | mov %r15,240($context) # restore context->R15 | ||
558 | |||
559 | .Lin_prologue: | ||
560 | mov 8(%rax),%rdi | ||
561 | mov 16(%rax),%rsi | ||
562 | mov %rax,152($context) # restore context->Rsp | ||
563 | mov %rsi,168($context) # restore context->Rsi | ||
564 | mov %rdi,176($context) # restore context->Rdi | ||
565 | |||
566 | mov 40($disp),%rdi # disp->ContextRecord | ||
567 | mov $context,%rsi # context | ||
568 | mov \$154,%ecx # sizeof(CONTEXT) | ||
569 | .long 0xa548f3fc # cld; rep movsq | ||
570 | |||
571 | mov $disp,%rsi | ||
572 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | ||
573 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | ||
574 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | ||
575 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | ||
576 | mov 40(%rsi),%r10 # disp->ContextRecord | ||
577 | lea 56(%rsi),%r11 # &disp->HandlerData | ||
578 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | ||
579 | mov %r10,32(%rsp) # arg5 | ||
580 | mov %r11,40(%rsp) # arg6 | ||
581 | mov %r12,48(%rsp) # arg7 | ||
582 | mov %rcx,56(%rsp) # arg8, (NULL) | ||
583 | call *__imp_RtlVirtualUnwind(%rip) | ||
584 | |||
585 | mov \$1,%eax # ExceptionContinueSearch | ||
586 | add \$64,%rsp | ||
587 | popfq | ||
588 | pop %r15 | ||
589 | pop %r14 | ||
590 | pop %r13 | ||
591 | pop %r12 | ||
592 | pop %rbp | ||
593 | pop %rbx | ||
594 | pop %rdi | ||
595 | pop %rsi | ||
596 | ret | ||
597 | .size se_handler,.-se_handler | ||
598 | |||
599 | .section .pdata | ||
600 | .align 4 | ||
601 | .rva .LSEH_begin_$func | ||
602 | .rva .LSEH_end_$func | ||
603 | .rva .LSEH_info_$func | ||
604 | |||
605 | .section .xdata | ||
606 | .align 8 | ||
607 | .LSEH_info_$func: | ||
608 | .byte 9,0,0,0 | ||
609 | .rva se_handler | ||
610 | ___ | ||
611 | } | ||
612 | |||
613 | sub reg_part { | ||
614 | my ($reg,$conv)=@_; | ||
615 | if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } | ||
616 | elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } | ||
617 | elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } | ||
618 | elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } | ||
619 | return $reg; | ||
620 | } | ||
621 | |||
622 | $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; | ||
623 | $code =~ s/\`([^\`]*)\`/eval $1/gem; | ||
624 | $code =~ s/pinsrw\s+\$0,/movd /gm; | ||
625 | |||
626 | $code =~ s/#md5#//gm if ($md5); | ||
627 | $code =~ s/#rc4#//gm if ($rc4); | ||
628 | |||
629 | print $code; | ||
630 | |||
631 | close STDOUT; | ||
diff --git a/src/lib/libcrypto/rc4/asm/rc4-parisc.pl b/src/lib/libcrypto/rc4/asm/rc4-parisc.pl new file mode 100644 index 0000000000..9165067080 --- /dev/null +++ b/src/lib/libcrypto/rc4/asm/rc4-parisc.pl | |||
@@ -0,0 +1,313 @@ | |||
1 | #!/usr/bin/env perl | ||
2 | |||
3 | # ==================================================================== | ||
4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
5 | # project. The module is, however, dual licensed under OpenSSL and | ||
6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
8 | # ==================================================================== | ||
9 | |||
10 | # RC4 for PA-RISC. | ||
11 | |||
12 | # June 2009. | ||
13 | # | ||
14 | # Performance is 33% better than gcc 3.2 generated code on PA-7100LC. | ||
15 | # For reference, [4x] unrolled loop is >40% faster than folded one. | ||
16 | # It's possible to unroll loop 8 times on PA-RISC 2.0, but improvement | ||
17 | # is believed to be not sufficient to justify the effort... | ||
18 | # | ||
19 | # Special thanks to polarhome.com for providing HP-UX account. | ||
20 | |||
21 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
22 | |||
23 | $flavour = shift; | ||
24 | $output = shift; | ||
25 | open STDOUT,">$output"; | ||
26 | |||
27 | if ($flavour =~ /64/) { | ||
28 | $LEVEL ="2.0W"; | ||
29 | $SIZE_T =8; | ||
30 | $FRAME_MARKER =80; | ||
31 | $SAVED_RP =16; | ||
32 | $PUSH ="std"; | ||
33 | $PUSHMA ="std,ma"; | ||
34 | $POP ="ldd"; | ||
35 | $POPMB ="ldd,mb"; | ||
36 | } else { | ||
37 | $LEVEL ="1.0"; | ||
38 | $SIZE_T =4; | ||
39 | $FRAME_MARKER =48; | ||
40 | $SAVED_RP =20; | ||
41 | $PUSH ="stw"; | ||
42 | $PUSHMA ="stwm"; | ||
43 | $POP ="ldw"; | ||
44 | $POPMB ="ldwm"; | ||
45 | } | ||
46 | |||
47 | $FRAME=4*$SIZE_T+$FRAME_MARKER; # 4 saved regs + frame marker | ||
48 | # [+ argument transfer] | ||
49 | $SZ=1; # defaults to RC4_CHAR | ||
50 | if (open CONF,"<${dir}../../opensslconf.h") { | ||
51 | while(<CONF>) { | ||
52 | if (m/#\s*define\s+RC4_INT\s+(.*)/) { | ||
53 | $SZ = ($1=~/char$/) ? 1 : 4; | ||
54 | last; | ||
55 | } | ||
56 | } | ||
57 | close CONF; | ||
58 | } | ||
59 | |||
60 | if ($SZ==1) { # RC4_CHAR | ||
61 | $LD="ldb"; | ||
62 | $LDX="ldbx"; | ||
63 | $MKX="addl"; | ||
64 | $ST="stb"; | ||
65 | } else { # RC4_INT (~5% faster than RC4_CHAR on PA-7100LC) | ||
66 | $LD="ldw"; | ||
67 | $LDX="ldwx,s"; | ||
68 | $MKX="sh2addl"; | ||
69 | $ST="stw"; | ||
70 | } | ||
71 | |||
72 | $key="%r26"; | ||
73 | $len="%r25"; | ||
74 | $inp="%r24"; | ||
75 | $out="%r23"; | ||
76 | |||
77 | @XX=("%r19","%r20"); | ||
78 | @TX=("%r21","%r22"); | ||
79 | $YY="%r28"; | ||
80 | $TY="%r29"; | ||
81 | |||
82 | $acc="%r1"; | ||
83 | $ix="%r2"; | ||
84 | $iy="%r3"; | ||
85 | $dat0="%r4"; | ||
86 | $dat1="%r5"; | ||
87 | $rem="%r6"; | ||
88 | $mask="%r31"; | ||
89 | |||
90 | sub unrolledloopbody { | ||
91 | for ($i=0;$i<4;$i++) { | ||
92 | $code.=<<___; | ||
93 | ldo 1($XX[0]),$XX[1] | ||
94 | `sprintf("$LDX %$TY(%$key),%$dat1") if ($i>0)` | ||
95 | and $mask,$XX[1],$XX[1] | ||
96 | $LDX $YY($key),$TY | ||
97 | $MKX $YY,$key,$ix | ||
98 | $LDX $XX[1]($key),$TX[1] | ||
99 | $MKX $XX[0],$key,$iy | ||
100 | $ST $TX[0],0($ix) | ||
101 | comclr,<> $XX[1],$YY,%r0 ; conditional | ||
102 | copy $TX[0],$TX[1] ; move | ||
103 | `sprintf("%sdep %$dat1,%d,8,%$acc",$i==1?"z":"",8*($i-1)+7) if ($i>0)` | ||
104 | $ST $TY,0($iy) | ||
105 | addl $TX[0],$TY,$TY | ||
106 | addl $TX[1],$YY,$YY | ||
107 | and $mask,$TY,$TY | ||
108 | and $mask,$YY,$YY | ||
109 | ___ | ||
110 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers | ||
111 | } } | ||
112 | |||
113 | sub foldedloop { | ||
114 | my ($label,$count)=@_; | ||
115 | $code.=<<___; | ||
116 | $label | ||
117 | $MKX $YY,$key,$iy | ||
118 | $LDX $YY($key),$TY | ||
119 | $MKX $XX[0],$key,$ix | ||
120 | $ST $TX[0],0($iy) | ||
121 | ldo 1($XX[0]),$XX[0] | ||
122 | $ST $TY,0($ix) | ||
123 | addl $TX[0],$TY,$TY | ||
124 | ldbx $inp($out),$dat1 | ||
125 | and $mask,$TY,$TY | ||
126 | and $mask,$XX[0],$XX[0] | ||
127 | $LDX $TY($key),$acc | ||
128 | $LDX $XX[0]($key),$TX[0] | ||
129 | ldo 1($out),$out | ||
130 | xor $dat1,$acc,$acc | ||
131 | addl $TX[0],$YY,$YY | ||
132 | stb $acc,-1($out) | ||
133 | addib,<> -1,$count,$label ; $count is always small | ||
134 | and $mask,$YY,$YY | ||
135 | ___ | ||
136 | } | ||
137 | |||
138 | $code=<<___; | ||
139 | .LEVEL $LEVEL | ||
140 | .SPACE \$TEXT\$ | ||
141 | .SUBSPA \$CODE\$,QUAD=0,ALIGN=8,ACCESS=0x2C,CODE_ONLY | ||
142 | |||
143 | .EXPORT RC4,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR,ARGW3=GR | ||
144 | RC4 | ||
145 | .PROC | ||
146 | .CALLINFO FRAME=`$FRAME-4*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=6 | ||
147 | .ENTRY | ||
148 | $PUSH %r2,-$SAVED_RP(%sp) ; standard prologue | ||
149 | $PUSHMA %r3,$FRAME(%sp) | ||
150 | $PUSH %r4,`-$FRAME+1*$SIZE_T`(%sp) | ||
151 | $PUSH %r5,`-$FRAME+2*$SIZE_T`(%sp) | ||
152 | $PUSH %r6,`-$FRAME+3*$SIZE_T`(%sp) | ||
153 | |||
154 | cmpib,*= 0,$len,L\$abort | ||
155 | sub $inp,$out,$inp ; distance between $inp and $out | ||
156 | |||
157 | $LD `0*$SZ`($key),$XX[0] | ||
158 | $LD `1*$SZ`($key),$YY | ||
159 | ldo `2*$SZ`($key),$key | ||
160 | |||
161 | ldi 0xff,$mask | ||
162 | ldi 3,$dat0 | ||
163 | |||
164 | ldo 1($XX[0]),$XX[0] ; warm up loop | ||
165 | and $mask,$XX[0],$XX[0] | ||
166 | $LDX $XX[0]($key),$TX[0] | ||
167 | addl $TX[0],$YY,$YY | ||
168 | cmpib,*>>= 6,$len,L\$oop1 ; is $len large enough to bother? | ||
169 | and $mask,$YY,$YY | ||
170 | |||
171 | and,<> $out,$dat0,$rem ; is $out aligned? | ||
172 | b L\$alignedout | ||
173 | subi 4,$rem,$rem | ||
174 | sub $len,$rem,$len | ||
175 | ___ | ||
176 | &foldedloop("L\$alignout",$rem); # process till $out is aligned | ||
177 | |||
178 | $code.=<<___; | ||
179 | L\$alignedout ; $len is at least 4 here | ||
180 | and,<> $inp,$dat0,$acc ; is $inp aligned? | ||
181 | b L\$oop4 | ||
182 | sub $inp,$acc,$rem ; align $inp | ||
183 | |||
184 | sh3addl $acc,%r0,$acc | ||
185 | subi 32,$acc,$acc | ||
186 | mtctl $acc,%cr11 ; load %sar with vshd align factor | ||
187 | ldwx $rem($out),$dat0 | ||
188 | ldo 4($rem),$rem | ||
189 | L\$oop4misalignedinp | ||
190 | ___ | ||
191 | &unrolledloopbody(); | ||
192 | $code.=<<___; | ||
193 | $LDX $TY($key),$ix | ||
194 | ldwx $rem($out),$dat1 | ||
195 | ldo -4($len),$len | ||
196 | or $ix,$acc,$acc ; last piece, no need to dep | ||
197 | vshd $dat0,$dat1,$iy ; align data | ||
198 | copy $dat1,$dat0 | ||
199 | xor $iy,$acc,$acc | ||
200 | stw $acc,0($out) | ||
201 | cmpib,*<< 3,$len,L\$oop4misalignedinp | ||
202 | ldo 4($out),$out | ||
203 | cmpib,*= 0,$len,L\$done | ||
204 | nop | ||
205 | b L\$oop1 | ||
206 | nop | ||
207 | |||
208 | .ALIGN 8 | ||
209 | L\$oop4 | ||
210 | ___ | ||
211 | &unrolledloopbody(); | ||
212 | $code.=<<___; | ||
213 | $LDX $TY($key),$ix | ||
214 | ldwx $inp($out),$dat0 | ||
215 | ldo -4($len),$len | ||
216 | or $ix,$acc,$acc ; last piece, no need to dep | ||
217 | xor $dat0,$acc,$acc | ||
218 | stw $acc,0($out) | ||
219 | cmpib,*<< 3,$len,L\$oop4 | ||
220 | ldo 4($out),$out | ||
221 | cmpib,*= 0,$len,L\$done | ||
222 | nop | ||
223 | ___ | ||
224 | &foldedloop("L\$oop1",$len); | ||
225 | $code.=<<___; | ||
226 | L\$done | ||
227 | $POP `-$FRAME-$SAVED_RP`(%sp),%r2 | ||
228 | ldo -1($XX[0]),$XX[0] ; chill out loop | ||
229 | sub $YY,$TX[0],$YY | ||
230 | and $mask,$XX[0],$XX[0] | ||
231 | and $mask,$YY,$YY | ||
232 | $ST $XX[0],`-2*$SZ`($key) | ||
233 | $ST $YY,`-1*$SZ`($key) | ||
234 | $POP `-$FRAME+1*$SIZE_T`(%sp),%r4 | ||
235 | $POP `-$FRAME+2*$SIZE_T`(%sp),%r5 | ||
236 | $POP `-$FRAME+3*$SIZE_T`(%sp),%r6 | ||
237 | L\$abort | ||
238 | bv (%r2) | ||
239 | .EXIT | ||
240 | $POPMB -$FRAME(%sp),%r3 | ||
241 | .PROCEND | ||
242 | ___ | ||
243 | |||
244 | $code.=<<___; | ||
245 | |||
246 | .EXPORT private_RC4_set_key,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR | ||
247 | .ALIGN 8 | ||
248 | private_RC4_set_key | ||
249 | .PROC | ||
250 | .CALLINFO NO_CALLS | ||
251 | .ENTRY | ||
252 | $ST %r0,`0*$SZ`($key) | ||
253 | $ST %r0,`1*$SZ`($key) | ||
254 | ldo `2*$SZ`($key),$key | ||
255 | copy %r0,@XX[0] | ||
256 | L\$1st | ||
257 | $ST @XX[0],0($key) | ||
258 | ldo 1(@XX[0]),@XX[0] | ||
259 | bb,>= @XX[0],`31-8`,L\$1st ; @XX[0]<256 | ||
260 | ldo $SZ($key),$key | ||
261 | |||
262 | ldo `-256*$SZ`($key),$key ; rewind $key | ||
263 | addl $len,$inp,$inp ; $inp to point at the end | ||
264 | sub %r0,$len,%r23 ; inverse index | ||
265 | copy %r0,@XX[0] | ||
266 | copy %r0,@XX[1] | ||
267 | ldi 0xff,$mask | ||
268 | |||
269 | L\$2nd | ||
270 | $LDX @XX[0]($key),@TX[0] | ||
271 | ldbx %r23($inp),@TX[1] | ||
272 | addi,nuv 1,%r23,%r23 ; increment and conditional | ||
273 | sub %r0,$len,%r23 ; inverse index | ||
274 | addl @TX[0],@XX[1],@XX[1] | ||
275 | addl @TX[1],@XX[1],@XX[1] | ||
276 | and $mask,@XX[1],@XX[1] | ||
277 | $MKX @XX[0],$key,$TY | ||
278 | $LDX @XX[1]($key),@TX[1] | ||
279 | $MKX @XX[1],$key,$YY | ||
280 | ldo 1(@XX[0]),@XX[0] | ||
281 | $ST @TX[0],0($YY) | ||
282 | bb,>= @XX[0],`31-8`,L\$2nd ; @XX[0]<256 | ||
283 | $ST @TX[1],0($TY) | ||
284 | |||
285 | bv,n (%r2) | ||
286 | .EXIT | ||
287 | nop | ||
288 | .PROCEND | ||
289 | |||
290 | .EXPORT RC4_options,ENTRY | ||
291 | .ALIGN 8 | ||
292 | RC4_options | ||
293 | .PROC | ||
294 | .CALLINFO NO_CALLS | ||
295 | .ENTRY | ||
296 | blr %r0,%r28 | ||
297 | ldi 3,%r1 | ||
298 | L\$pic | ||
299 | andcm %r28,%r1,%r28 | ||
300 | bv (%r2) | ||
301 | .EXIT | ||
302 | ldo L\$opts-L\$pic(%r28),%r28 | ||
303 | .PROCEND | ||
304 | .ALIGN 8 | ||
305 | L\$opts | ||
306 | .STRINGZ "rc4(4x,`$SZ==1?"char":"int"`)" | ||
307 | .STRINGZ "RC4 for PA-RISC, CRYPTOGAMS by <appro\@openssl.org>" | ||
308 | ___ | ||
309 | $code =~ s/\`([^\`]*)\`/eval $1/gem; | ||
310 | $code =~ s/cmpib,\*/comib,/gm if ($SIZE_T==4); | ||
311 | |||
312 | print $code; | ||
313 | close STDOUT; | ||
diff --git a/src/lib/libcrypto/rc4/asm/rc4-s390x.pl b/src/lib/libcrypto/rc4/asm/rc4-s390x.pl index 96681fa05e..7528ece13c 100644 --- a/src/lib/libcrypto/rc4/asm/rc4-s390x.pl +++ b/src/lib/libcrypto/rc4/asm/rc4-s390x.pl | |||
@@ -13,6 +13,29 @@ | |||
13 | # "cluster" Address Generation Interlocks, so that one pipeline stall | 13 | # "cluster" Address Generation Interlocks, so that one pipeline stall |
14 | # resolves several dependencies. | 14 | # resolves several dependencies. |
15 | 15 | ||
16 | # November 2010. | ||
17 | # | ||
18 | # Adapt for -m31 build. If kernel supports what's called "highgprs" | ||
19 | # feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit | ||
20 | # instructions and achieve "64-bit" performance even in 31-bit legacy | ||
21 | # application context. The feature is not specific to any particular | ||
22 | # processor, as long as it's "z-CPU". Latter implies that the code | ||
23 | # remains z/Architecture specific. On z990 it was measured to perform | ||
24 | # 50% better than code generated by gcc 4.3. | ||
25 | |||
26 | $flavour = shift; | ||
27 | |||
28 | if ($flavour =~ /3[12]/) { | ||
29 | $SIZE_T=4; | ||
30 | $g=""; | ||
31 | } else { | ||
32 | $SIZE_T=8; | ||
33 | $g="g"; | ||
34 | } | ||
35 | |||
36 | while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} | ||
37 | open STDOUT,">$output"; | ||
38 | |||
16 | $rp="%r14"; | 39 | $rp="%r14"; |
17 | $sp="%r15"; | 40 | $sp="%r15"; |
18 | $code=<<___; | 41 | $code=<<___; |
@@ -39,7 +62,12 @@ $code.=<<___; | |||
39 | .type RC4,\@function | 62 | .type RC4,\@function |
40 | .align 64 | 63 | .align 64 |
41 | RC4: | 64 | RC4: |
42 | stmg %r6,%r11,48($sp) | 65 | stm${g} %r6,%r11,6*$SIZE_T($sp) |
66 | ___ | ||
67 | $code.=<<___ if ($flavour =~ /3[12]/); | ||
68 | llgfr $len,$len | ||
69 | ___ | ||
70 | $code.=<<___; | ||
43 | llgc $XX[0],0($key) | 71 | llgc $XX[0],0($key) |
44 | llgc $YY,1($key) | 72 | llgc $YY,1($key) |
45 | la $XX[0],1($XX[0]) | 73 | la $XX[0],1($XX[0]) |
@@ -90,7 +118,7 @@ $code.=<<___; | |||
90 | xgr $acc,$TX[1] | 118 | xgr $acc,$TX[1] |
91 | stg $acc,0($out) | 119 | stg $acc,0($out) |
92 | la $out,8($out) | 120 | la $out,8($out) |
93 | brct $cnt,.Loop8 | 121 | brctg $cnt,.Loop8 |
94 | 122 | ||
95 | .Lshort: | 123 | .Lshort: |
96 | lghi $acc,7 | 124 | lghi $acc,7 |
@@ -122,7 +150,7 @@ $code.=<<___; | |||
122 | ahi $XX[0],-1 | 150 | ahi $XX[0],-1 |
123 | stc $XX[0],0($key) | 151 | stc $XX[0],0($key) |
124 | stc $YY,1($key) | 152 | stc $YY,1($key) |
125 | lmg %r6,%r11,48($sp) | 153 | lm${g} %r6,%r11,6*$SIZE_T($sp) |
126 | br $rp | 154 | br $rp |
127 | .size RC4,.-RC4 | 155 | .size RC4,.-RC4 |
128 | .string "RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>" | 156 | .string "RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>" |
@@ -143,11 +171,11 @@ $ikey="%r7"; | |||
143 | $iinp="%r8"; | 171 | $iinp="%r8"; |
144 | 172 | ||
145 | $code.=<<___; | 173 | $code.=<<___; |
146 | .globl RC4_set_key | 174 | .globl private_RC4_set_key |
147 | .type RC4_set_key,\@function | 175 | .type private_RC4_set_key,\@function |
148 | .align 64 | 176 | .align 64 |
149 | RC4_set_key: | 177 | private_RC4_set_key: |
150 | stmg %r6,%r8,48($sp) | 178 | stm${g} %r6,%r8,6*$SIZE_T($sp) |
151 | lhi $cnt,256 | 179 | lhi $cnt,256 |
152 | la $idx,0(%r0) | 180 | la $idx,0(%r0) |
153 | sth $idx,0($key) | 181 | sth $idx,0($key) |
@@ -180,9 +208,9 @@ RC4_set_key: | |||
180 | la $iinp,0(%r0) | 208 | la $iinp,0(%r0) |
181 | j .L2ndloop | 209 | j .L2ndloop |
182 | .Ldone: | 210 | .Ldone: |
183 | lmg %r6,%r8,48($sp) | 211 | lm${g} %r6,%r8,6*$SIZE_T($sp) |
184 | br $rp | 212 | br $rp |
185 | .size RC4_set_key,.-RC4_set_key | 213 | .size private_RC4_set_key,.-private_RC4_set_key |
186 | 214 | ||
187 | ___ | 215 | ___ |
188 | } | 216 | } |
@@ -203,3 +231,4 @@ RC4_options: | |||
203 | ___ | 231 | ___ |
204 | 232 | ||
205 | print $code; | 233 | print $code; |
234 | close STDOUT; # force flush | ||
diff --git a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl index 677be5fe25..d6eac205e9 100755 --- a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl +++ b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl | |||
@@ -7,6 +7,8 @@ | |||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | 7 | # details see http://www.openssl.org/~appro/cryptogams/. |
8 | # ==================================================================== | 8 | # ==================================================================== |
9 | # | 9 | # |
10 | # July 2004 | ||
11 | # | ||
10 | # 2.22x RC4 tune-up:-) It should be noted though that my hand [as in | 12 | # 2.22x RC4 tune-up:-) It should be noted though that my hand [as in |
11 | # "hand-coded assembler"] doesn't stand for the whole improvement | 13 | # "hand-coded assembler"] doesn't stand for the whole improvement |
12 | # coefficient. It turned out that eliminating RC4_CHAR from config | 14 | # coefficient. It turned out that eliminating RC4_CHAR from config |
@@ -19,6 +21,8 @@ | |||
19 | # to operate on partial registers, it turned out to be the best bet. | 21 | # to operate on partial registers, it turned out to be the best bet. |
20 | # At least for AMD... How IA32E would perform remains to be seen... | 22 | # At least for AMD... How IA32E would perform remains to be seen... |
21 | 23 | ||
24 | # November 2004 | ||
25 | # | ||
22 | # As was shown by Marc Bevand reordering of couple of load operations | 26 | # As was shown by Marc Bevand reordering of couple of load operations |
23 | # results in even higher performance gain of 3.3x:-) At least on | 27 | # results in even higher performance gain of 3.3x:-) At least on |
24 | # Opteron... For reference, 1x in this case is RC4_CHAR C-code | 28 | # Opteron... For reference, 1x in this case is RC4_CHAR C-code |
@@ -26,6 +30,8 @@ | |||
26 | # Latter means that if you want to *estimate* what to expect from | 30 | # Latter means that if you want to *estimate* what to expect from |
27 | # *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz. | 31 | # *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz. |
28 | 32 | ||
33 | # November 2004 | ||
34 | # | ||
29 | # Intel P4 EM64T core was found to run the AMD64 code really slow... | 35 | # Intel P4 EM64T core was found to run the AMD64 code really slow... |
30 | # The only way to achieve comparable performance on P4 was to keep | 36 | # The only way to achieve comparable performance on P4 was to keep |
31 | # RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to | 37 | # RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to |
@@ -33,10 +39,14 @@ | |||
33 | # on either AMD and Intel platforms, I implement both cases. See | 39 | # on either AMD and Intel platforms, I implement both cases. See |
34 | # rc4_skey.c for further details... | 40 | # rc4_skey.c for further details... |
35 | 41 | ||
42 | # April 2005 | ||
43 | # | ||
36 | # P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing | 44 | # P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing |
37 | # those with add/sub results in 50% performance improvement of folded | 45 | # those with add/sub results in 50% performance improvement of folded |
38 | # loop... | 46 | # loop... |
39 | 47 | ||
48 | # May 2005 | ||
49 | # | ||
40 | # As was shown by Zou Nanhai loop unrolling can improve Intel EM64T | 50 | # As was shown by Zou Nanhai loop unrolling can improve Intel EM64T |
41 | # performance by >30% [unlike P4 32-bit case that is]. But this is | 51 | # performance by >30% [unlike P4 32-bit case that is]. But this is |
42 | # provided that loads are reordered even more aggressively! Both code | 52 | # provided that loads are reordered even more aggressively! Both code |
@@ -50,6 +60,8 @@ | |||
50 | # is not implemented, then this final RC4_CHAR code-path should be | 60 | # is not implemented, then this final RC4_CHAR code-path should be |
51 | # preferred, as it provides better *all-round* performance]. | 61 | # preferred, as it provides better *all-round* performance]. |
52 | 62 | ||
63 | # March 2007 | ||
64 | # | ||
53 | # Intel Core2 was observed to perform poorly on both code paths:-( It | 65 | # Intel Core2 was observed to perform poorly on both code paths:-( It |
54 | # apparently suffers from some kind of partial register stall, which | 66 | # apparently suffers from some kind of partial register stall, which |
55 | # occurs in 64-bit mode only [as virtually identical 32-bit loop was | 67 | # occurs in 64-bit mode only [as virtually identical 32-bit loop was |
@@ -58,6 +70,37 @@ | |||
58 | # fit for Core2 and therefore the code was modified to skip cloop8 on | 70 | # fit for Core2 and therefore the code was modified to skip cloop8 on |
59 | # this CPU. | 71 | # this CPU. |
60 | 72 | ||
73 | # May 2010 | ||
74 | # | ||
75 | # Intel Westmere was observed to perform suboptimally. Adding yet | ||
76 | # another movzb to cloop1 improved performance by almost 50%! Core2 | ||
77 | # performance is improved too, but nominally... | ||
78 | |||
79 | # May 2011 | ||
80 | # | ||
81 | # The only code path that was not modified is P4-specific one. Non-P4 | ||
82 | # Intel code path optimization is heavily based on submission by Maxim | ||
83 | # Perminov, Maxim Locktyukhin and Jim Guilford of Intel. I've used | ||
84 | # some of the ideas even in attempt to optmize the original RC4_INT | ||
85 | # code path... Current performance in cycles per processed byte (less | ||
86 | # is better) and improvement coefficients relative to previous | ||
87 | # version of this module are: | ||
88 | # | ||
89 | # Opteron 5.3/+0%(*) | ||
90 | # P4 6.5 | ||
91 | # Core2 6.2/+15%(**) | ||
92 | # Westmere 4.2/+60% | ||
93 | # Sandy Bridge 4.2/+120% | ||
94 | # Atom 9.3/+80% | ||
95 | # | ||
96 | # (*) But corresponding loop has less instructions, which should have | ||
97 | # positive effect on upcoming Bulldozer, which has one less ALU. | ||
98 | # For reference, Intel code runs at 6.8 cpb rate on Opteron. | ||
99 | # (**) Note that Core2 result is ~15% lower than corresponding result | ||
100 | # for 32-bit code, meaning that it's possible to improve it, | ||
101 | # but more than likely at the cost of the others (see rc4-586.pl | ||
102 | # to get the idea)... | ||
103 | |||
61 | $flavour = shift; | 104 | $flavour = shift; |
62 | $output = shift; | 105 | $output = shift; |
63 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | 106 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } |
@@ -76,13 +119,10 @@ $len="%rsi"; # arg2 | |||
76 | $inp="%rdx"; # arg3 | 119 | $inp="%rdx"; # arg3 |
77 | $out="%rcx"; # arg4 | 120 | $out="%rcx"; # arg4 |
78 | 121 | ||
79 | @XX=("%r8","%r10"); | 122 | { |
80 | @TX=("%r9","%r11"); | ||
81 | $YY="%r12"; | ||
82 | $TY="%r13"; | ||
83 | |||
84 | $code=<<___; | 123 | $code=<<___; |
85 | .text | 124 | .text |
125 | .extern OPENSSL_ia32cap_P | ||
86 | 126 | ||
87 | .globl RC4 | 127 | .globl RC4 |
88 | .type RC4,\@function,4 | 128 | .type RC4,\@function,4 |
@@ -95,48 +135,173 @@ RC4: or $len,$len | |||
95 | push %r12 | 135 | push %r12 |
96 | push %r13 | 136 | push %r13 |
97 | .Lprologue: | 137 | .Lprologue: |
138 | mov $len,%r11 | ||
139 | mov $inp,%r12 | ||
140 | mov $out,%r13 | ||
141 | ___ | ||
142 | my $len="%r11"; # reassign input arguments | ||
143 | my $inp="%r12"; | ||
144 | my $out="%r13"; | ||
98 | 145 | ||
99 | add \$8,$dat | 146 | my @XX=("%r10","%rsi"); |
100 | movl -8($dat),$XX[0]#d | 147 | my @TX=("%rax","%rbx"); |
101 | movl -4($dat),$YY#d | 148 | my $YY="%rcx"; |
149 | my $TY="%rdx"; | ||
150 | |||
151 | $code.=<<___; | ||
152 | xor $XX[0],$XX[0] | ||
153 | xor $YY,$YY | ||
154 | |||
155 | lea 8($dat),$dat | ||
156 | mov -8($dat),$XX[0]#b | ||
157 | mov -4($dat),$YY#b | ||
102 | cmpl \$-1,256($dat) | 158 | cmpl \$-1,256($dat) |
103 | je .LRC4_CHAR | 159 | je .LRC4_CHAR |
160 | mov OPENSSL_ia32cap_P(%rip),%r8d | ||
161 | xor $TX[1],$TX[1] | ||
104 | inc $XX[0]#b | 162 | inc $XX[0]#b |
163 | sub $XX[0],$TX[1] | ||
164 | sub $inp,$out | ||
105 | movl ($dat,$XX[0],4),$TX[0]#d | 165 | movl ($dat,$XX[0],4),$TX[0]#d |
106 | test \$-8,$len | 166 | test \$-16,$len |
107 | jz .Lloop1 | 167 | jz .Lloop1 |
108 | jmp .Lloop8 | 168 | bt \$30,%r8d # Intel CPU? |
169 | jc .Lintel | ||
170 | and \$7,$TX[1] | ||
171 | lea 1($XX[0]),$XX[1] | ||
172 | jz .Loop8 | ||
173 | sub $TX[1],$len | ||
174 | .Loop8_warmup: | ||
175 | add $TX[0]#b,$YY#b | ||
176 | movl ($dat,$YY,4),$TY#d | ||
177 | movl $TX[0]#d,($dat,$YY,4) | ||
178 | movl $TY#d,($dat,$XX[0],4) | ||
179 | add $TY#b,$TX[0]#b | ||
180 | inc $XX[0]#b | ||
181 | movl ($dat,$TX[0],4),$TY#d | ||
182 | movl ($dat,$XX[0],4),$TX[0]#d | ||
183 | xorb ($inp),$TY#b | ||
184 | movb $TY#b,($out,$inp) | ||
185 | lea 1($inp),$inp | ||
186 | dec $TX[1] | ||
187 | jnz .Loop8_warmup | ||
188 | |||
189 | lea 1($XX[0]),$XX[1] | ||
190 | jmp .Loop8 | ||
109 | .align 16 | 191 | .align 16 |
110 | .Lloop8: | 192 | .Loop8: |
111 | ___ | 193 | ___ |
112 | for ($i=0;$i<8;$i++) { | 194 | for ($i=0;$i<8;$i++) { |
195 | $code.=<<___ if ($i==7); | ||
196 | add \$8,$XX[1]#b | ||
197 | ___ | ||
113 | $code.=<<___; | 198 | $code.=<<___; |
114 | add $TX[0]#b,$YY#b | 199 | add $TX[0]#b,$YY#b |
115 | mov $XX[0],$XX[1] | ||
116 | movl ($dat,$YY,4),$TY#d | 200 | movl ($dat,$YY,4),$TY#d |
117 | ror \$8,%rax # ror is redundant when $i=0 | ||
118 | inc $XX[1]#b | ||
119 | movl ($dat,$XX[1],4),$TX[1]#d | ||
120 | cmp $XX[1],$YY | ||
121 | movl $TX[0]#d,($dat,$YY,4) | 201 | movl $TX[0]#d,($dat,$YY,4) |
122 | cmove $TX[0],$TX[1] | 202 | movl `4*($i==7?-1:$i)`($dat,$XX[1],4),$TX[1]#d |
123 | movl $TY#d,($dat,$XX[0],4) | 203 | ror \$8,%r8 # ror is redundant when $i=0 |
204 | movl $TY#d,4*$i($dat,$XX[0],4) | ||
124 | add $TX[0]#b,$TY#b | 205 | add $TX[0]#b,$TY#b |
125 | movb ($dat,$TY,4),%al | 206 | movb ($dat,$TY,4),%r8b |
126 | ___ | 207 | ___ |
127 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers | 208 | push(@TX,shift(@TX)); #push(@XX,shift(@XX)); # "rotate" registers |
128 | } | 209 | } |
129 | $code.=<<___; | 210 | $code.=<<___; |
130 | ror \$8,%rax | 211 | add \$8,$XX[0]#b |
212 | ror \$8,%r8 | ||
131 | sub \$8,$len | 213 | sub \$8,$len |
132 | 214 | ||
133 | xor ($inp),%rax | 215 | xor ($inp),%r8 |
134 | add \$8,$inp | 216 | mov %r8,($out,$inp) |
135 | mov %rax,($out) | 217 | lea 8($inp),$inp |
136 | add \$8,$out | ||
137 | 218 | ||
138 | test \$-8,$len | 219 | test \$-8,$len |
139 | jnz .Lloop8 | 220 | jnz .Loop8 |
221 | cmp \$0,$len | ||
222 | jne .Lloop1 | ||
223 | jmp .Lexit | ||
224 | |||
225 | .align 16 | ||
226 | .Lintel: | ||
227 | test \$-32,$len | ||
228 | jz .Lloop1 | ||
229 | and \$15,$TX[1] | ||
230 | jz .Loop16_is_hot | ||
231 | sub $TX[1],$len | ||
232 | .Loop16_warmup: | ||
233 | add $TX[0]#b,$YY#b | ||
234 | movl ($dat,$YY,4),$TY#d | ||
235 | movl $TX[0]#d,($dat,$YY,4) | ||
236 | movl $TY#d,($dat,$XX[0],4) | ||
237 | add $TY#b,$TX[0]#b | ||
238 | inc $XX[0]#b | ||
239 | movl ($dat,$TX[0],4),$TY#d | ||
240 | movl ($dat,$XX[0],4),$TX[0]#d | ||
241 | xorb ($inp),$TY#b | ||
242 | movb $TY#b,($out,$inp) | ||
243 | lea 1($inp),$inp | ||
244 | dec $TX[1] | ||
245 | jnz .Loop16_warmup | ||
246 | |||
247 | mov $YY,$TX[1] | ||
248 | xor $YY,$YY | ||
249 | mov $TX[1]#b,$YY#b | ||
250 | |||
251 | .Loop16_is_hot: | ||
252 | lea ($dat,$XX[0],4),$XX[1] | ||
253 | ___ | ||
254 | sub RC4_loop { | ||
255 | my $i=shift; | ||
256 | my $j=$i<0?0:$i; | ||
257 | my $xmm="%xmm".($j&1); | ||
258 | |||
259 | $code.=" add \$16,$XX[0]#b\n" if ($i==15); | ||
260 | $code.=" movdqu ($inp),%xmm2\n" if ($i==15); | ||
261 | $code.=" add $TX[0]#b,$YY#b\n" if ($i<=0); | ||
262 | $code.=" movl ($dat,$YY,4),$TY#d\n"; | ||
263 | $code.=" pxor %xmm0,%xmm2\n" if ($i==0); | ||
264 | $code.=" psllq \$8,%xmm1\n" if ($i==0); | ||
265 | $code.=" pxor $xmm,$xmm\n" if ($i<=1); | ||
266 | $code.=" movl $TX[0]#d,($dat,$YY,4)\n"; | ||
267 | $code.=" add $TY#b,$TX[0]#b\n"; | ||
268 | $code.=" movl `4*($j+1)`($XX[1]),$TX[1]#d\n" if ($i<15); | ||
269 | $code.=" movz $TX[0]#b,$TX[0]#d\n"; | ||
270 | $code.=" movl $TY#d,4*$j($XX[1])\n"; | ||
271 | $code.=" pxor %xmm1,%xmm2\n" if ($i==0); | ||
272 | $code.=" lea ($dat,$XX[0],4),$XX[1]\n" if ($i==15); | ||
273 | $code.=" add $TX[1]#b,$YY#b\n" if ($i<15); | ||
274 | $code.=" pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n"; | ||
275 | $code.=" movdqu %xmm2,($out,$inp)\n" if ($i==0); | ||
276 | $code.=" lea 16($inp),$inp\n" if ($i==0); | ||
277 | $code.=" movl ($XX[1]),$TX[1]#d\n" if ($i==15); | ||
278 | } | ||
279 | RC4_loop(-1); | ||
280 | $code.=<<___; | ||
281 | jmp .Loop16_enter | ||
282 | .align 16 | ||
283 | .Loop16: | ||
284 | ___ | ||
285 | |||
286 | for ($i=0;$i<16;$i++) { | ||
287 | $code.=".Loop16_enter:\n" if ($i==1); | ||
288 | RC4_loop($i); | ||
289 | push(@TX,shift(@TX)); # "rotate" registers | ||
290 | } | ||
291 | $code.=<<___; | ||
292 | mov $YY,$TX[1] | ||
293 | xor $YY,$YY # keyword to partial register | ||
294 | sub \$16,$len | ||
295 | mov $TX[1]#b,$YY#b | ||
296 | test \$-16,$len | ||
297 | jnz .Loop16 | ||
298 | |||
299 | psllq \$8,%xmm1 | ||
300 | pxor %xmm0,%xmm2 | ||
301 | pxor %xmm1,%xmm2 | ||
302 | movdqu %xmm2,($out,$inp) | ||
303 | lea 16($inp),$inp | ||
304 | |||
140 | cmp \$0,$len | 305 | cmp \$0,$len |
141 | jne .Lloop1 | 306 | jne .Lloop1 |
142 | jmp .Lexit | 307 | jmp .Lexit |
@@ -152,9 +317,8 @@ $code.=<<___; | |||
152 | movl ($dat,$TX[0],4),$TY#d | 317 | movl ($dat,$TX[0],4),$TY#d |
153 | movl ($dat,$XX[0],4),$TX[0]#d | 318 | movl ($dat,$XX[0],4),$TX[0]#d |
154 | xorb ($inp),$TY#b | 319 | xorb ($inp),$TY#b |
155 | inc $inp | 320 | movb $TY#b,($out,$inp) |
156 | movb $TY#b,($out) | 321 | lea 1($inp),$inp |
157 | inc $out | ||
158 | dec $len | 322 | dec $len |
159 | jnz .Lloop1 | 323 | jnz .Lloop1 |
160 | jmp .Lexit | 324 | jmp .Lexit |
@@ -165,13 +329,11 @@ $code.=<<___; | |||
165 | movzb ($dat,$XX[0]),$TX[0]#d | 329 | movzb ($dat,$XX[0]),$TX[0]#d |
166 | test \$-8,$len | 330 | test \$-8,$len |
167 | jz .Lcloop1 | 331 | jz .Lcloop1 |
168 | cmpl \$0,260($dat) | ||
169 | jnz .Lcloop1 | ||
170 | jmp .Lcloop8 | 332 | jmp .Lcloop8 |
171 | .align 16 | 333 | .align 16 |
172 | .Lcloop8: | 334 | .Lcloop8: |
173 | mov ($inp),%eax | 335 | mov ($inp),%r8d |
174 | mov 4($inp),%ebx | 336 | mov 4($inp),%r9d |
175 | ___ | 337 | ___ |
176 | # unroll 2x4-wise, because 64-bit rotates kill Intel P4... | 338 | # unroll 2x4-wise, because 64-bit rotates kill Intel P4... |
177 | for ($i=0;$i<4;$i++) { | 339 | for ($i=0;$i<4;$i++) { |
@@ -188,8 +350,8 @@ $code.=<<___; | |||
188 | mov $TX[0],$TX[1] | 350 | mov $TX[0],$TX[1] |
189 | .Lcmov$i: | 351 | .Lcmov$i: |
190 | add $TX[0]#b,$TY#b | 352 | add $TX[0]#b,$TY#b |
191 | xor ($dat,$TY),%al | 353 | xor ($dat,$TY),%r8b |
192 | ror \$8,%eax | 354 | ror \$8,%r8d |
193 | ___ | 355 | ___ |
194 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers | 356 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers |
195 | } | 357 | } |
@@ -207,16 +369,16 @@ $code.=<<___; | |||
207 | mov $TX[0],$TX[1] | 369 | mov $TX[0],$TX[1] |
208 | .Lcmov$i: | 370 | .Lcmov$i: |
209 | add $TX[0]#b,$TY#b | 371 | add $TX[0]#b,$TY#b |
210 | xor ($dat,$TY),%bl | 372 | xor ($dat,$TY),%r9b |
211 | ror \$8,%ebx | 373 | ror \$8,%r9d |
212 | ___ | 374 | ___ |
213 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers | 375 | push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers |
214 | } | 376 | } |
215 | $code.=<<___; | 377 | $code.=<<___; |
216 | lea -8($len),$len | 378 | lea -8($len),$len |
217 | mov %eax,($out) | 379 | mov %r8d,($out) |
218 | lea 8($inp),$inp | 380 | lea 8($inp),$inp |
219 | mov %ebx,4($out) | 381 | mov %r9d,4($out) |
220 | lea 8($out),$out | 382 | lea 8($out),$out |
221 | 383 | ||
222 | test \$-8,$len | 384 | test \$-8,$len |
@@ -229,6 +391,7 @@ $code.=<<___; | |||
229 | .align 16 | 391 | .align 16 |
230 | .Lcloop1: | 392 | .Lcloop1: |
231 | add $TX[0]#b,$YY#b | 393 | add $TX[0]#b,$YY#b |
394 | movzb $YY#b,$YY#d | ||
232 | movzb ($dat,$YY),$TY#d | 395 | movzb ($dat,$YY),$TY#d |
233 | movb $TX[0]#b,($dat,$YY) | 396 | movb $TX[0]#b,($dat,$YY) |
234 | movb $TY#b,($dat,$XX[0]) | 397 | movb $TY#b,($dat,$XX[0]) |
@@ -260,16 +423,16 @@ $code.=<<___; | |||
260 | ret | 423 | ret |
261 | .size RC4,.-RC4 | 424 | .size RC4,.-RC4 |
262 | ___ | 425 | ___ |
426 | } | ||
263 | 427 | ||
264 | $idx="%r8"; | 428 | $idx="%r8"; |
265 | $ido="%r9"; | 429 | $ido="%r9"; |
266 | 430 | ||
267 | $code.=<<___; | 431 | $code.=<<___; |
268 | .extern OPENSSL_ia32cap_P | 432 | .globl private_RC4_set_key |
269 | .globl RC4_set_key | 433 | .type private_RC4_set_key,\@function,3 |
270 | .type RC4_set_key,\@function,3 | ||
271 | .align 16 | 434 | .align 16 |
272 | RC4_set_key: | 435 | private_RC4_set_key: |
273 | lea 8($dat),$dat | 436 | lea 8($dat),$dat |
274 | lea ($inp,$len),$inp | 437 | lea ($inp,$len),$inp |
275 | neg $len | 438 | neg $len |
@@ -280,12 +443,9 @@ RC4_set_key: | |||
280 | xor %r11,%r11 | 443 | xor %r11,%r11 |
281 | 444 | ||
282 | mov OPENSSL_ia32cap_P(%rip),$idx#d | 445 | mov OPENSSL_ia32cap_P(%rip),$idx#d |
283 | bt \$20,$idx#d | 446 | bt \$20,$idx#d # RC4_CHAR? |
284 | jnc .Lw1stloop | 447 | jc .Lc1stloop |
285 | bt \$30,$idx#d | 448 | jmp .Lw1stloop |
286 | setc $ido#b | ||
287 | mov $ido#d,260($dat) | ||
288 | jmp .Lc1stloop | ||
289 | 449 | ||
290 | .align 16 | 450 | .align 16 |
291 | .Lw1stloop: | 451 | .Lw1stloop: |
@@ -339,7 +499,7 @@ RC4_set_key: | |||
339 | mov %eax,-8($dat) | 499 | mov %eax,-8($dat) |
340 | mov %eax,-4($dat) | 500 | mov %eax,-4($dat) |
341 | ret | 501 | ret |
342 | .size RC4_set_key,.-RC4_set_key | 502 | .size private_RC4_set_key,.-private_RC4_set_key |
343 | 503 | ||
344 | .globl RC4_options | 504 | .globl RC4_options |
345 | .type RC4_options,\@abi-omnipotent | 505 | .type RC4_options,\@abi-omnipotent |
@@ -348,18 +508,20 @@ RC4_options: | |||
348 | lea .Lopts(%rip),%rax | 508 | lea .Lopts(%rip),%rax |
349 | mov OPENSSL_ia32cap_P(%rip),%edx | 509 | mov OPENSSL_ia32cap_P(%rip),%edx |
350 | bt \$20,%edx | 510 | bt \$20,%edx |
351 | jnc .Ldone | 511 | jc .L8xchar |
352 | add \$12,%rax | ||
353 | bt \$30,%edx | 512 | bt \$30,%edx |
354 | jnc .Ldone | 513 | jnc .Ldone |
355 | add \$13,%rax | 514 | add \$25,%rax |
515 | ret | ||
516 | .L8xchar: | ||
517 | add \$12,%rax | ||
356 | .Ldone: | 518 | .Ldone: |
357 | ret | 519 | ret |
358 | .align 64 | 520 | .align 64 |
359 | .Lopts: | 521 | .Lopts: |
360 | .asciz "rc4(8x,int)" | 522 | .asciz "rc4(8x,int)" |
361 | .asciz "rc4(8x,char)" | 523 | .asciz "rc4(8x,char)" |
362 | .asciz "rc4(1x,char)" | 524 | .asciz "rc4(16x,int)" |
363 | .asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>" | 525 | .asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>" |
364 | .align 64 | 526 | .align 64 |
365 | .size RC4_options,.-RC4_options | 527 | .size RC4_options,.-RC4_options |
@@ -482,22 +644,32 @@ key_se_handler: | |||
482 | .rva .LSEH_end_RC4 | 644 | .rva .LSEH_end_RC4 |
483 | .rva .LSEH_info_RC4 | 645 | .rva .LSEH_info_RC4 |
484 | 646 | ||
485 | .rva .LSEH_begin_RC4_set_key | 647 | .rva .LSEH_begin_private_RC4_set_key |
486 | .rva .LSEH_end_RC4_set_key | 648 | .rva .LSEH_end_private_RC4_set_key |
487 | .rva .LSEH_info_RC4_set_key | 649 | .rva .LSEH_info_private_RC4_set_key |
488 | 650 | ||
489 | .section .xdata | 651 | .section .xdata |
490 | .align 8 | 652 | .align 8 |
491 | .LSEH_info_RC4: | 653 | .LSEH_info_RC4: |
492 | .byte 9,0,0,0 | 654 | .byte 9,0,0,0 |
493 | .rva stream_se_handler | 655 | .rva stream_se_handler |
494 | .LSEH_info_RC4_set_key: | 656 | .LSEH_info_private_RC4_set_key: |
495 | .byte 9,0,0,0 | 657 | .byte 9,0,0,0 |
496 | .rva key_se_handler | 658 | .rva key_se_handler |
497 | ___ | 659 | ___ |
498 | } | 660 | } |
499 | 661 | ||
500 | $code =~ s/#([bwd])/$1/gm; | 662 | sub reg_part { |
663 | my ($reg,$conv)=@_; | ||
664 | if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } | ||
665 | elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } | ||
666 | elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } | ||
667 | elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } | ||
668 | return $reg; | ||
669 | } | ||
670 | |||
671 | $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; | ||
672 | $code =~ s/\`([^\`]*)\`/eval $1/gem; | ||
501 | 673 | ||
502 | print $code; | 674 | print $code; |
503 | 675 | ||
diff --git a/src/lib/libcrypto/rc4/rc4.h b/src/lib/libcrypto/rc4/rc4.h index 29d1acccf5..88ceb46bc5 100644 --- a/src/lib/libcrypto/rc4/rc4.h +++ b/src/lib/libcrypto/rc4/rc4.h | |||
@@ -79,6 +79,7 @@ typedef struct rc4_key_st | |||
79 | 79 | ||
80 | const char *RC4_options(void); | 80 | const char *RC4_options(void); |
81 | void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data); | 81 | void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data); |
82 | void private_RC4_set_key(RC4_KEY *key, int len, const unsigned char *data); | ||
82 | void RC4(RC4_KEY *key, size_t len, const unsigned char *indata, | 83 | void RC4(RC4_KEY *key, size_t len, const unsigned char *indata, |
83 | unsigned char *outdata); | 84 | unsigned char *outdata); |
84 | 85 | ||
diff --git a/src/lib/libcrypto/rc4/rc4_skey.c b/src/lib/libcrypto/rc4/rc4_skey.c index b22c40b0bd..fda27636e7 100644 --- a/src/lib/libcrypto/rc4/rc4_skey.c +++ b/src/lib/libcrypto/rc4/rc4_skey.c | |||
@@ -85,7 +85,7 @@ const char *RC4_options(void) | |||
85 | * Date: Wed, 14 Sep 1994 06:35:31 GMT | 85 | * Date: Wed, 14 Sep 1994 06:35:31 GMT |
86 | */ | 86 | */ |
87 | 87 | ||
88 | void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data) | 88 | void private_RC4_set_key(RC4_KEY *key, int len, const unsigned char *data) |
89 | { | 89 | { |
90 | register RC4_INT tmp; | 90 | register RC4_INT tmp; |
91 | register int id1,id2; | 91 | register int id1,id2; |
@@ -104,40 +104,6 @@ void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data) | |||
104 | d[(n)]=d[id2]; \ | 104 | d[(n)]=d[id2]; \ |
105 | d[id2]=tmp; } | 105 | d[id2]=tmp; } |
106 | 106 | ||
107 | #if defined(OPENSSL_CPUID_OBJ) && !defined(OPENSSL_NO_ASM) | ||
108 | # if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ | ||
109 | defined(__INTEL__) || \ | ||
110 | defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) | ||
111 | if (sizeof(RC4_INT) > 1) { | ||
112 | /* | ||
113 | * Unlike all other x86 [and x86_64] implementations, | ||
114 | * Intel P4 core [including EM64T] was found to perform | ||
115 | * poorly with wider RC4_INT. Performance improvement | ||
116 | * for IA-32 hand-coded assembler turned out to be 2.8x | ||
117 | * if re-coded for RC4_CHAR! It's however inappropriate | ||
118 | * to just switch to RC4_CHAR for x86[_64], as non-P4 | ||
119 | * implementations suffer from significant performance | ||
120 | * losses then, e.g. PIII exhibits >2x deterioration, | ||
121 | * and so does Opteron. In order to assure optimal | ||
122 | * all-round performance, let us [try to] detect P4 at | ||
123 | * run-time by checking upon HTT bit in CPU capability | ||
124 | * vector and set up compressed key schedule, which is | ||
125 | * recognized by correspondingly updated assembler | ||
126 | * module... | ||
127 | * <appro@fy.chalmers.se> | ||
128 | */ | ||
129 | if (OPENSSL_ia32cap_P & (1<<28)) { | ||
130 | unsigned char *cp=(unsigned char *)d; | ||
131 | |||
132 | for (i=0;i<256;i++) cp[i]=i; | ||
133 | for (i=0;i<256;i++) SK_LOOP(cp,i); | ||
134 | /* mark schedule as compressed! */ | ||
135 | d[256/sizeof(RC4_INT)]=-1; | ||
136 | return; | ||
137 | } | ||
138 | } | ||
139 | # endif | ||
140 | #endif | ||
141 | for (i=0; i < 256; i++) d[i]=i; | 107 | for (i=0; i < 256; i++) d[i]=i; |
142 | for (i=0; i < 256; i+=4) | 108 | for (i=0; i < 256; i+=4) |
143 | { | 109 | { |