summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/rc4
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/libcrypto/rc4')
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-586.pl270
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-ia64.S159
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-ia64.pl755
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl631
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-parisc.pl313
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-s390x.pl234
-rwxr-xr-xsrc/lib/libcrypto/rc4/asm/rc4-x86_64.pl504
-rw-r--r--src/lib/libcrypto/rc4/rc4.h89
-rw-r--r--src/lib/libcrypto/rc4/rc4_enc.c315
-rw-r--r--src/lib/libcrypto/rc4/rc4_locl.h5
-rw-r--r--src/lib/libcrypto/rc4/rc4_skey.c150
-rw-r--r--src/lib/libcrypto/rc4/rc4_utl.c (renamed from src/lib/libcrypto/rc4/rc4_fblk.c)33
-rw-r--r--src/lib/libcrypto/rc4/rc4test.c6
13 files changed, 3276 insertions, 188 deletions
diff --git a/src/lib/libcrypto/rc4/asm/rc4-586.pl b/src/lib/libcrypto/rc4/asm/rc4-586.pl
new file mode 100644
index 0000000000..38a44a70ef
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-586.pl
@@ -0,0 +1,270 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# At some point it became apparent that the original SSLeay RC4
11# assembler implementation performs suboptimally on latest IA-32
12# microarchitectures. After re-tuning performance has changed as
13# following:
14#
15# Pentium -10%
16# Pentium III +12%
17# AMD +50%(*)
18# P4 +250%(**)
19#
20# (*) This number is actually a trade-off:-) It's possible to
21# achieve +72%, but at the cost of -48% off PIII performance.
22# In other words code performing further 13% faster on AMD
23# would perform almost 2 times slower on Intel PIII...
24# For reference! This code delivers ~80% of rc4-amd64.pl
25# performance on the same Opteron machine.
26# (**) This number requires compressed key schedule set up by
27# RC4_set_key [see commentary below for further details].
28#
29# <appro@fy.chalmers.se>
30
31$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
32push(@INC,"${dir}","${dir}../../perlasm");
33require "x86asm.pl";
34
35&asm_init($ARGV[0],"rc4-586.pl");
36
37$xx="eax";
38$yy="ebx";
39$tx="ecx";
40$ty="edx";
41$inp="esi";
42$out="ebp";
43$dat="edi";
44
45sub RC4_loop {
46 my $i=shift;
47 my $func = ($i==0)?*mov:*or;
48
49 &add (&LB($yy),&LB($tx));
50 &mov ($ty,&DWP(0,$dat,$yy,4));
51 &mov (&DWP(0,$dat,$yy,4),$tx);
52 &mov (&DWP(0,$dat,$xx,4),$ty);
53 &add ($ty,$tx);
54 &inc (&LB($xx));
55 &and ($ty,0xff);
56 &ror ($out,8) if ($i!=0);
57 if ($i<3) {
58 &mov ($tx,&DWP(0,$dat,$xx,4));
59 } else {
60 &mov ($tx,&wparam(3)); # reload [re-biased] out
61 }
62 &$func ($out,&DWP(0,$dat,$ty,4));
63}
64
65# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
66&function_begin("RC4");
67 &mov ($dat,&wparam(0)); # load key schedule pointer
68 &mov ($ty, &wparam(1)); # load len
69 &mov ($inp,&wparam(2)); # load inp
70 &mov ($out,&wparam(3)); # load out
71
72 &xor ($xx,$xx); # avoid partial register stalls
73 &xor ($yy,$yy);
74
75 &cmp ($ty,0); # safety net
76 &je (&label("abort"));
77
78 &mov (&LB($xx),&BP(0,$dat)); # load key->x
79 &mov (&LB($yy),&BP(4,$dat)); # load key->y
80 &add ($dat,8);
81
82 &lea ($tx,&DWP(0,$inp,$ty));
83 &sub ($out,$inp); # re-bias out
84 &mov (&wparam(1),$tx); # save input+len
85
86 &inc (&LB($xx));
87
88 # detect compressed key schedule...
89 &cmp (&DWP(256,$dat),-1);
90 &je (&label("RC4_CHAR"));
91
92 &mov ($tx,&DWP(0,$dat,$xx,4));
93
94 &and ($ty,-4); # how many 4-byte chunks?
95 &jz (&label("loop1"));
96
97 &lea ($ty,&DWP(-4,$inp,$ty));
98 &mov (&wparam(2),$ty); # save input+(len/4)*4-4
99 &mov (&wparam(3),$out); # $out as accumulator in this loop
100
101 &set_label("loop4",16);
102 for ($i=0;$i<4;$i++) { RC4_loop($i); }
103 &ror ($out,8);
104 &xor ($out,&DWP(0,$inp));
105 &cmp ($inp,&wparam(2)); # compare to input+(len/4)*4-4
106 &mov (&DWP(0,$tx,$inp),$out);# $tx holds re-biased out here
107 &lea ($inp,&DWP(4,$inp));
108 &mov ($tx,&DWP(0,$dat,$xx,4));
109 &jb (&label("loop4"));
110
111 &cmp ($inp,&wparam(1)); # compare to input+len
112 &je (&label("done"));
113 &mov ($out,&wparam(3)); # restore $out
114
115 &set_label("loop1",16);
116 &add (&LB($yy),&LB($tx));
117 &mov ($ty,&DWP(0,$dat,$yy,4));
118 &mov (&DWP(0,$dat,$yy,4),$tx);
119 &mov (&DWP(0,$dat,$xx,4),$ty);
120 &add ($ty,$tx);
121 &inc (&LB($xx));
122 &and ($ty,0xff);
123 &mov ($ty,&DWP(0,$dat,$ty,4));
124 &xor (&LB($ty),&BP(0,$inp));
125 &lea ($inp,&DWP(1,$inp));
126 &mov ($tx,&DWP(0,$dat,$xx,4));
127 &cmp ($inp,&wparam(1)); # compare to input+len
128 &mov (&BP(-1,$out,$inp),&LB($ty));
129 &jb (&label("loop1"));
130
131 &jmp (&label("done"));
132
133# this is essentially Intel P4 specific codepath...
134&set_label("RC4_CHAR",16);
135 &movz ($tx,&BP(0,$dat,$xx));
136 # strangely enough unrolled loop performs over 20% slower...
137 &set_label("cloop1");
138 &add (&LB($yy),&LB($tx));
139 &movz ($ty,&BP(0,$dat,$yy));
140 &mov (&BP(0,$dat,$yy),&LB($tx));
141 &mov (&BP(0,$dat,$xx),&LB($ty));
142 &add (&LB($ty),&LB($tx));
143 &movz ($ty,&BP(0,$dat,$ty));
144 &add (&LB($xx),1);
145 &xor (&LB($ty),&BP(0,$inp));
146 &lea ($inp,&DWP(1,$inp));
147 &movz ($tx,&BP(0,$dat,$xx));
148 &cmp ($inp,&wparam(1));
149 &mov (&BP(-1,$out,$inp),&LB($ty));
150 &jb (&label("cloop1"));
151
152&set_label("done");
153 &dec (&LB($xx));
154 &mov (&BP(-4,$dat),&LB($yy)); # save key->y
155 &mov (&BP(-8,$dat),&LB($xx)); # save key->x
156&set_label("abort");
157&function_end("RC4");
158
159########################################################################
160
161$inp="esi";
162$out="edi";
163$idi="ebp";
164$ido="ecx";
165$idx="edx";
166
167&external_label("OPENSSL_ia32cap_P");
168
169# void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data);
170&function_begin("RC4_set_key");
171 &mov ($out,&wparam(0)); # load key
172 &mov ($idi,&wparam(1)); # load len
173 &mov ($inp,&wparam(2)); # load data
174 &picmeup($idx,"OPENSSL_ia32cap_P");
175
176 &lea ($out,&DWP(2*4,$out)); # &key->data
177 &lea ($inp,&DWP(0,$inp,$idi)); # $inp to point at the end
178 &neg ($idi);
179 &xor ("eax","eax");
180 &mov (&DWP(-4,$out),$idi); # borrow key->y
181
182 &bt (&DWP(0,$idx),20); # check for bit#20
183 &jc (&label("c1stloop"));
184
185&set_label("w1stloop",16);
186 &mov (&DWP(0,$out,"eax",4),"eax"); # key->data[i]=i;
187 &add (&LB("eax"),1); # i++;
188 &jnc (&label("w1stloop"));
189
190 &xor ($ido,$ido);
191 &xor ($idx,$idx);
192
193&set_label("w2ndloop",16);
194 &mov ("eax",&DWP(0,$out,$ido,4));
195 &add (&LB($idx),&BP(0,$inp,$idi));
196 &add (&LB($idx),&LB("eax"));
197 &add ($idi,1);
198 &mov ("ebx",&DWP(0,$out,$idx,4));
199 &jnz (&label("wnowrap"));
200 &mov ($idi,&DWP(-4,$out));
201 &set_label("wnowrap");
202 &mov (&DWP(0,$out,$idx,4),"eax");
203 &mov (&DWP(0,$out,$ido,4),"ebx");
204 &add (&LB($ido),1);
205 &jnc (&label("w2ndloop"));
206&jmp (&label("exit"));
207
208# Unlike all other x86 [and x86_64] implementations, Intel P4 core
209# [including EM64T] was found to perform poorly with above "32-bit" key
210# schedule, a.k.a. RC4_INT. Performance improvement for IA-32 hand-coded
211# assembler turned out to be 3.5x if re-coded for compressed 8-bit one,
212# a.k.a. RC4_CHAR! It's however inappropriate to just switch to 8-bit
213# schedule for x86[_64], because non-P4 implementations suffer from
214# significant performance losses then, e.g. PIII exhibits >2x
215# deterioration, and so does Opteron. In order to assure optimal
216# all-round performance, we detect P4 at run-time and set up compressed
217# key schedule, which is recognized by RC4 procedure.
218
219&set_label("c1stloop",16);
220 &mov (&BP(0,$out,"eax"),&LB("eax")); # key->data[i]=i;
221 &add (&LB("eax"),1); # i++;
222 &jnc (&label("c1stloop"));
223
224 &xor ($ido,$ido);
225 &xor ($idx,$idx);
226 &xor ("ebx","ebx");
227
228&set_label("c2ndloop",16);
229 &mov (&LB("eax"),&BP(0,$out,$ido));
230 &add (&LB($idx),&BP(0,$inp,$idi));
231 &add (&LB($idx),&LB("eax"));
232 &add ($idi,1);
233 &mov (&LB("ebx"),&BP(0,$out,$idx));
234 &jnz (&label("cnowrap"));
235 &mov ($idi,&DWP(-4,$out));
236 &set_label("cnowrap");
237 &mov (&BP(0,$out,$idx),&LB("eax"));
238 &mov (&BP(0,$out,$ido),&LB("ebx"));
239 &add (&LB($ido),1);
240 &jnc (&label("c2ndloop"));
241
242 &mov (&DWP(256,$out),-1); # mark schedule as compressed
243
244&set_label("exit");
245 &xor ("eax","eax");
246 &mov (&DWP(-8,$out),"eax"); # key->x=0;
247 &mov (&DWP(-4,$out),"eax"); # key->y=0;
248&function_end("RC4_set_key");
249
250# const char *RC4_options(void);
251&function_begin_B("RC4_options");
252 &call (&label("pic_point"));
253&set_label("pic_point");
254 &blindpop("eax");
255 &lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax"));
256 &picmeup("edx","OPENSSL_ia32cap_P");
257 &bt (&DWP(0,"edx"),20);
258 &jnc (&label("skip"));
259 &add ("eax",12);
260 &set_label("skip");
261 &ret ();
262&set_label("opts",64);
263&asciz ("rc4(4x,int)");
264&asciz ("rc4(1x,char)");
265&asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>");
266&align (64);
267&function_end_B("RC4_options");
268
269&asm_finish();
270
diff --git a/src/lib/libcrypto/rc4/asm/rc4-ia64.S b/src/lib/libcrypto/rc4/asm/rc4-ia64.S
deleted file mode 100644
index 8210c47d04..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-ia64.S
+++ /dev/null
@@ -1,159 +0,0 @@
1// ====================================================================
2// Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
3// project.
4//
5// Rights for redistribution and usage in source and binary forms are
6// granted according to the OpenSSL license. Warranty of any kind is
7// disclaimed.
8// ====================================================================
9
10.ident "rc4-ia64.S, Version 2.0"
11.ident "IA-64 ISA artwork by Andy Polyakov <appro@fy.chalmers.se>"
12
13// What's wrong with compiler generated code? Because of the nature of
14// C language, compiler doesn't [dare to] reorder load and stores. But
15// being memory-bound, RC4 should benefit from reorder [on in-order-
16// execution core such as IA-64]. But what can we reorder? At the very
17// least we can safely reorder references to key schedule in respect
18// to input and output streams. Secondly, from the first [close] glance
19// it appeared that it's possible to pull up some references to
20// elements of the key schedule itself. Original rationale ["prior
21// loads are not safe only for "degenerated" key schedule, when some
22// elements equal to the same value"] was kind of sloppy. I should have
23// formulated as it really was: if we assume that pulling up reference
24// to key[x+1] is not safe, then it would mean that key schedule would
25// "degenerate," which is never the case. The problem is that this
26// holds true in respect to references to key[x], but not to key[y].
27// Legitimate "collisions" do occur within every 256^2 bytes window.
28// Fortunately there're enough free instruction slots to keep prior
29// reference to key[x+1], detect "collision" and compensate for it.
30// All this without sacrificing a single clock cycle:-) Throughput is
31// ~210MBps on 900MHz CPU, which is is >3x faster than gcc generated
32// code and +30% - if compared to HP-UX C. Unrolling loop below should
33// give >30% on top of that...
34
35.text
36.explicit
37
38#if defined(_HPUX_SOURCE) && !defined(_LP64)
39# define ADDP addp4
40#else
41# define ADDP add
42#endif
43
44#ifndef SZ
45#define SZ 4 // this is set to sizeof(RC4_INT)
46#endif
47// SZ==4 seems to be optimal. At least SZ==8 is not any faster, not for
48// assembler implementation, while SZ==1 code is ~30% slower.
49#if SZ==1 // RC4_INT is unsigned char
50# define LDKEY ld1
51# define STKEY st1
52# define OFF 0
53#elif SZ==4 // RC4_INT is unsigned int
54# define LDKEY ld4
55# define STKEY st4
56# define OFF 2
57#elif SZ==8 // RC4_INT is unsigned long
58# define LDKEY ld8
59# define STKEY st8
60# define OFF 3
61#endif
62
63out=r8; // [expanded] output pointer
64inp=r9; // [expanded] output pointer
65prsave=r10;
66key=r28; // [expanded] pointer to RC4_KEY
67ksch=r29; // (key->data+255)[&~(sizeof(key->data)-1)]
68xx=r30;
69yy=r31;
70
71// void RC4(RC4_KEY *key,size_t len,const void *inp,void *out);
72.global RC4#
73.proc RC4#
74.align 32
75.skip 16
76RC4:
77 .prologue
78 .save ar.pfs,r2
79{ .mii; alloc r2=ar.pfs,4,12,0,16
80 .save pr,prsave
81 mov prsave=pr
82 ADDP key=0,in0 };;
83{ .mib; cmp.eq p6,p0=0,in1 // len==0?
84 .save ar.lc,r3
85 mov r3=ar.lc
86(p6) br.ret.spnt.many b0 };; // emergency exit
87
88 .body
89 .rotr dat[4],key_x[4],tx[2],rnd[2],key_y[2],ty[1];
90
91{ .mib; LDKEY xx=[key],SZ // load key->x
92 add in1=-1,in1 // adjust len for loop counter
93 nop.b 0 }
94{ .mib; ADDP inp=0,in2
95 ADDP out=0,in3
96 brp.loop.imp .Ltop,.Lexit-16 };;
97{ .mmi; LDKEY yy=[key] // load key->y
98 add ksch=SZ,key
99 mov ar.lc=in1 }
100{ .mmi; mov key_y[1]=r0 // guarantee inequality
101 // in first iteration
102 add xx=1,xx
103 mov pr.rot=1<<16 };;
104{ .mii; nop.m 0
105 dep key_x[1]=xx,r0,OFF,8
106 mov ar.ec=3 };; // note that epilogue counter
107 // is off by 1. I compensate
108 // for this at exit...
109.Ltop:
110// The loop is scheduled for 4*(n+2) spin-rate on Itanium 2, which
111// theoretically gives asymptotic performance of clock frequency
112// divided by 4 bytes per seconds, or 400MBps on 1.6GHz CPU. This is
113// for sizeof(RC4_INT)==4. For smaller RC4_INT STKEY inadvertently
114// splits the last bundle and you end up with 5*n spin-rate:-(
115// Originally the loop was scheduled for 3*n and relied on key
116// schedule to be aligned at 256*sizeof(RC4_INT) boundary. But
117// *(out++)=dat, which maps to st1, had same effect [inadvertent
118// bundle split] and holded the loop back. Rescheduling for 4*n
119// made it possible to eliminate dependence on specific alignment
120// and allow OpenSSH keep "abusing" our API. Reaching for 3*n would
121// require unrolling, sticking to variable shift instruction for
122// collecting output [to avoid starvation for integer shifter] and
123// copying of key schedule to controlled place in stack [so that
124// deposit instruction can serve as substitute for whole
125// key->data+((x&255)<<log2(sizeof(key->data[0])))]...
126{ .mmi; (p19) st1 [out]=dat[3],1 // *(out++)=dat
127 (p16) add xx=1,xx // x++
128 (p18) dep rnd[1]=rnd[1],r0,OFF,8 } // ((tx+ty)&255)<<OFF
129{ .mmi; (p16) add key_x[1]=ksch,key_x[1] // &key[xx&255]
130 (p17) add key_y[1]=ksch,key_y[1] };; // &key[yy&255]
131{ .mmi; (p16) LDKEY tx[0]=[key_x[1]] // tx=key[xx]
132 (p17) LDKEY ty[0]=[key_y[1]] // ty=key[yy]
133 (p16) dep key_x[0]=xx,r0,OFF,8 } // (xx&255)<<OFF
134{ .mmi; (p18) add rnd[1]=ksch,rnd[1] // &key[(tx+ty)&255]
135 (p16) cmp.ne.unc p20,p21=key_x[1],key_y[1] };;
136{ .mmi; (p18) LDKEY rnd[1]=[rnd[1]] // rnd=key[(tx+ty)&255]
137 (p16) ld1 dat[0]=[inp],1 } // dat=*(inp++)
138.pred.rel "mutex",p20,p21
139{ .mmi; (p21) add yy=yy,tx[1] // (p16)
140 (p20) add yy=yy,tx[0] // (p16) y+=tx
141 (p21) mov tx[0]=tx[1] };; // (p16)
142{ .mmi; (p17) STKEY [key_y[1]]=tx[1] // key[yy]=tx
143 (p17) STKEY [key_x[2]]=ty[0] // key[xx]=ty
144 (p16) dep key_y[0]=yy,r0,OFF,8 } // &key[yy&255]
145{ .mmb; (p17) add rnd[0]=tx[1],ty[0] // tx+=ty
146 (p18) xor dat[2]=dat[2],rnd[1] // dat^=rnd
147 br.ctop.sptk .Ltop };;
148.Lexit:
149{ .mib; STKEY [key]=yy,-SZ // save key->y
150 mov pr=prsave,0x1ffff
151 nop.b 0 }
152{ .mib; st1 [out]=dat[3],1 // compensate for truncated
153 // epilogue counter
154 add xx=-1,xx
155 nop.b 0 };;
156{ .mib; STKEY [key]=xx // save key->x
157 mov ar.lc=r3
158 br.ret.sptk.many b0 };;
159.endp RC4#
diff --git a/src/lib/libcrypto/rc4/asm/rc4-ia64.pl b/src/lib/libcrypto/rc4/asm/rc4-ia64.pl
new file mode 100644
index 0000000000..49cd5b5e69
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-ia64.pl
@@ -0,0 +1,755 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by David Mosberger <David.Mosberger@acm.org> based on the
5# Itanium optimized Crypto code which was released by HP Labs at
6# http://www.hpl.hp.com/research/linux/crypto/.
7#
8# Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
9#
10# Permission is hereby granted, free of charge, to any person obtaining
11# a copy of this software and associated documentation files (the
12# "Software"), to deal in the Software without restriction, including
13# without limitation the rights to use, copy, modify, merge, publish,
14# distribute, sublicense, and/or sell copies of the Software, and to
15# permit persons to whom the Software is furnished to do so, subject to
16# the following conditions:
17#
18# The above copyright notice and this permission notice shall be
19# included in all copies or substantial portions of the Software.
20
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
25# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
26# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
27# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
28
29
30
31# This is a little helper program which generates a software-pipelined
32# for RC4 encryption. The basic algorithm looks like this:
33#
34# for (counter = 0; counter < len; ++counter)
35# {
36# in = inp[counter];
37# SI = S[I];
38# J = (SI + J) & 0xff;
39# SJ = S[J];
40# T = (SI + SJ) & 0xff;
41# S[I] = SJ, S[J] = SI;
42# ST = S[T];
43# outp[counter] = in ^ ST;
44# I = (I + 1) & 0xff;
45# }
46#
47# Pipelining this loop isn't easy, because the stores to the S[] array
48# need to be observed in the right order. The loop generated by the
49# code below has the following pipeline diagram:
50#
51# cycle
52# | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |10 |11 |12 |13 |14 |15 |16 |17 |
53# iter
54# 1: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
55# 2: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
56# 3: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
57#
58# where:
59# LDI = load of S[I]
60# LDJ = load of S[J]
61# SWP = swap of S[I] and S[J]
62# LDT = load of S[T]
63#
64# Note that in the above diagram, the major trouble-spot is that LDI
65# of the 2nd iteration is performed BEFORE the SWP of the first
66# iteration. Fortunately, this is easy to detect (I of the 1st
67# iteration will be equal to J of the 2nd iteration) and when this
68# happens, we simply forward the proper value from the 1st iteration
69# to the 2nd one. The proper value in this case is simply the value
70# of S[I] from the first iteration (thanks to the fact that SWP
71# simply swaps the contents of S[I] and S[J]).
72#
73# Another potential trouble-spot is in cycle 7, where SWP of the 1st
74# iteration issues at the same time as the LDI of the 3rd iteration.
75# However, thanks to IA-64 execution semantics, this can be taken
76# care of simply by placing LDI later in the instruction-group than
77# SWP. IA-64 CPUs will automatically forward the value if they
78# detect that the SWP and LDI are accessing the same memory-location.
79
80# The core-loop that can be pipelined then looks like this (annotated
81# with McKinley/Madison issue port & latency numbers, assuming L1
82# cache hits for the most part):
83
84# operation: instruction: issue-ports: latency
85# ------------------ ----------------------------- ------------- -------
86
87# Data = *inp++ ld1 data = [inp], 1 M0-M1 1 cyc c0
88# shladd Iptr = I, KeyTable, 3 M0-M3, I0, I1 1 cyc
89# I = (I + 1) & 0xff padd1 nextI = I, one M0-M3, I0, I1 3 cyc
90# ;;
91# SI = S[I] ld8 SI = [Iptr] M0-M1 1 cyc c1 * after SWAP!
92# ;;
93# cmp.eq.unc pBypass = I, J * after J is valid!
94# J = SI + J add J = J, SI M0-M3, I0, I1 1 cyc c2
95# (pBypass) br.cond.spnt Bypass
96# ;;
97# ---------------------------------------------------------------------------------------
98# J = J & 0xff zxt1 J = J I0, I1, 1 cyc c3
99# ;;
100# shladd Jptr = J, KeyTable, 3 M0-M3, I0, I1 1 cyc c4
101# ;;
102# SJ = S[J] ld8 SJ = [Jptr] M0-M1 1 cyc c5
103# ;;
104# ---------------------------------------------------------------------------------------
105# T = (SI + SJ) add T = SI, SJ M0-M3, I0, I1 1 cyc c6
106# ;;
107# T = T & 0xff zxt1 T = T I0, I1 1 cyc
108# S[I] = SJ st8 [Iptr] = SJ M2-M3 c7
109# S[J] = SI st8 [Jptr] = SI M2-M3
110# ;;
111# shladd Tptr = T, KeyTable, 3 M0-M3, I0, I1 1 cyc c8
112# ;;
113# ---------------------------------------------------------------------------------------
114# T = S[T] ld8 T = [Tptr] M0-M1 1 cyc c9
115# ;;
116# data ^= T xor data = data, T M0-M3, I0, I1 1 cyc c10
117# ;;
118# *out++ = Data ^ T dep word = word, data, 8, POS I0, I1 1 cyc c11
119# ;;
120# ---------------------------------------------------------------------------------------
121
122# There are several points worth making here:
123
124# - Note that due to the bypass/forwarding-path, the first two
125# phases of the loop are strangly mingled together. In
126# particular, note that the first stage of the pipeline is
127# using the value of "J", as calculated by the second stage.
128# - Each bundle-pair will have exactly 6 instructions.
129# - Pipelined, the loop can execute in 3 cycles/iteration and
130# 4 stages. However, McKinley/Madison can issue "st1" to
131# the same bank at a rate of at most one per 4 cycles. Thus,
132# instead of storing each byte, we accumulate them in a word
133# and then write them back at once with a single "st8" (this
134# implies that the setup code needs to ensure that the output
135# buffer is properly aligned, if need be, by encoding the
136# first few bytes separately).
137# - There is no space for a "br.ctop" instruction. For this
138# reason we can't use module-loop support in IA-64 and have
139# to do a traditional, purely software-pipelined loop.
140# - We can't replace any of the remaining "add/zxt1" pairs with
141# "padd1" because the latency for that instruction is too high
142# and would push the loop to the point where more bypasses
143# would be needed, which we don't have space for.
144# - The above loop runs at around 3.26 cycles/byte, or roughly
145# 440 MByte/sec on a 1.5GHz Madison. This is well below the
146# system bus bandwidth and hence with judicious use of
147# "lfetch" this loop can run at (almost) peak speed even when
148# the input and output data reside in memory. The
149# max. latency that can be tolerated is (PREFETCH_DISTANCE *
150# L2_LINE_SIZE * 3 cyc), or about 384 cycles assuming (at
151# least) 1-ahead prefetching of 128 byte cache-lines. Note
152# that we do NOT prefetch into L1, since that would only
153# interfere with the S[] table values stored there. This is
154# acceptable because there is a 10 cycle latency between
155# load and first use of the input data.
156# - We use a branch to out-of-line bypass-code of cycle-pressure:
157# we calculate the next J, check for the need to activate the
158# bypass path, and activate the bypass path ALL IN THE SAME
159# CYCLE. If we didn't have these constraints, we could do
160# the bypass with a simple conditional move instruction.
161# Fortunately, the bypass paths get activated relatively
162# infrequently, so the extra branches don't cost all that much
163# (about 0.04 cycles/byte, measured on a 16396 byte file with
164# random input data).
165#
166
167$phases = 4; # number of stages/phases in the pipelined-loop
168$unroll_count = 6; # number of times we unrolled it
169$pComI = (1 << 0);
170$pComJ = (1 << 1);
171$pComT = (1 << 2);
172$pOut = (1 << 3);
173
174$NData = 4;
175$NIP = 3;
176$NJP = 2;
177$NI = 2;
178$NSI = 3;
179$NSJ = 2;
180$NT = 2;
181$NOutWord = 2;
182
183#
184# $threshold is the minimum length before we attempt to use the
185# big software-pipelined loop. It MUST be greater-or-equal
186# to:
187# PHASES * (UNROLL_COUNT + 1) + 7
188#
189# The "+ 7" comes from the fact we may have to encode up to
190# 7 bytes separately before the output pointer is aligned.
191#
192$threshold = (3 * ($phases * ($unroll_count + 1)) + 7);
193
194sub I {
195 local *code = shift;
196 local $format = shift;
197 $code .= sprintf ("\t\t".$format."\n", @_);
198}
199
200sub P {
201 local *code = shift;
202 local $format = shift;
203 $code .= sprintf ($format."\n", @_);
204}
205
206sub STOP {
207 local *code = shift;
208 $code .=<<___;
209 ;;
210___
211}
212
213sub emit_body {
214 local *c = shift;
215 local *bypass = shift;
216 local ($iteration, $p) = @_;
217
218 local $i0 = $iteration;
219 local $i1 = $iteration - 1;
220 local $i2 = $iteration - 2;
221 local $i3 = $iteration - 3;
222 local $iw0 = ($iteration - 3) / 8;
223 local $iw1 = ($iteration > 3) ? ($iteration - 4) / 8 : 1;
224 local $byte_num = ($iteration - 3) % 8;
225 local $label = $iteration + 1;
226 local $pAny = ($p & 0xf) == 0xf;
227 local $pByp = (($p & $pComI) && ($iteration > 0));
228
229 $c.=<<___;
230//////////////////////////////////////////////////
231___
232
233 if (($p & 0xf) == 0) {
234 $c.="#ifdef HOST_IS_BIG_ENDIAN\n";
235 &I(\$c,"shr.u OutWord[%u] = OutWord[%u], 32;;",
236 $iw1 % $NOutWord, $iw1 % $NOutWord);
237 $c.="#endif\n";
238 &I(\$c, "st4 [OutPtr] = OutWord[%u], 4", $iw1 % $NOutWord);
239 return;
240 }
241
242 # Cycle 0
243 &I(\$c, "{ .mmi") if ($pAny);
244 &I(\$c, "ld1 Data[%u] = [InPtr], 1", $i0 % $NData) if ($p & $pComI);
245 &I(\$c, "padd1 I[%u] = One, I[%u]", $i0 % $NI, $i1 % $NI)if ($p & $pComI);
246 &I(\$c, "zxt1 J = J") if ($p & $pComJ);
247 &I(\$c, "}") if ($pAny);
248 &I(\$c, "{ .mmi") if ($pAny);
249 &I(\$c, "LKEY T[%u] = [T[%u]]", $i1 % $NT, $i1 % $NT) if ($p & $pOut);
250 &I(\$c, "add T[%u] = SI[%u], SJ[%u]",
251 $i0 % $NT, $i2 % $NSI, $i1 % $NSJ) if ($p & $pComT);
252 &I(\$c, "KEYADDR(IPr[%u], I[%u])", $i0 % $NIP, $i1 % $NI) if ($p & $pComI);
253 &I(\$c, "}") if ($pAny);
254 &STOP(\$c);
255
256 # Cycle 1
257 &I(\$c, "{ .mmi") if ($pAny);
258 &I(\$c, "SKEY [IPr[%u]] = SJ[%u]", $i2 % $NIP, $i1%$NSJ)if ($p & $pComT);
259 &I(\$c, "SKEY [JP[%u]] = SI[%u]", $i1 % $NJP, $i2%$NSI) if ($p & $pComT);
260 &I(\$c, "zxt1 T[%u] = T[%u]", $i0 % $NT, $i0 % $NT) if ($p & $pComT);
261 &I(\$c, "}") if ($pAny);
262 &I(\$c, "{ .mmi") if ($pAny);
263 &I(\$c, "LKEY SI[%u] = [IPr[%u]]", $i0 % $NSI, $i0%$NIP)if ($p & $pComI);
264 &I(\$c, "KEYADDR(JP[%u], J)", $i0 % $NJP) if ($p & $pComJ);
265 &I(\$c, "xor Data[%u] = Data[%u], T[%u]",
266 $i3 % $NData, $i3 % $NData, $i1 % $NT) if ($p & $pOut);
267 &I(\$c, "}") if ($pAny);
268 &STOP(\$c);
269
270 # Cycle 2
271 &I(\$c, "{ .mmi") if ($pAny);
272 &I(\$c, "LKEY SJ[%u] = [JP[%u]]", $i0 % $NSJ, $i0%$NJP) if ($p & $pComJ);
273 &I(\$c, "cmp.eq pBypass, p0 = I[%u], J", $i1 % $NI) if ($pByp);
274 &I(\$c, "dep OutWord[%u] = Data[%u], OutWord[%u], BYTE_POS(%u), 8",
275 $iw0%$NOutWord, $i3%$NData, $iw1%$NOutWord, $byte_num) if ($p & $pOut);
276 &I(\$c, "}") if ($pAny);
277 &I(\$c, "{ .mmb") if ($pAny);
278 &I(\$c, "add J = J, SI[%u]", $i0 % $NSI) if ($p & $pComI);
279 &I(\$c, "KEYADDR(T[%u], T[%u])", $i0 % $NT, $i0 % $NT) if ($p & $pComT);
280 &P(\$c, "(pBypass)\tbr.cond.spnt.many .rc4Bypass%u",$label)if ($pByp);
281 &I(\$c, "}") if ($pAny);
282 &STOP(\$c);
283
284 &P(\$c, ".rc4Resume%u:", $label) if ($pByp);
285 if ($byte_num == 0 && $iteration >= $phases) {
286 &I(\$c, "st8 [OutPtr] = OutWord[%u], 8",
287 $iw1 % $NOutWord) if ($p & $pOut);
288 if ($iteration == (1 + $unroll_count) * $phases - 1) {
289 if ($unroll_count == 6) {
290 &I(\$c, "mov OutWord[%u] = OutWord[%u]",
291 $iw1 % $NOutWord, $iw0 % $NOutWord);
292 }
293 &I(\$c, "lfetch.nt1 [InPrefetch], %u",
294 $unroll_count * $phases);
295 &I(\$c, "lfetch.excl.nt1 [OutPrefetch], %u",
296 $unroll_count * $phases);
297 &I(\$c, "br.cloop.sptk.few .rc4Loop");
298 }
299 }
300
301 if ($pByp) {
302 &P(\$bypass, ".rc4Bypass%u:", $label);
303 &I(\$bypass, "sub J = J, SI[%u]", $i0 % $NSI);
304 &I(\$bypass, "nop 0");
305 &I(\$bypass, "nop 0");
306 &I(\$bypass, ";;");
307 &I(\$bypass, "add J = J, SI[%u]", $i1 % $NSI);
308 &I(\$bypass, "mov SI[%u] = SI[%u]", $i0 % $NSI, $i1 % $NSI);
309 &I(\$bypass, "br.sptk.many .rc4Resume%u\n", $label);
310 &I(\$bypass, ";;");
311 }
312}
313
314$code=<<___;
315.ident \"rc4-ia64.s, version 3.0\"
316.ident \"Copyright (c) 2005 Hewlett-Packard Development Company, L.P.\"
317
318#define LCSave r8
319#define PRSave r9
320
321/* Inputs become invalid once rotation begins! */
322
323#define StateTable in0
324#define DataLen in1
325#define InputBuffer in2
326#define OutputBuffer in3
327
328#define KTable r14
329#define J r15
330#define InPtr r16
331#define OutPtr r17
332#define InPrefetch r18
333#define OutPrefetch r19
334#define One r20
335#define LoopCount r21
336#define Remainder r22
337#define IFinal r23
338#define EndPtr r24
339
340#define tmp0 r25
341#define tmp1 r26
342
343#define pBypass p6
344#define pDone p7
345#define pSmall p8
346#define pAligned p9
347#define pUnaligned p10
348
349#define pComputeI pPhase[0]
350#define pComputeJ pPhase[1]
351#define pComputeT pPhase[2]
352#define pOutput pPhase[3]
353
354#define RetVal r8
355#define L_OK p7
356#define L_NOK p8
357
358#define _NINPUTS 4
359#define _NOUTPUT 0
360
361#define _NROTATE 24
362#define _NLOCALS (_NROTATE - _NINPUTS - _NOUTPUT)
363
364#ifndef SZ
365# define SZ 4 // this must be set to sizeof(RC4_INT)
366#endif
367
368#if SZ == 1
369# define LKEY ld1
370# define SKEY st1
371# define KEYADDR(dst, i) add dst = i, KTable
372#elif SZ == 2
373# define LKEY ld2
374# define SKEY st2
375# define KEYADDR(dst, i) shladd dst = i, 1, KTable
376#elif SZ == 4
377# define LKEY ld4
378# define SKEY st4
379# define KEYADDR(dst, i) shladd dst = i, 2, KTable
380#else
381# define LKEY ld8
382# define SKEY st8
383# define KEYADDR(dst, i) shladd dst = i, 3, KTable
384#endif
385
386#if defined(_HPUX_SOURCE) && !defined(_LP64)
387# define ADDP addp4
388#else
389# define ADDP add
390#endif
391
392/* Define a macro for the bit number of the n-th byte: */
393
394#if defined(_HPUX_SOURCE) || defined(B_ENDIAN)
395# define HOST_IS_BIG_ENDIAN
396# define BYTE_POS(n) (56 - (8 * (n)))
397#else
398# define BYTE_POS(n) (8 * (n))
399#endif
400
401/*
402 We must perform the first phase of the pipeline explicitly since
403 we will always load from the stable the first time. The br.cexit
404 will never be taken since regardless of the number of bytes because
405 the epilogue count is 4.
406*/
407/* MODSCHED_RC4 macro was split to _PROLOGUE and _LOOP, because HP-UX
408 assembler failed on original macro with syntax error. <appro> */
409#define MODSCHED_RC4_PROLOGUE \\
410 { \\
411 ld1 Data[0] = [InPtr], 1; \\
412 add IFinal = 1, I[1]; \\
413 KEYADDR(IPr[0], I[1]); \\
414 } ;; \\
415 { \\
416 LKEY SI[0] = [IPr[0]]; \\
417 mov pr.rot = 0x10000; \\
418 mov ar.ec = 4; \\
419 } ;; \\
420 { \\
421 add J = J, SI[0]; \\
422 zxt1 I[0] = IFinal; \\
423 br.cexit.spnt.few .+16; /* never taken */ \\
424 } ;;
425#define MODSCHED_RC4_LOOP(label) \\
426label: \\
427 { .mmi; \\
428 (pComputeI) ld1 Data[0] = [InPtr], 1; \\
429 (pComputeI) add IFinal = 1, I[1]; \\
430 (pComputeJ) zxt1 J = J; \\
431 }{ .mmi; \\
432 (pOutput) LKEY T[1] = [T[1]]; \\
433 (pComputeT) add T[0] = SI[2], SJ[1]; \\
434 (pComputeI) KEYADDR(IPr[0], I[1]); \\
435 } ;; \\
436 { .mmi; \\
437 (pComputeT) SKEY [IPr[2]] = SJ[1]; \\
438 (pComputeT) SKEY [JP[1]] = SI[2]; \\
439 (pComputeT) zxt1 T[0] = T[0]; \\
440 }{ .mmi; \\
441 (pComputeI) LKEY SI[0] = [IPr[0]]; \\
442 (pComputeJ) KEYADDR(JP[0], J); \\
443 (pComputeI) cmp.eq.unc pBypass, p0 = I[1], J; \\
444 } ;; \\
445 { .mmi; \\
446 (pComputeJ) LKEY SJ[0] = [JP[0]]; \\
447 (pOutput) xor Data[3] = Data[3], T[1]; \\
448 nop 0x0; \\
449 }{ .mmi; \\
450 (pComputeT) KEYADDR(T[0], T[0]); \\
451 (pBypass) mov SI[0] = SI[1]; \\
452 (pComputeI) zxt1 I[0] = IFinal; \\
453 } ;; \\
454 { .mmb; \\
455 (pOutput) st1 [OutPtr] = Data[3], 1; \\
456 (pComputeI) add J = J, SI[0]; \\
457 br.ctop.sptk.few label; \\
458 } ;;
459
460 .text
461
462 .align 32
463
464 .type RC4, \@function
465 .global RC4
466
467 .proc RC4
468 .prologue
469
470RC4:
471 {
472 .mmi
473 alloc r2 = ar.pfs, _NINPUTS, _NLOCALS, _NOUTPUT, _NROTATE
474
475 .rotr Data[4], I[2], IPr[3], SI[3], JP[2], SJ[2], T[2], \\
476 OutWord[2]
477 .rotp pPhase[4]
478
479 ADDP InPrefetch = 0, InputBuffer
480 ADDP KTable = 0, StateTable
481 }
482 {
483 .mmi
484 ADDP InPtr = 0, InputBuffer
485 ADDP OutPtr = 0, OutputBuffer
486 mov RetVal = r0
487 }
488 ;;
489 {
490 .mmi
491 lfetch.nt1 [InPrefetch], 0x80
492 ADDP OutPrefetch = 0, OutputBuffer
493 }
494 { // Return 0 if the input length is nonsensical
495 .mib
496 ADDP StateTable = 0, StateTable
497 cmp.ge.unc L_NOK, L_OK = r0, DataLen
498 (L_NOK) br.ret.sptk.few rp
499 }
500 ;;
501 {
502 .mib
503 cmp.eq.or L_NOK, L_OK = r0, InPtr
504 cmp.eq.or L_NOK, L_OK = r0, OutPtr
505 nop 0x0
506 }
507 {
508 .mib
509 cmp.eq.or L_NOK, L_OK = r0, StateTable
510 nop 0x0
511 (L_NOK) br.ret.sptk.few rp
512 }
513 ;;
514 LKEY I[1] = [KTable], SZ
515/* Prefetch the state-table. It contains 256 elements of size SZ */
516
517#if SZ == 1
518 ADDP tmp0 = 1*128, StateTable
519#elif SZ == 2
520 ADDP tmp0 = 3*128, StateTable
521 ADDP tmp1 = 2*128, StateTable
522#elif SZ == 4
523 ADDP tmp0 = 7*128, StateTable
524 ADDP tmp1 = 6*128, StateTable
525#elif SZ == 8
526 ADDP tmp0 = 15*128, StateTable
527 ADDP tmp1 = 14*128, StateTable
528#endif
529 ;;
530#if SZ >= 8
531 lfetch.fault.nt1 [tmp0], -256 // 15
532 lfetch.fault.nt1 [tmp1], -256;;
533 lfetch.fault.nt1 [tmp0], -256 // 13
534 lfetch.fault.nt1 [tmp1], -256;;
535 lfetch.fault.nt1 [tmp0], -256 // 11
536 lfetch.fault.nt1 [tmp1], -256;;
537 lfetch.fault.nt1 [tmp0], -256 // 9
538 lfetch.fault.nt1 [tmp1], -256;;
539#endif
540#if SZ >= 4
541 lfetch.fault.nt1 [tmp0], -256 // 7
542 lfetch.fault.nt1 [tmp1], -256;;
543 lfetch.fault.nt1 [tmp0], -256 // 5
544 lfetch.fault.nt1 [tmp1], -256;;
545#endif
546#if SZ >= 2
547 lfetch.fault.nt1 [tmp0], -256 // 3
548 lfetch.fault.nt1 [tmp1], -256;;
549#endif
550 {
551 .mii
552 lfetch.fault.nt1 [tmp0] // 1
553 add I[1]=1,I[1];;
554 zxt1 I[1]=I[1]
555 }
556 {
557 .mmi
558 lfetch.nt1 [InPrefetch], 0x80
559 lfetch.excl.nt1 [OutPrefetch], 0x80
560 .save pr, PRSave
561 mov PRSave = pr
562 } ;;
563 {
564 .mmi
565 lfetch.excl.nt1 [OutPrefetch], 0x80
566 LKEY J = [KTable], SZ
567 ADDP EndPtr = DataLen, InPtr
568 } ;;
569 {
570 .mmi
571 ADDP EndPtr = -1, EndPtr // Make it point to
572 // last data byte.
573 mov One = 1
574 .save ar.lc, LCSave
575 mov LCSave = ar.lc
576 .body
577 } ;;
578 {
579 .mmb
580 sub Remainder = 0, OutPtr
581 cmp.gtu pSmall, p0 = $threshold, DataLen
582(pSmall) br.cond.dpnt .rc4Remainder // Data too small for
583 // big loop.
584 } ;;
585 {
586 .mmi
587 and Remainder = 0x7, Remainder
588 ;;
589 cmp.eq pAligned, pUnaligned = Remainder, r0
590 nop 0x0
591 } ;;
592 {
593 .mmb
594.pred.rel "mutex",pUnaligned,pAligned
595(pUnaligned) add Remainder = -1, Remainder
596(pAligned) sub Remainder = EndPtr, InPtr
597(pAligned) br.cond.dptk.many .rc4Aligned
598 } ;;
599 {
600 .mmi
601 nop 0x0
602 nop 0x0
603 mov.i ar.lc = Remainder
604 }
605
606/* Do the initial few bytes via the compact, modulo-scheduled loop
607 until the output pointer is 8-byte-aligned. */
608
609 MODSCHED_RC4_PROLOGUE
610 MODSCHED_RC4_LOOP(.RC4AlignLoop)
611
612 {
613 .mib
614 sub Remainder = EndPtr, InPtr
615 zxt1 IFinal = IFinal
616 clrrrb // Clear CFM.rrb.pr so
617 ;; // next "mov pr.rot = N"
618 // does the right thing.
619 }
620 {
621 .mmi
622 mov I[1] = IFinal
623 nop 0x0
624 nop 0x0
625 } ;;
626
627
628.rc4Aligned:
629
630/*
631 Unrolled loop count = (Remainder - ($unroll_count+1)*$phases)/($unroll_count*$phases)
632 */
633
634 {
635 .mlx
636 add LoopCount = 1 - ($unroll_count + 1)*$phases, Remainder
637 movl Remainder = 0xaaaaaaaaaaaaaaab
638 } ;;
639 {
640 .mmi
641 setf.sig f6 = LoopCount // M2, M3 6 cyc
642 setf.sig f7 = Remainder // M2, M3 6 cyc
643 nop 0x0
644 } ;;
645 {
646 .mfb
647 nop 0x0
648 xmpy.hu f6 = f6, f7
649 nop 0x0
650 } ;;
651 {
652 .mmi
653 getf.sig LoopCount = f6;; // M2 5 cyc
654 nop 0x0
655 shr.u LoopCount = LoopCount, 4
656 } ;;
657 {
658 .mmi
659 nop 0x0
660 nop 0x0
661 mov.i ar.lc = LoopCount
662 } ;;
663
664/* Now comes the unrolled loop: */
665
666.rc4Prologue:
667___
668
669$iteration = 0;
670
671# Generate the prologue:
672$predicates = 1;
673for ($i = 0; $i < $phases; ++$i) {
674 &emit_body (\$code, \$bypass, $iteration++, $predicates);
675 $predicates = ($predicates << 1) | 1;
676}
677
678$code.=<<___;
679.rc4Loop:
680___
681
682# Generate the body:
683for ($i = 0; $i < $unroll_count*$phases; ++$i) {
684 &emit_body (\$code, \$bypass, $iteration++, $predicates);
685}
686
687$code.=<<___;
688.rc4Epilogue:
689___
690
691# Generate the epilogue:
692for ($i = 0; $i < $phases; ++$i) {
693 $predicates <<= 1;
694 &emit_body (\$code, \$bypass, $iteration++, $predicates);
695}
696
697$code.=<<___;
698 {
699 .mmi
700 lfetch.nt1 [EndPtr] // fetch line with last byte
701 mov IFinal = I[1]
702 nop 0x0
703 }
704
705.rc4Remainder:
706 {
707 .mmi
708 sub Remainder = EndPtr, InPtr // Calculate
709 // # of bytes
710 // left - 1
711 nop 0x0
712 nop 0x0
713 } ;;
714 {
715 .mib
716 cmp.eq pDone, p0 = -1, Remainder // done already?
717 mov.i ar.lc = Remainder
718(pDone) br.cond.dptk.few .rc4Complete
719 }
720
721/* Do the remaining bytes via the compact, modulo-scheduled loop */
722
723 MODSCHED_RC4_PROLOGUE
724 MODSCHED_RC4_LOOP(.RC4RestLoop)
725
726.rc4Complete:
727 {
728 .mmi
729 add KTable = -SZ, KTable
730 add IFinal = -1, IFinal
731 mov ar.lc = LCSave
732 } ;;
733 {
734 .mii
735 SKEY [KTable] = J,-SZ
736 zxt1 IFinal = IFinal
737 mov pr = PRSave, 0x1FFFF
738 } ;;
739 {
740 .mib
741 SKEY [KTable] = IFinal
742 add RetVal = 1, r0
743 br.ret.sptk.few rp
744 } ;;
745___
746
747# Last but not least, emit the code for the bypass-code of the unrolled loop:
748
749$code.=$bypass;
750
751$code.=<<___;
752 .endp RC4
753___
754
755print $code;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl
new file mode 100644
index 0000000000..7f684092d4
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-md5-x86_64.pl
@@ -0,0 +1,631 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# June 2011
11#
12# This is RC4+MD5 "stitch" implementation. The idea, as spelled in
13# http://download.intel.com/design/intarch/papers/323686.pdf, is that
14# since both algorithms exhibit instruction-level parallelism, ILP,
15# below theoretical maximum, interleaving them would allow to utilize
16# processor resources better and achieve better performance. RC4
17# instruction sequence is virtually identical to rc4-x86_64.pl, which
18# is heavily based on submission by Maxim Perminov, Maxim Locktyukhin
19# and Jim Guilford of Intel. MD5 is fresh implementation aiming to
20# minimize register usage, which was used as "main thread" with RC4
21# weaved into it, one RC4 round per one MD5 round. In addition to the
22# stiched subroutine the script can generate standalone replacement
23# md5_block_asm_data_order and RC4. Below are performance numbers in
24# cycles per processed byte, less is better, for these the standalone
25# subroutines, sum of them, and stitched one:
26#
27# RC4 MD5 RC4+MD5 stitch gain
28# Opteron 6.5(*) 5.4 11.9 7.0 +70%(*)
29# Core2 6.5 5.8 12.3 7.7 +60%
30# Westmere 4.3 5.2 9.5 7.0 +36%
31# Sandy Bridge 4.2 5.5 9.7 6.8 +43%
32# Atom 9.3 6.5 15.8 11.1 +42%
33#
34# (*) rc4-x86_64.pl delivers 5.3 on Opteron, so real improvement
35# is +53%...
36
37my ($rc4,$md5)=(1,1); # what to generate?
38my $D="#" if (!$md5); # if set to "#", MD5 is stitched into RC4(),
39 # but its result is discarded. Idea here is
40 # to be able to use 'openssl speed rc4' for
41 # benchmarking the stitched subroutine...
42
43my $flavour = shift;
44my $output = shift;
45if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
46
47my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
48
49$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; my $xlate;
50( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
51( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
52die "can't locate x86_64-xlate.pl";
53
54open STDOUT,"| $^X $xlate $flavour $output";
55
56my ($dat,$in0,$out,$ctx,$inp,$len, $func,$nargs);
57
58if ($rc4 && !$md5) {
59 ($dat,$len,$in0,$out) = ("%rdi","%rsi","%rdx","%rcx");
60 $func="RC4"; $nargs=4;
61} elsif ($md5 && !$rc4) {
62 ($ctx,$inp,$len) = ("%rdi","%rsi","%rdx");
63 $func="md5_block_asm_data_order"; $nargs=3;
64} else {
65 ($dat,$in0,$out,$ctx,$inp,$len) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
66 $func="rc4_md5_enc"; $nargs=6;
67 # void rc4_md5_enc(
68 # RC4_KEY *key, #
69 # const void *in0, # RC4 input
70 # void *out, # RC4 output
71 # MD5_CTX *ctx, #
72 # const void *inp, # MD5 input
73 # size_t len); # number of 64-byte blocks
74}
75
76my @K=( 0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee,
77 0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501,
78 0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be,
79 0x6b901122,0xfd987193,0xa679438e,0x49b40821,
80
81 0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa,
82 0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8,
83 0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed,
84 0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a,
85
86 0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c,
87 0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70,
88 0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05,
89 0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665,
90
91 0xf4292244,0x432aff97,0xab9423a7,0xfc93a039,
92 0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1,
93 0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1,
94 0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391 );
95
96my @V=("%r8d","%r9d","%r10d","%r11d"); # MD5 registers
97my $tmp="%r12d";
98
99my @XX=("%rbp","%rsi"); # RC4 registers
100my @TX=("%rax","%rbx");
101my $YY="%rcx";
102my $TY="%rdx";
103
104my $MOD=32; # 16, 32 or 64
105
106$code.=<<___;
107.text
108.align 16
109
110.globl $func
111.type $func,\@function,$nargs
112$func:
113 cmp \$0,$len
114 je .Labort
115 push %rbx
116 push %rbp
117 push %r12
118 push %r13
119 push %r14
120 push %r15
121 sub \$40,%rsp
122.Lbody:
123___
124if ($rc4) {
125$code.=<<___;
126$D#md5# mov $ctx,%r11 # reassign arguments
127 mov $len,%r12
128 mov $in0,%r13
129 mov $out,%r14
130$D#md5# mov $inp,%r15
131___
132 $ctx="%r11" if ($md5); # reassign arguments
133 $len="%r12";
134 $in0="%r13";
135 $out="%r14";
136 $inp="%r15" if ($md5);
137 $inp=$in0 if (!$md5);
138$code.=<<___;
139 xor $XX[0],$XX[0]
140 xor $YY,$YY
141
142 lea 8($dat),$dat
143 mov -8($dat),$XX[0]#b
144 mov -4($dat),$YY#b
145
146 inc $XX[0]#b
147 sub $in0,$out
148 movl ($dat,$XX[0],4),$TX[0]#d
149___
150$code.=<<___ if (!$md5);
151 xor $TX[1],$TX[1]
152 test \$-128,$len
153 jz .Loop1
154 sub $XX[0],$TX[1]
155 and \$`$MOD-1`,$TX[1]
156 jz .Loop${MOD}_is_hot
157 sub $TX[1],$len
158.Loop${MOD}_warmup:
159 add $TX[0]#b,$YY#b
160 movl ($dat,$YY,4),$TY#d
161 movl $TX[0]#d,($dat,$YY,4)
162 movl $TY#d,($dat,$XX[0],4)
163 add $TY#b,$TX[0]#b
164 inc $XX[0]#b
165 movl ($dat,$TX[0],4),$TY#d
166 movl ($dat,$XX[0],4),$TX[0]#d
167 xorb ($in0),$TY#b
168 movb $TY#b,($out,$in0)
169 lea 1($in0),$in0
170 dec $TX[1]
171 jnz .Loop${MOD}_warmup
172
173 mov $YY,$TX[1]
174 xor $YY,$YY
175 mov $TX[1]#b,$YY#b
176
177.Loop${MOD}_is_hot:
178 mov $len,32(%rsp) # save original $len
179 shr \$6,$len # number of 64-byte blocks
180___
181 if ($D && !$md5) { # stitch in dummy MD5
182 $md5=1;
183 $ctx="%r11";
184 $inp="%r15";
185 $code.=<<___;
186 mov %rsp,$ctx
187 mov $in0,$inp
188___
189 }
190}
191$code.=<<___;
192#rc4# add $TX[0]#b,$YY#b
193#rc4# lea ($dat,$XX[0],4),$XX[1]
194 shl \$6,$len
195 add $inp,$len # pointer to the end of input
196 mov $len,16(%rsp)
197
198#md5# mov $ctx,24(%rsp) # save pointer to MD5_CTX
199#md5# mov 0*4($ctx),$V[0] # load current hash value from MD5_CTX
200#md5# mov 1*4($ctx),$V[1]
201#md5# mov 2*4($ctx),$V[2]
202#md5# mov 3*4($ctx),$V[3]
203 jmp .Loop
204
205.align 16
206.Loop:
207#md5# mov $V[0],0*4(%rsp) # put aside current hash value
208#md5# mov $V[1],1*4(%rsp)
209#md5# mov $V[2],2*4(%rsp)
210#md5# mov $V[3],$tmp # forward reference
211#md5# mov $V[3],3*4(%rsp)
212___
213
214sub R0 {
215 my ($i,$a,$b,$c,$d)=@_;
216 my @rot0=(7,12,17,22);
217 my $j=$i%16;
218 my $k=$i%$MOD;
219 my $xmm="%xmm".($j&1);
220 $code.=" movdqu ($in0),%xmm2\n" if ($rc4 && $j==15);
221 $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1);
222 $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1);
223 $code.=<<___;
224#rc4# movl ($dat,$YY,4),$TY#d
225#md5# xor $c,$tmp
226#rc4# movl $TX[0]#d,($dat,$YY,4)
227#md5# and $b,$tmp
228#md5# add 4*`$j`($inp),$a
229#rc4# add $TY#b,$TX[0]#b
230#rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d
231#md5# add \$$K[$i],$a
232#md5# xor $d,$tmp
233#rc4# movz $TX[0]#b,$TX[0]#d
234#rc4# movl $TY#d,4*$k($XX[1])
235#md5# add $tmp,$a
236#rc4# add $TX[1]#b,$YY#b
237#md5# rol \$$rot0[$j%4],$a
238#md5# mov `$j==15?"$b":"$c"`,$tmp # forward reference
239#rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n
240#md5# add $b,$a
241___
242 $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1);
243 mov $YY,$XX[1]
244 xor $YY,$YY # keyword to partial register
245 mov $XX[1]#b,$YY#b
246 lea ($dat,$XX[0],4),$XX[1]
247___
248 $code.=<<___ if ($rc4 && $j==15);
249 psllq \$8,%xmm1
250 pxor %xmm0,%xmm2
251 pxor %xmm1,%xmm2
252___
253}
254sub R1 {
255 my ($i,$a,$b,$c,$d)=@_;
256 my @rot1=(5,9,14,20);
257 my $j=$i%16;
258 my $k=$i%$MOD;
259 my $xmm="%xmm".($j&1);
260 $code.=" movdqu 16($in0),%xmm3\n" if ($rc4 && $j==15);
261 $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1);
262 $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1);
263 $code.=<<___;
264#rc4# movl ($dat,$YY,4),$TY#d
265#md5# xor $b,$tmp
266#rc4# movl $TX[0]#d,($dat,$YY,4)
267#md5# and $d,$tmp
268#md5# add 4*`((1+5*$j)%16)`($inp),$a
269#rc4# add $TY#b,$TX[0]#b
270#rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d
271#md5# add \$$K[$i],$a
272#md5# xor $c,$tmp
273#rc4# movz $TX[0]#b,$TX[0]#d
274#rc4# movl $TY#d,4*$k($XX[1])
275#md5# add $tmp,$a
276#rc4# add $TX[1]#b,$YY#b
277#md5# rol \$$rot1[$j%4],$a
278#md5# mov `$j==15?"$c":"$b"`,$tmp # forward reference
279#rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n
280#md5# add $b,$a
281___
282 $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1);
283 mov $YY,$XX[1]
284 xor $YY,$YY # keyword to partial register
285 mov $XX[1]#b,$YY#b
286 lea ($dat,$XX[0],4),$XX[1]
287___
288 $code.=<<___ if ($rc4 && $j==15);
289 psllq \$8,%xmm1
290 pxor %xmm0,%xmm3
291 pxor %xmm1,%xmm3
292___
293}
294sub R2 {
295 my ($i,$a,$b,$c,$d)=@_;
296 my @rot2=(4,11,16,23);
297 my $j=$i%16;
298 my $k=$i%$MOD;
299 my $xmm="%xmm".($j&1);
300 $code.=" movdqu 32($in0),%xmm4\n" if ($rc4 && $j==15);
301 $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1);
302 $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1);
303 $code.=<<___;
304#rc4# movl ($dat,$YY,4),$TY#d
305#md5# xor $c,$tmp
306#rc4# movl $TX[0]#d,($dat,$YY,4)
307#md5# xor $b,$tmp
308#md5# add 4*`((5+3*$j)%16)`($inp),$a
309#rc4# add $TY#b,$TX[0]#b
310#rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d
311#md5# add \$$K[$i],$a
312#rc4# movz $TX[0]#b,$TX[0]#d
313#md5# add $tmp,$a
314#rc4# movl $TY#d,4*$k($XX[1])
315#rc4# add $TX[1]#b,$YY#b
316#md5# rol \$$rot2[$j%4],$a
317#md5# mov `$j==15?"\\\$-1":"$c"`,$tmp # forward reference
318#rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n
319#md5# add $b,$a
320___
321 $code.=<<___ if ($rc4 && $j==15 && $k==$MOD-1);
322 mov $YY,$XX[1]
323 xor $YY,$YY # keyword to partial register
324 mov $XX[1]#b,$YY#b
325 lea ($dat,$XX[0],4),$XX[1]
326___
327 $code.=<<___ if ($rc4 && $j==15);
328 psllq \$8,%xmm1
329 pxor %xmm0,%xmm4
330 pxor %xmm1,%xmm4
331___
332}
333sub R3 {
334 my ($i,$a,$b,$c,$d)=@_;
335 my @rot3=(6,10,15,21);
336 my $j=$i%16;
337 my $k=$i%$MOD;
338 my $xmm="%xmm".($j&1);
339 $code.=" movdqu 48($in0),%xmm5\n" if ($rc4 && $j==15);
340 $code.=" add \$$MOD,$XX[0]#b\n" if ($rc4 && $j==15 && $k==$MOD-1);
341 $code.=" pxor $xmm,$xmm\n" if ($rc4 && $j<=1);
342 $code.=<<___;
343#rc4# movl ($dat,$YY,4),$TY#d
344#md5# xor $d,$tmp
345#rc4# movl $TX[0]#d,($dat,$YY,4)
346#md5# or $b,$tmp
347#md5# add 4*`((7*$j)%16)`($inp),$a
348#rc4# add $TY#b,$TX[0]#b
349#rc4# movl `4*(($k+1)%$MOD)`(`$k==$MOD-1?"$dat,$XX[0],4":"$XX[1]"`),$TX[1]#d
350#md5# add \$$K[$i],$a
351#rc4# movz $TX[0]#b,$TX[0]#d
352#md5# xor $c,$tmp
353#rc4# movl $TY#d,4*$k($XX[1])
354#md5# add $tmp,$a
355#rc4# add $TX[1]#b,$YY#b
356#md5# rol \$$rot3[$j%4],$a
357#md5# mov \$-1,$tmp # forward reference
358#rc4# pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n
359#md5# add $b,$a
360___
361 $code.=<<___ if ($rc4 && $j==15);
362 mov $XX[0],$XX[1]
363 xor $XX[0],$XX[0] # keyword to partial register
364 mov $XX[1]#b,$XX[0]#b
365 mov $YY,$XX[1]
366 xor $YY,$YY # keyword to partial register
367 mov $XX[1]#b,$YY#b
368 lea ($dat,$XX[0],4),$XX[1]
369 psllq \$8,%xmm1
370 pxor %xmm0,%xmm5
371 pxor %xmm1,%xmm5
372___
373}
374
375my $i=0;
376for(;$i<16;$i++) { R0($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); }
377for(;$i<32;$i++) { R1($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); }
378for(;$i<48;$i++) { R2($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); }
379for(;$i<64;$i++) { R3($i,@V); unshift(@V,pop(@V)); push(@TX,shift(@TX)); }
380
381$code.=<<___;
382#md5# add 0*4(%rsp),$V[0] # accumulate hash value
383#md5# add 1*4(%rsp),$V[1]
384#md5# add 2*4(%rsp),$V[2]
385#md5# add 3*4(%rsp),$V[3]
386
387#rc4# movdqu %xmm2,($out,$in0) # write RC4 output
388#rc4# movdqu %xmm3,16($out,$in0)
389#rc4# movdqu %xmm4,32($out,$in0)
390#rc4# movdqu %xmm5,48($out,$in0)
391#md5# lea 64($inp),$inp
392#rc4# lea 64($in0),$in0
393 cmp 16(%rsp),$inp # are we done?
394 jb .Loop
395
396#md5# mov 24(%rsp),$len # restore pointer to MD5_CTX
397#rc4# sub $TX[0]#b,$YY#b # correct $YY
398#md5# mov $V[0],0*4($len) # write MD5_CTX
399#md5# mov $V[1],1*4($len)
400#md5# mov $V[2],2*4($len)
401#md5# mov $V[3],3*4($len)
402___
403$code.=<<___ if ($rc4 && (!$md5 || $D));
404 mov 32(%rsp),$len # restore original $len
405 and \$63,$len # remaining bytes
406 jnz .Loop1
407 jmp .Ldone
408
409.align 16
410.Loop1:
411 add $TX[0]#b,$YY#b
412 movl ($dat,$YY,4),$TY#d
413 movl $TX[0]#d,($dat,$YY,4)
414 movl $TY#d,($dat,$XX[0],4)
415 add $TY#b,$TX[0]#b
416 inc $XX[0]#b
417 movl ($dat,$TX[0],4),$TY#d
418 movl ($dat,$XX[0],4),$TX[0]#d
419 xorb ($in0),$TY#b
420 movb $TY#b,($out,$in0)
421 lea 1($in0),$in0
422 dec $len
423 jnz .Loop1
424
425.Ldone:
426___
427$code.=<<___;
428#rc4# sub \$1,$XX[0]#b
429#rc4# movl $XX[0]#d,-8($dat)
430#rc4# movl $YY#d,-4($dat)
431
432 mov 40(%rsp),%r15
433 mov 48(%rsp),%r14
434 mov 56(%rsp),%r13
435 mov 64(%rsp),%r12
436 mov 72(%rsp),%rbp
437 mov 80(%rsp),%rbx
438 lea 88(%rsp),%rsp
439.Lepilogue:
440.Labort:
441 ret
442.size $func,.-$func
443___
444
445if ($rc4 && $D) { # sole purpose of this section is to provide
446 # option to use the generated module as drop-in
447 # replacement for rc4-x86_64.pl for debugging
448 # and testing purposes...
449my ($idx,$ido)=("%r8","%r9");
450my ($dat,$len,$inp)=("%rdi","%rsi","%rdx");
451
452$code.=<<___;
453.globl RC4_set_key
454.type RC4_set_key,\@function,3
455.align 16
456RC4_set_key:
457 lea 8($dat),$dat
458 lea ($inp,$len),$inp
459 neg $len
460 mov $len,%rcx
461 xor %eax,%eax
462 xor $ido,$ido
463 xor %r10,%r10
464 xor %r11,%r11
465 jmp .Lw1stloop
466
467.align 16
468.Lw1stloop:
469 mov %eax,($dat,%rax,4)
470 add \$1,%al
471 jnc .Lw1stloop
472
473 xor $ido,$ido
474 xor $idx,$idx
475.align 16
476.Lw2ndloop:
477 mov ($dat,$ido,4),%r10d
478 add ($inp,$len,1),$idx#b
479 add %r10b,$idx#b
480 add \$1,$len
481 mov ($dat,$idx,4),%r11d
482 cmovz %rcx,$len
483 mov %r10d,($dat,$idx,4)
484 mov %r11d,($dat,$ido,4)
485 add \$1,$ido#b
486 jnc .Lw2ndloop
487
488 xor %eax,%eax
489 mov %eax,-8($dat)
490 mov %eax,-4($dat)
491 ret
492.size RC4_set_key,.-RC4_set_key
493
494.globl RC4_options
495.type RC4_options,\@abi-omnipotent
496.align 16
497RC4_options:
498 lea .Lopts(%rip),%rax
499 ret
500.align 64
501.Lopts:
502.asciz "rc4(64x,int)"
503.align 64
504.size RC4_options,.-RC4_options
505___
506}
507# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
508# CONTEXT *context,DISPATCHER_CONTEXT *disp)
509if ($win64) {
510my $rec="%rcx";
511my $frame="%rdx";
512my $context="%r8";
513my $disp="%r9";
514
515$code.=<<___;
516.extern __imp_RtlVirtualUnwind
517.type se_handler,\@abi-omnipotent
518.align 16
519se_handler:
520 push %rsi
521 push %rdi
522 push %rbx
523 push %rbp
524 push %r12
525 push %r13
526 push %r14
527 push %r15
528 pushfq
529 sub \$64,%rsp
530
531 mov 120($context),%rax # pull context->Rax
532 mov 248($context),%rbx # pull context->Rip
533
534 lea .Lbody(%rip),%r10
535 cmp %r10,%rbx # context->Rip<.Lbody
536 jb .Lin_prologue
537
538 mov 152($context),%rax # pull context->Rsp
539
540 lea .Lepilogue(%rip),%r10
541 cmp %r10,%rbx # context->Rip>=.Lepilogue
542 jae .Lin_prologue
543
544 mov 40(%rax),%r15
545 mov 48(%rax),%r14
546 mov 56(%rax),%r13
547 mov 64(%rax),%r12
548 mov 72(%rax),%rbp
549 mov 80(%rax),%rbx
550 lea 88(%rax),%rax
551
552 mov %rbx,144($context) # restore context->Rbx
553 mov %rbp,160($context) # restore context->Rbp
554 mov %r12,216($context) # restore context->R12
555 mov %r13,224($context) # restore context->R12
556 mov %r14,232($context) # restore context->R14
557 mov %r15,240($context) # restore context->R15
558
559.Lin_prologue:
560 mov 8(%rax),%rdi
561 mov 16(%rax),%rsi
562 mov %rax,152($context) # restore context->Rsp
563 mov %rsi,168($context) # restore context->Rsi
564 mov %rdi,176($context) # restore context->Rdi
565
566 mov 40($disp),%rdi # disp->ContextRecord
567 mov $context,%rsi # context
568 mov \$154,%ecx # sizeof(CONTEXT)
569 .long 0xa548f3fc # cld; rep movsq
570
571 mov $disp,%rsi
572 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
573 mov 8(%rsi),%rdx # arg2, disp->ImageBase
574 mov 0(%rsi),%r8 # arg3, disp->ControlPc
575 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
576 mov 40(%rsi),%r10 # disp->ContextRecord
577 lea 56(%rsi),%r11 # &disp->HandlerData
578 lea 24(%rsi),%r12 # &disp->EstablisherFrame
579 mov %r10,32(%rsp) # arg5
580 mov %r11,40(%rsp) # arg6
581 mov %r12,48(%rsp) # arg7
582 mov %rcx,56(%rsp) # arg8, (NULL)
583 call *__imp_RtlVirtualUnwind(%rip)
584
585 mov \$1,%eax # ExceptionContinueSearch
586 add \$64,%rsp
587 popfq
588 pop %r15
589 pop %r14
590 pop %r13
591 pop %r12
592 pop %rbp
593 pop %rbx
594 pop %rdi
595 pop %rsi
596 ret
597.size se_handler,.-se_handler
598
599.section .pdata
600.align 4
601 .rva .LSEH_begin_$func
602 .rva .LSEH_end_$func
603 .rva .LSEH_info_$func
604
605.section .xdata
606.align 8
607.LSEH_info_$func:
608 .byte 9,0,0,0
609 .rva se_handler
610___
611}
612
613sub reg_part {
614my ($reg,$conv)=@_;
615 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
616 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
617 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
618 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
619 return $reg;
620}
621
622$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
623$code =~ s/\`([^\`]*)\`/eval $1/gem;
624$code =~ s/pinsrw\s+\$0,/movd /gm;
625
626$code =~ s/#md5#//gm if ($md5);
627$code =~ s/#rc4#//gm if ($rc4);
628
629print $code;
630
631close STDOUT;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-parisc.pl b/src/lib/libcrypto/rc4/asm/rc4-parisc.pl
new file mode 100644
index 0000000000..9165067080
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-parisc.pl
@@ -0,0 +1,313 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# RC4 for PA-RISC.
11
12# June 2009.
13#
14# Performance is 33% better than gcc 3.2 generated code on PA-7100LC.
15# For reference, [4x] unrolled loop is >40% faster than folded one.
16# It's possible to unroll loop 8 times on PA-RISC 2.0, but improvement
17# is believed to be not sufficient to justify the effort...
18#
19# Special thanks to polarhome.com for providing HP-UX account.
20
21$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
22
23$flavour = shift;
24$output = shift;
25open STDOUT,">$output";
26
27if ($flavour =~ /64/) {
28 $LEVEL ="2.0W";
29 $SIZE_T =8;
30 $FRAME_MARKER =80;
31 $SAVED_RP =16;
32 $PUSH ="std";
33 $PUSHMA ="std,ma";
34 $POP ="ldd";
35 $POPMB ="ldd,mb";
36} else {
37 $LEVEL ="1.0";
38 $SIZE_T =4;
39 $FRAME_MARKER =48;
40 $SAVED_RP =20;
41 $PUSH ="stw";
42 $PUSHMA ="stwm";
43 $POP ="ldw";
44 $POPMB ="ldwm";
45}
46
47$FRAME=4*$SIZE_T+$FRAME_MARKER; # 4 saved regs + frame marker
48 # [+ argument transfer]
49$SZ=1; # defaults to RC4_CHAR
50if (open CONF,"<${dir}../../opensslconf.h") {
51 while(<CONF>) {
52 if (m/#\s*define\s+RC4_INT\s+(.*)/) {
53 $SZ = ($1=~/char$/) ? 1 : 4;
54 last;
55 }
56 }
57 close CONF;
58}
59
60if ($SZ==1) { # RC4_CHAR
61 $LD="ldb";
62 $LDX="ldbx";
63 $MKX="addl";
64 $ST="stb";
65} else { # RC4_INT (~5% faster than RC4_CHAR on PA-7100LC)
66 $LD="ldw";
67 $LDX="ldwx,s";
68 $MKX="sh2addl";
69 $ST="stw";
70}
71
72$key="%r26";
73$len="%r25";
74$inp="%r24";
75$out="%r23";
76
77@XX=("%r19","%r20");
78@TX=("%r21","%r22");
79$YY="%r28";
80$TY="%r29";
81
82$acc="%r1";
83$ix="%r2";
84$iy="%r3";
85$dat0="%r4";
86$dat1="%r5";
87$rem="%r6";
88$mask="%r31";
89
90sub unrolledloopbody {
91for ($i=0;$i<4;$i++) {
92$code.=<<___;
93 ldo 1($XX[0]),$XX[1]
94 `sprintf("$LDX %$TY(%$key),%$dat1") if ($i>0)`
95 and $mask,$XX[1],$XX[1]
96 $LDX $YY($key),$TY
97 $MKX $YY,$key,$ix
98 $LDX $XX[1]($key),$TX[1]
99 $MKX $XX[0],$key,$iy
100 $ST $TX[0],0($ix)
101 comclr,<> $XX[1],$YY,%r0 ; conditional
102 copy $TX[0],$TX[1] ; move
103 `sprintf("%sdep %$dat1,%d,8,%$acc",$i==1?"z":"",8*($i-1)+7) if ($i>0)`
104 $ST $TY,0($iy)
105 addl $TX[0],$TY,$TY
106 addl $TX[1],$YY,$YY
107 and $mask,$TY,$TY
108 and $mask,$YY,$YY
109___
110push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
111} }
112
113sub foldedloop {
114my ($label,$count)=@_;
115$code.=<<___;
116$label
117 $MKX $YY,$key,$iy
118 $LDX $YY($key),$TY
119 $MKX $XX[0],$key,$ix
120 $ST $TX[0],0($iy)
121 ldo 1($XX[0]),$XX[0]
122 $ST $TY,0($ix)
123 addl $TX[0],$TY,$TY
124 ldbx $inp($out),$dat1
125 and $mask,$TY,$TY
126 and $mask,$XX[0],$XX[0]
127 $LDX $TY($key),$acc
128 $LDX $XX[0]($key),$TX[0]
129 ldo 1($out),$out
130 xor $dat1,$acc,$acc
131 addl $TX[0],$YY,$YY
132 stb $acc,-1($out)
133 addib,<> -1,$count,$label ; $count is always small
134 and $mask,$YY,$YY
135___
136}
137
138$code=<<___;
139 .LEVEL $LEVEL
140 .SPACE \$TEXT\$
141 .SUBSPA \$CODE\$,QUAD=0,ALIGN=8,ACCESS=0x2C,CODE_ONLY
142
143 .EXPORT RC4,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR,ARGW3=GR
144RC4
145 .PROC
146 .CALLINFO FRAME=`$FRAME-4*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=6
147 .ENTRY
148 $PUSH %r2,-$SAVED_RP(%sp) ; standard prologue
149 $PUSHMA %r3,$FRAME(%sp)
150 $PUSH %r4,`-$FRAME+1*$SIZE_T`(%sp)
151 $PUSH %r5,`-$FRAME+2*$SIZE_T`(%sp)
152 $PUSH %r6,`-$FRAME+3*$SIZE_T`(%sp)
153
154 cmpib,*= 0,$len,L\$abort
155 sub $inp,$out,$inp ; distance between $inp and $out
156
157 $LD `0*$SZ`($key),$XX[0]
158 $LD `1*$SZ`($key),$YY
159 ldo `2*$SZ`($key),$key
160
161 ldi 0xff,$mask
162 ldi 3,$dat0
163
164 ldo 1($XX[0]),$XX[0] ; warm up loop
165 and $mask,$XX[0],$XX[0]
166 $LDX $XX[0]($key),$TX[0]
167 addl $TX[0],$YY,$YY
168 cmpib,*>>= 6,$len,L\$oop1 ; is $len large enough to bother?
169 and $mask,$YY,$YY
170
171 and,<> $out,$dat0,$rem ; is $out aligned?
172 b L\$alignedout
173 subi 4,$rem,$rem
174 sub $len,$rem,$len
175___
176&foldedloop("L\$alignout",$rem); # process till $out is aligned
177
178$code.=<<___;
179L\$alignedout ; $len is at least 4 here
180 and,<> $inp,$dat0,$acc ; is $inp aligned?
181 b L\$oop4
182 sub $inp,$acc,$rem ; align $inp
183
184 sh3addl $acc,%r0,$acc
185 subi 32,$acc,$acc
186 mtctl $acc,%cr11 ; load %sar with vshd align factor
187 ldwx $rem($out),$dat0
188 ldo 4($rem),$rem
189L\$oop4misalignedinp
190___
191&unrolledloopbody();
192$code.=<<___;
193 $LDX $TY($key),$ix
194 ldwx $rem($out),$dat1
195 ldo -4($len),$len
196 or $ix,$acc,$acc ; last piece, no need to dep
197 vshd $dat0,$dat1,$iy ; align data
198 copy $dat1,$dat0
199 xor $iy,$acc,$acc
200 stw $acc,0($out)
201 cmpib,*<< 3,$len,L\$oop4misalignedinp
202 ldo 4($out),$out
203 cmpib,*= 0,$len,L\$done
204 nop
205 b L\$oop1
206 nop
207
208 .ALIGN 8
209L\$oop4
210___
211&unrolledloopbody();
212$code.=<<___;
213 $LDX $TY($key),$ix
214 ldwx $inp($out),$dat0
215 ldo -4($len),$len
216 or $ix,$acc,$acc ; last piece, no need to dep
217 xor $dat0,$acc,$acc
218 stw $acc,0($out)
219 cmpib,*<< 3,$len,L\$oop4
220 ldo 4($out),$out
221 cmpib,*= 0,$len,L\$done
222 nop
223___
224&foldedloop("L\$oop1",$len);
225$code.=<<___;
226L\$done
227 $POP `-$FRAME-$SAVED_RP`(%sp),%r2
228 ldo -1($XX[0]),$XX[0] ; chill out loop
229 sub $YY,$TX[0],$YY
230 and $mask,$XX[0],$XX[0]
231 and $mask,$YY,$YY
232 $ST $XX[0],`-2*$SZ`($key)
233 $ST $YY,`-1*$SZ`($key)
234 $POP `-$FRAME+1*$SIZE_T`(%sp),%r4
235 $POP `-$FRAME+2*$SIZE_T`(%sp),%r5
236 $POP `-$FRAME+3*$SIZE_T`(%sp),%r6
237L\$abort
238 bv (%r2)
239 .EXIT
240 $POPMB -$FRAME(%sp),%r3
241 .PROCEND
242___
243
244$code.=<<___;
245
246 .EXPORT private_RC4_set_key,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR
247 .ALIGN 8
248private_RC4_set_key
249 .PROC
250 .CALLINFO NO_CALLS
251 .ENTRY
252 $ST %r0,`0*$SZ`($key)
253 $ST %r0,`1*$SZ`($key)
254 ldo `2*$SZ`($key),$key
255 copy %r0,@XX[0]
256L\$1st
257 $ST @XX[0],0($key)
258 ldo 1(@XX[0]),@XX[0]
259 bb,>= @XX[0],`31-8`,L\$1st ; @XX[0]<256
260 ldo $SZ($key),$key
261
262 ldo `-256*$SZ`($key),$key ; rewind $key
263 addl $len,$inp,$inp ; $inp to point at the end
264 sub %r0,$len,%r23 ; inverse index
265 copy %r0,@XX[0]
266 copy %r0,@XX[1]
267 ldi 0xff,$mask
268
269L\$2nd
270 $LDX @XX[0]($key),@TX[0]
271 ldbx %r23($inp),@TX[1]
272 addi,nuv 1,%r23,%r23 ; increment and conditional
273 sub %r0,$len,%r23 ; inverse index
274 addl @TX[0],@XX[1],@XX[1]
275 addl @TX[1],@XX[1],@XX[1]
276 and $mask,@XX[1],@XX[1]
277 $MKX @XX[0],$key,$TY
278 $LDX @XX[1]($key),@TX[1]
279 $MKX @XX[1],$key,$YY
280 ldo 1(@XX[0]),@XX[0]
281 $ST @TX[0],0($YY)
282 bb,>= @XX[0],`31-8`,L\$2nd ; @XX[0]<256
283 $ST @TX[1],0($TY)
284
285 bv,n (%r2)
286 .EXIT
287 nop
288 .PROCEND
289
290 .EXPORT RC4_options,ENTRY
291 .ALIGN 8
292RC4_options
293 .PROC
294 .CALLINFO NO_CALLS
295 .ENTRY
296 blr %r0,%r28
297 ldi 3,%r1
298L\$pic
299 andcm %r28,%r1,%r28
300 bv (%r2)
301 .EXIT
302 ldo L\$opts-L\$pic(%r28),%r28
303 .PROCEND
304 .ALIGN 8
305L\$opts
306 .STRINGZ "rc4(4x,`$SZ==1?"char":"int"`)"
307 .STRINGZ "RC4 for PA-RISC, CRYPTOGAMS by <appro\@openssl.org>"
308___
309$code =~ s/\`([^\`]*)\`/eval $1/gem;
310$code =~ s/cmpib,\*/comib,/gm if ($SIZE_T==4);
311
312print $code;
313close STDOUT;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-s390x.pl b/src/lib/libcrypto/rc4/asm/rc4-s390x.pl
new file mode 100644
index 0000000000..7528ece13c
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-s390x.pl
@@ -0,0 +1,234 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# February 2009
11#
12# Performance is 2x of gcc 3.4.6 on z10. Coding "secret" is to
13# "cluster" Address Generation Interlocks, so that one pipeline stall
14# resolves several dependencies.
15
16# November 2010.
17#
18# Adapt for -m31 build. If kernel supports what's called "highgprs"
19# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
20# instructions and achieve "64-bit" performance even in 31-bit legacy
21# application context. The feature is not specific to any particular
22# processor, as long as it's "z-CPU". Latter implies that the code
23# remains z/Architecture specific. On z990 it was measured to perform
24# 50% better than code generated by gcc 4.3.
25
26$flavour = shift;
27
28if ($flavour =~ /3[12]/) {
29 $SIZE_T=4;
30 $g="";
31} else {
32 $SIZE_T=8;
33 $g="g";
34}
35
36while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
37open STDOUT,">$output";
38
39$rp="%r14";
40$sp="%r15";
41$code=<<___;
42.text
43
44___
45
46# void RC4(RC4_KEY *key,size_t len,const void *inp,void *out)
47{
48$acc="%r0";
49$cnt="%r1";
50$key="%r2";
51$len="%r3";
52$inp="%r4";
53$out="%r5";
54
55@XX=("%r6","%r7");
56@TX=("%r8","%r9");
57$YY="%r10";
58$TY="%r11";
59
60$code.=<<___;
61.globl RC4
62.type RC4,\@function
63.align 64
64RC4:
65 stm${g} %r6,%r11,6*$SIZE_T($sp)
66___
67$code.=<<___ if ($flavour =~ /3[12]/);
68 llgfr $len,$len
69___
70$code.=<<___;
71 llgc $XX[0],0($key)
72 llgc $YY,1($key)
73 la $XX[0],1($XX[0])
74 nill $XX[0],0xff
75 srlg $cnt,$len,3
76 ltgr $cnt,$cnt
77 llgc $TX[0],2($XX[0],$key)
78 jz .Lshort
79 j .Loop8
80
81.align 64
82.Loop8:
83___
84for ($i=0;$i<8;$i++) {
85$code.=<<___;
86 la $YY,0($YY,$TX[0]) # $i
87 nill $YY,255
88 la $XX[1],1($XX[0])
89 nill $XX[1],255
90___
91$code.=<<___ if ($i==1);
92 llgc $acc,2($TY,$key)
93___
94$code.=<<___ if ($i>1);
95 sllg $acc,$acc,8
96 ic $acc,2($TY,$key)
97___
98$code.=<<___;
99 llgc $TY,2($YY,$key)
100 stc $TX[0],2($YY,$key)
101 llgc $TX[1],2($XX[1],$key)
102 stc $TY,2($XX[0],$key)
103 cr $XX[1],$YY
104 jne .Lcmov$i
105 la $TX[1],0($TX[0])
106.Lcmov$i:
107 la $TY,0($TY,$TX[0])
108 nill $TY,255
109___
110push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
111}
112
113$code.=<<___;
114 lg $TX[1],0($inp)
115 sllg $acc,$acc,8
116 la $inp,8($inp)
117 ic $acc,2($TY,$key)
118 xgr $acc,$TX[1]
119 stg $acc,0($out)
120 la $out,8($out)
121 brctg $cnt,.Loop8
122
123.Lshort:
124 lghi $acc,7
125 ngr $len,$acc
126 jz .Lexit
127 j .Loop1
128
129.align 16
130.Loop1:
131 la $YY,0($YY,$TX[0])
132 nill $YY,255
133 llgc $TY,2($YY,$key)
134 stc $TX[0],2($YY,$key)
135 stc $TY,2($XX[0],$key)
136 ar $TY,$TX[0]
137 ahi $XX[0],1
138 nill $TY,255
139 nill $XX[0],255
140 llgc $acc,0($inp)
141 la $inp,1($inp)
142 llgc $TY,2($TY,$key)
143 llgc $TX[0],2($XX[0],$key)
144 xr $acc,$TY
145 stc $acc,0($out)
146 la $out,1($out)
147 brct $len,.Loop1
148
149.Lexit:
150 ahi $XX[0],-1
151 stc $XX[0],0($key)
152 stc $YY,1($key)
153 lm${g} %r6,%r11,6*$SIZE_T($sp)
154 br $rp
155.size RC4,.-RC4
156.string "RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
157
158___
159}
160
161# void RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)
162{
163$cnt="%r0";
164$idx="%r1";
165$key="%r2";
166$len="%r3";
167$inp="%r4";
168$acc="%r5";
169$dat="%r6";
170$ikey="%r7";
171$iinp="%r8";
172
173$code.=<<___;
174.globl private_RC4_set_key
175.type private_RC4_set_key,\@function
176.align 64
177private_RC4_set_key:
178 stm${g} %r6,%r8,6*$SIZE_T($sp)
179 lhi $cnt,256
180 la $idx,0(%r0)
181 sth $idx,0($key)
182.align 4
183.L1stloop:
184 stc $idx,2($idx,$key)
185 la $idx,1($idx)
186 brct $cnt,.L1stloop
187
188 lghi $ikey,-256
189 lr $cnt,$len
190 la $iinp,0(%r0)
191 la $idx,0(%r0)
192.align 16
193.L2ndloop:
194 llgc $acc,2+256($ikey,$key)
195 llgc $dat,0($iinp,$inp)
196 la $idx,0($idx,$acc)
197 la $ikey,1($ikey)
198 la $idx,0($idx,$dat)
199 nill $idx,255
200 la $iinp,1($iinp)
201 tml $ikey,255
202 llgc $dat,2($idx,$key)
203 stc $dat,2+256-1($ikey,$key)
204 stc $acc,2($idx,$key)
205 jz .Ldone
206 brct $cnt,.L2ndloop
207 lr $cnt,$len
208 la $iinp,0(%r0)
209 j .L2ndloop
210.Ldone:
211 lm${g} %r6,%r8,6*$SIZE_T($sp)
212 br $rp
213.size private_RC4_set_key,.-private_RC4_set_key
214
215___
216}
217
218# const char *RC4_options()
219$code.=<<___;
220.globl RC4_options
221.type RC4_options,\@function
222.align 16
223RC4_options:
224 larl %r2,.Loptions
225 br %r14
226.size RC4_options,.-RC4_options
227.section .rodata
228.Loptions:
229.align 8
230.string "rc4(8x,char)"
231___
232
233print $code;
234close STDOUT; # force flush
diff --git a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
new file mode 100755
index 0000000000..544386bf53
--- /dev/null
+++ b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
@@ -0,0 +1,504 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
11# "hand-coded assembler"] doesn't stand for the whole improvement
12# coefficient. It turned out that eliminating RC4_CHAR from config
13# line results in ~40% improvement (yes, even for C implementation).
14# Presumably it has everything to do with AMD cache architecture and
15# RAW or whatever penalties. Once again! The module *requires* config
16# line *without* RC4_CHAR! As for coding "secret," I bet on partial
17# register arithmetics. For example instead of 'inc %r8; and $255,%r8'
18# I simply 'inc %r8b'. Even though optimization manual discourages
19# to operate on partial registers, it turned out to be the best bet.
20# At least for AMD... How IA32E would perform remains to be seen...
21
22# As was shown by Marc Bevand reordering of couple of load operations
23# results in even higher performance gain of 3.3x:-) At least on
24# Opteron... For reference, 1x in this case is RC4_CHAR C-code
25# compiled with gcc 3.3.2, which performs at ~54MBps per 1GHz clock.
26# Latter means that if you want to *estimate* what to expect from
27# *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
28
29# Intel P4 EM64T core was found to run the AMD64 code really slow...
30# The only way to achieve comparable performance on P4 was to keep
31# RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
32# compose blended code, which would perform even within 30% marginal
33# on either AMD and Intel platforms, I implement both cases. See
34# rc4_skey.c for further details...
35
36# P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
37# those with add/sub results in 50% performance improvement of folded
38# loop...
39
40# As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
41# performance by >30% [unlike P4 32-bit case that is]. But this is
42# provided that loads are reordered even more aggressively! Both code
43# pathes, AMD64 and EM64T, reorder loads in essentially same manner
44# as my IA-64 implementation. On Opteron this resulted in modest 5%
45# improvement [I had to test it], while final Intel P4 performance
46# achieves respectful 432MBps on 2.8GHz processor now. For reference.
47# If executed on Xeon, current RC4_CHAR code-path is 2.7x faster than
48# RC4_INT code-path. While if executed on Opteron, it's only 25%
49# slower than the RC4_INT one [meaning that if CPU µ-arch detection
50# is not implemented, then this final RC4_CHAR code-path should be
51# preferred, as it provides better *all-round* performance].
52
53# Intel Core2 was observed to perform poorly on both code paths:-( It
54# apparently suffers from some kind of partial register stall, which
55# occurs in 64-bit mode only [as virtually identical 32-bit loop was
56# observed to outperform 64-bit one by almost 50%]. Adding two movzb to
57# cloop1 boosts its performance by 80%! This loop appears to be optimal
58# fit for Core2 and therefore the code was modified to skip cloop8 on
59# this CPU.
60
61$flavour = shift;
62$output = shift;
63if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
64
65$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
66
67$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
68( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
69( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
70die "can't locate x86_64-xlate.pl";
71
72open STDOUT,"| $^X $xlate $flavour $output";
73
74$dat="%rdi"; # arg1
75$len="%rsi"; # arg2
76$inp="%rdx"; # arg3
77$out="%rcx"; # arg4
78
79@XX=("%r8","%r10");
80@TX=("%r9","%r11");
81$YY="%r12";
82$TY="%r13";
83
84$code=<<___;
85.text
86
87.globl RC4
88.type RC4,\@function,4
89.align 16
90RC4: or $len,$len
91 jne .Lentry
92 ret
93.Lentry:
94 push %rbx
95 push %r12
96 push %r13
97.Lprologue:
98
99 add \$8,$dat
100 movl -8($dat),$XX[0]#d
101 movl -4($dat),$YY#d
102 cmpl \$-1,256($dat)
103 je .LRC4_CHAR
104 inc $XX[0]#b
105 movl ($dat,$XX[0],4),$TX[0]#d
106 test \$-8,$len
107 jz .Lloop1
108 jmp .Lloop8
109.align 16
110.Lloop8:
111___
112for ($i=0;$i<8;$i++) {
113$code.=<<___;
114 add $TX[0]#b,$YY#b
115 mov $XX[0],$XX[1]
116 movl ($dat,$YY,4),$TY#d
117 ror \$8,%rax # ror is redundant when $i=0
118 inc $XX[1]#b
119 movl ($dat,$XX[1],4),$TX[1]#d
120 cmp $XX[1],$YY
121 movl $TX[0]#d,($dat,$YY,4)
122 cmove $TX[0],$TX[1]
123 movl $TY#d,($dat,$XX[0],4)
124 add $TX[0]#b,$TY#b
125 movb ($dat,$TY,4),%al
126___
127push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
128}
129$code.=<<___;
130 ror \$8,%rax
131 sub \$8,$len
132
133 xor ($inp),%rax
134 add \$8,$inp
135 mov %rax,($out)
136 add \$8,$out
137
138 test \$-8,$len
139 jnz .Lloop8
140 cmp \$0,$len
141 jne .Lloop1
142 jmp .Lexit
143
144.align 16
145.Lloop1:
146 add $TX[0]#b,$YY#b
147 movl ($dat,$YY,4),$TY#d
148 movl $TX[0]#d,($dat,$YY,4)
149 movl $TY#d,($dat,$XX[0],4)
150 add $TY#b,$TX[0]#b
151 inc $XX[0]#b
152 movl ($dat,$TX[0],4),$TY#d
153 movl ($dat,$XX[0],4),$TX[0]#d
154 xorb ($inp),$TY#b
155 inc $inp
156 movb $TY#b,($out)
157 inc $out
158 dec $len
159 jnz .Lloop1
160 jmp .Lexit
161
162.align 16
163.LRC4_CHAR:
164 add \$1,$XX[0]#b
165 movzb ($dat,$XX[0]),$TX[0]#d
166 test \$-8,$len
167 jz .Lcloop1
168 cmpl \$0,260($dat)
169 jnz .Lcloop1
170 jmp .Lcloop8
171.align 16
172.Lcloop8:
173 mov ($inp),%eax
174 mov 4($inp),%ebx
175___
176# unroll 2x4-wise, because 64-bit rotates kill Intel P4...
177for ($i=0;$i<4;$i++) {
178$code.=<<___;
179 add $TX[0]#b,$YY#b
180 lea 1($XX[0]),$XX[1]
181 movzb ($dat,$YY),$TY#d
182 movzb $XX[1]#b,$XX[1]#d
183 movzb ($dat,$XX[1]),$TX[1]#d
184 movb $TX[0]#b,($dat,$YY)
185 cmp $XX[1],$YY
186 movb $TY#b,($dat,$XX[0])
187 jne .Lcmov$i # Intel cmov is sloooow...
188 mov $TX[0],$TX[1]
189.Lcmov$i:
190 add $TX[0]#b,$TY#b
191 xor ($dat,$TY),%al
192 ror \$8,%eax
193___
194push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
195}
196for ($i=4;$i<8;$i++) {
197$code.=<<___;
198 add $TX[0]#b,$YY#b
199 lea 1($XX[0]),$XX[1]
200 movzb ($dat,$YY),$TY#d
201 movzb $XX[1]#b,$XX[1]#d
202 movzb ($dat,$XX[1]),$TX[1]#d
203 movb $TX[0]#b,($dat,$YY)
204 cmp $XX[1],$YY
205 movb $TY#b,($dat,$XX[0])
206 jne .Lcmov$i # Intel cmov is sloooow...
207 mov $TX[0],$TX[1]
208.Lcmov$i:
209 add $TX[0]#b,$TY#b
210 xor ($dat,$TY),%bl
211 ror \$8,%ebx
212___
213push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
214}
215$code.=<<___;
216 lea -8($len),$len
217 mov %eax,($out)
218 lea 8($inp),$inp
219 mov %ebx,4($out)
220 lea 8($out),$out
221
222 test \$-8,$len
223 jnz .Lcloop8
224 cmp \$0,$len
225 jne .Lcloop1
226 jmp .Lexit
227___
228$code.=<<___;
229.align 16
230.Lcloop1:
231 add $TX[0]#b,$YY#b
232 movzb ($dat,$YY),$TY#d
233 movb $TX[0]#b,($dat,$YY)
234 movb $TY#b,($dat,$XX[0])
235 add $TX[0]#b,$TY#b
236 add \$1,$XX[0]#b
237 movzb $TY#b,$TY#d
238 movzb $XX[0]#b,$XX[0]#d
239 movzb ($dat,$TY),$TY#d
240 movzb ($dat,$XX[0]),$TX[0]#d
241 xorb ($inp),$TY#b
242 lea 1($inp),$inp
243 movb $TY#b,($out)
244 lea 1($out),$out
245 sub \$1,$len
246 jnz .Lcloop1
247 jmp .Lexit
248
249.align 16
250.Lexit:
251 sub \$1,$XX[0]#b
252 movl $XX[0]#d,-8($dat)
253 movl $YY#d,-4($dat)
254
255 mov (%rsp),%r13
256 mov 8(%rsp),%r12
257 mov 16(%rsp),%rbx
258 add \$24,%rsp
259.Lepilogue:
260 ret
261.size RC4,.-RC4
262___
263
264$idx="%r8";
265$ido="%r9";
266
267$code.=<<___;
268.extern OPENSSL_ia32cap_P
269.globl RC4_set_key
270.type RC4_set_key,\@function,3
271.align 16
272RC4_set_key:
273 lea 8($dat),$dat
274 lea ($inp,$len),$inp
275 neg $len
276 mov $len,%rcx
277 xor %eax,%eax
278 xor $ido,$ido
279 xor %r10,%r10
280 xor %r11,%r11
281
282 mov PIC_GOT(OPENSSL_ia32cap_P),$idx#d
283 bt \$20,$idx#d
284 jnc .Lw1stloop
285 bt \$30,$idx#d
286 setc $ido#b
287 mov $ido#d,260($dat)
288 jmp .Lc1stloop
289
290.align 16
291.Lw1stloop:
292 mov %eax,($dat,%rax,4)
293 add \$1,%al
294 jnc .Lw1stloop
295
296 xor $ido,$ido
297 xor $idx,$idx
298.align 16
299.Lw2ndloop:
300 mov ($dat,$ido,4),%r10d
301 add ($inp,$len,1),$idx#b
302 add %r10b,$idx#b
303 add \$1,$len
304 mov ($dat,$idx,4),%r11d
305 cmovz %rcx,$len
306 mov %r10d,($dat,$idx,4)
307 mov %r11d,($dat,$ido,4)
308 add \$1,$ido#b
309 jnc .Lw2ndloop
310 jmp .Lexit_key
311
312.align 16
313.Lc1stloop:
314 mov %al,($dat,%rax)
315 add \$1,%al
316 jnc .Lc1stloop
317
318 xor $ido,$ido
319 xor $idx,$idx
320.align 16
321.Lc2ndloop:
322 mov ($dat,$ido),%r10b
323 add ($inp,$len),$idx#b
324 add %r10b,$idx#b
325 add \$1,$len
326 mov ($dat,$idx),%r11b
327 jnz .Lcnowrap
328 mov %rcx,$len
329.Lcnowrap:
330 mov %r10b,($dat,$idx)
331 mov %r11b,($dat,$ido)
332 add \$1,$ido#b
333 jnc .Lc2ndloop
334 movl \$-1,256($dat)
335
336.align 16
337.Lexit_key:
338 xor %eax,%eax
339 mov %eax,-8($dat)
340 mov %eax,-4($dat)
341 ret
342.size RC4_set_key,.-RC4_set_key
343
344.globl RC4_options
345.type RC4_options,\@abi-omnipotent
346.align 16
347RC4_options:
348 lea .Lopts(%rip),%rax
349 mov PIC_GOT(OPENSSL_ia32cap_P),%edx
350 bt \$20,%edx
351 jnc .Ldone
352 add \$12,%rax
353 bt \$30,%edx
354 jnc .Ldone
355 add \$13,%rax
356.Ldone:
357 ret
358.align 64
359.Lopts:
360.asciz "rc4(8x,int)"
361.asciz "rc4(8x,char)"
362.asciz "rc4(1x,char)"
363.asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
364.align 64
365.size RC4_options,.-RC4_options
366___
367
368# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
369# CONTEXT *context,DISPATCHER_CONTEXT *disp)
370if ($win64) {
371$rec="%rcx";
372$frame="%rdx";
373$context="%r8";
374$disp="%r9";
375
376$code.=<<___;
377.extern __imp_RtlVirtualUnwind
378.type stream_se_handler,\@abi-omnipotent
379.align 16
380stream_se_handler:
381 push %rsi
382 push %rdi
383 push %rbx
384 push %rbp
385 push %r12
386 push %r13
387 push %r14
388 push %r15
389 pushfq
390 sub \$64,%rsp
391
392 mov 120($context),%rax # pull context->Rax
393 mov 248($context),%rbx # pull context->Rip
394
395 lea .Lprologue(%rip),%r10
396 cmp %r10,%rbx # context->Rip<prologue label
397 jb .Lin_prologue
398
399 mov 152($context),%rax # pull context->Rsp
400
401 lea .Lepilogue(%rip),%r10
402 cmp %r10,%rbx # context->Rip>=epilogue label
403 jae .Lin_prologue
404
405 lea 24(%rax),%rax
406
407 mov -8(%rax),%rbx
408 mov -16(%rax),%r12
409 mov -24(%rax),%r13
410 mov %rbx,144($context) # restore context->Rbx
411 mov %r12,216($context) # restore context->R12
412 mov %r13,224($context) # restore context->R13
413
414.Lin_prologue:
415 mov 8(%rax),%rdi
416 mov 16(%rax),%rsi
417 mov %rax,152($context) # restore context->Rsp
418 mov %rsi,168($context) # restore context->Rsi
419 mov %rdi,176($context) # restore context->Rdi
420
421 jmp .Lcommon_seh_exit
422.size stream_se_handler,.-stream_se_handler
423
424.type key_se_handler,\@abi-omnipotent
425.align 16
426key_se_handler:
427 push %rsi
428 push %rdi
429 push %rbx
430 push %rbp
431 push %r12
432 push %r13
433 push %r14
434 push %r15
435 pushfq
436 sub \$64,%rsp
437
438 mov 152($context),%rax # pull context->Rsp
439 mov 8(%rax),%rdi
440 mov 16(%rax),%rsi
441 mov %rsi,168($context) # restore context->Rsi
442 mov %rdi,176($context) # restore context->Rdi
443
444.Lcommon_seh_exit:
445
446 mov 40($disp),%rdi # disp->ContextRecord
447 mov $context,%rsi # context
448 mov \$154,%ecx # sizeof(CONTEXT)
449 .long 0xa548f3fc # cld; rep movsq
450
451 mov $disp,%rsi
452 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
453 mov 8(%rsi),%rdx # arg2, disp->ImageBase
454 mov 0(%rsi),%r8 # arg3, disp->ControlPc
455 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
456 mov 40(%rsi),%r10 # disp->ContextRecord
457 lea 56(%rsi),%r11 # &disp->HandlerData
458 lea 24(%rsi),%r12 # &disp->EstablisherFrame
459 mov %r10,32(%rsp) # arg5
460 mov %r11,40(%rsp) # arg6
461 mov %r12,48(%rsp) # arg7
462 mov %rcx,56(%rsp) # arg8, (NULL)
463 call *__imp_RtlVirtualUnwind(%rip)
464
465 mov \$1,%eax # ExceptionContinueSearch
466 add \$64,%rsp
467 popfq
468 pop %r15
469 pop %r14
470 pop %r13
471 pop %r12
472 pop %rbp
473 pop %rbx
474 pop %rdi
475 pop %rsi
476 ret
477.size key_se_handler,.-key_se_handler
478
479.section .pdata
480.align 4
481 .rva .LSEH_begin_RC4
482 .rva .LSEH_end_RC4
483 .rva .LSEH_info_RC4
484
485 .rva .LSEH_begin_RC4_set_key
486 .rva .LSEH_end_RC4_set_key
487 .rva .LSEH_info_RC4_set_key
488
489.section .xdata
490.align 8
491.LSEH_info_RC4:
492 .byte 9,0,0,0
493 .rva stream_se_handler
494.LSEH_info_RC4_set_key:
495 .byte 9,0,0,0
496 .rva key_se_handler
497___
498}
499
500$code =~ s/#([bwd])/$1/gm;
501
502print $code;
503
504close STDOUT;
diff --git a/src/lib/libcrypto/rc4/rc4.h b/src/lib/libcrypto/rc4/rc4.h
new file mode 100644
index 0000000000..29d1acccf5
--- /dev/null
+++ b/src/lib/libcrypto/rc4/rc4.h
@@ -0,0 +1,89 @@
1/* crypto/rc4/rc4.h */
2/* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
3 * All rights reserved.
4 *
5 * This package is an SSL implementation written
6 * by Eric Young (eay@cryptsoft.com).
7 * The implementation was written so as to conform with Netscapes SSL.
8 *
9 * This library is free for commercial and non-commercial use as long as
10 * the following conditions are aheared to. The following conditions
11 * apply to all code found in this distribution, be it the RC4, RSA,
12 * lhash, DES, etc., code; not just the SSL code. The SSL documentation
13 * included with this distribution is covered by the same copyright terms
14 * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15 *
16 * Copyright remains Eric Young's, and as such any Copyright notices in
17 * the code are not to be removed.
18 * If this package is used in a product, Eric Young should be given attribution
19 * as the author of the parts of the library used.
20 * This can be in the form of a textual message at program startup or
21 * in documentation (online or textual) provided with the package.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 * 1. Redistributions of source code must retain the copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in the
30 * documentation and/or other materials provided with the distribution.
31 * 3. All advertising materials mentioning features or use of this software
32 * must display the following acknowledgement:
33 * "This product includes cryptographic software written by
34 * Eric Young (eay@cryptsoft.com)"
35 * The word 'cryptographic' can be left out if the rouines from the library
36 * being used are not cryptographic related :-).
37 * 4. If you include any Windows specific code (or a derivative thereof) from
38 * the apps directory (application code) you must include an acknowledgement:
39 * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40 *
41 * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 *
53 * The licence and distribution terms for any publically available version or
54 * derivative of this code cannot be changed. i.e. this code cannot simply be
55 * copied and put under another distribution licence
56 * [including the GNU Public Licence.]
57 */
58
59#ifndef HEADER_RC4_H
60#define HEADER_RC4_H
61
62#include <openssl/opensslconf.h> /* OPENSSL_NO_RC4, RC4_INT */
63#ifdef OPENSSL_NO_RC4
64#error RC4 is disabled.
65#endif
66
67#include <stddef.h>
68
69#ifdef __cplusplus
70extern "C" {
71#endif
72
73typedef struct rc4_key_st
74 {
75 RC4_INT x,y;
76 RC4_INT data[256];
77 } RC4_KEY;
78
79
80const char *RC4_options(void);
81void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data);
82void RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
83 unsigned char *outdata);
84
85#ifdef __cplusplus
86}
87#endif
88
89#endif
diff --git a/src/lib/libcrypto/rc4/rc4_enc.c b/src/lib/libcrypto/rc4/rc4_enc.c
new file mode 100644
index 0000000000..8c4fc6c7a3
--- /dev/null
+++ b/src/lib/libcrypto/rc4/rc4_enc.c
@@ -0,0 +1,315 @@
1/* crypto/rc4/rc4_enc.c */
2/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3 * All rights reserved.
4 *
5 * This package is an SSL implementation written
6 * by Eric Young (eay@cryptsoft.com).
7 * The implementation was written so as to conform with Netscapes SSL.
8 *
9 * This library is free for commercial and non-commercial use as long as
10 * the following conditions are aheared to. The following conditions
11 * apply to all code found in this distribution, be it the RC4, RSA,
12 * lhash, DES, etc., code; not just the SSL code. The SSL documentation
13 * included with this distribution is covered by the same copyright terms
14 * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15 *
16 * Copyright remains Eric Young's, and as such any Copyright notices in
17 * the code are not to be removed.
18 * If this package is used in a product, Eric Young should be given attribution
19 * as the author of the parts of the library used.
20 * This can be in the form of a textual message at program startup or
21 * in documentation (online or textual) provided with the package.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 * 1. Redistributions of source code must retain the copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in the
30 * documentation and/or other materials provided with the distribution.
31 * 3. All advertising materials mentioning features or use of this software
32 * must display the following acknowledgement:
33 * "This product includes cryptographic software written by
34 * Eric Young (eay@cryptsoft.com)"
35 * The word 'cryptographic' can be left out if the rouines from the library
36 * being used are not cryptographic related :-).
37 * 4. If you include any Windows specific code (or a derivative thereof) from
38 * the apps directory (application code) you must include an acknowledgement:
39 * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40 *
41 * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 *
53 * The licence and distribution terms for any publically available version or
54 * derivative of this code cannot be changed. i.e. this code cannot simply be
55 * copied and put under another distribution licence
56 * [including the GNU Public Licence.]
57 */
58
59#include <openssl/rc4.h>
60#include "rc4_locl.h"
61
62/* RC4 as implemented from a posting from
63 * Newsgroups: sci.crypt
64 * From: sterndark@netcom.com (David Sterndark)
65 * Subject: RC4 Algorithm revealed.
66 * Message-ID: <sternCvKL4B.Hyy@netcom.com>
67 * Date: Wed, 14 Sep 1994 06:35:31 GMT
68 */
69
70void RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
71 unsigned char *outdata)
72 {
73 register RC4_INT *d;
74 register RC4_INT x,y,tx,ty;
75 size_t i;
76
77 x=key->x;
78 y=key->y;
79 d=key->data;
80
81#if defined(RC4_CHUNK)
82 /*
83 * The original reason for implementing this(*) was the fact that
84 * pre-21164a Alpha CPUs don't have byte load/store instructions
85 * and e.g. a byte store has to be done with 64-bit load, shift,
86 * and, or and finally 64-bit store. Peaking data and operating
87 * at natural word size made it possible to reduce amount of
88 * instructions as well as to perform early read-ahead without
89 * suffering from RAW (read-after-write) hazard. This resulted
90 * in ~40%(**) performance improvement on 21064 box with gcc.
91 * But it's not only Alpha users who win here:-) Thanks to the
92 * early-n-wide read-ahead this implementation also exhibits
93 * >40% speed-up on SPARC and 20-30% on 64-bit MIPS (depending
94 * on sizeof(RC4_INT)).
95 *
96 * (*) "this" means code which recognizes the case when input
97 * and output pointers appear to be aligned at natural CPU
98 * word boundary
99 * (**) i.e. according to 'apps/openssl speed rc4' benchmark,
100 * crypto/rc4/rc4speed.c exhibits almost 70% speed-up...
101 *
102 * Cavets.
103 *
104 * - RC4_CHUNK="unsigned long long" should be a #1 choice for
105 * UltraSPARC. Unfortunately gcc generates very slow code
106 * (2.5-3 times slower than one generated by Sun's WorkShop
107 * C) and therefore gcc (at least 2.95 and earlier) should
108 * always be told that RC4_CHUNK="unsigned long".
109 *
110 * <appro@fy.chalmers.se>
111 */
112
113# define RC4_STEP ( \
114 x=(x+1) &0xff, \
115 tx=d[x], \
116 y=(tx+y)&0xff, \
117 ty=d[y], \
118 d[y]=tx, \
119 d[x]=ty, \
120 (RC4_CHUNK)d[(tx+ty)&0xff]\
121 )
122
123 if ( ( ((size_t)indata & (sizeof(RC4_CHUNK)-1)) |
124 ((size_t)outdata & (sizeof(RC4_CHUNK)-1)) ) == 0 )
125 {
126 RC4_CHUNK ichunk,otp;
127 const union { long one; char little; } is_endian = {1};
128
129 /*
130 * I reckon we can afford to implement both endian
131 * cases and to decide which way to take at run-time
132 * because the machine code appears to be very compact
133 * and redundant 1-2KB is perfectly tolerable (i.e.
134 * in case the compiler fails to eliminate it:-). By
135 * suggestion from Terrel Larson <terr@terralogic.net>
136 * who also stands for the is_endian union:-)
137 *
138 * Special notes.
139 *
140 * - is_endian is declared automatic as doing otherwise
141 * (declaring static) prevents gcc from eliminating
142 * the redundant code;
143 * - compilers (those I've tried) don't seem to have
144 * problems eliminating either the operators guarded
145 * by "if (sizeof(RC4_CHUNK)==8)" or the condition
146 * expressions themselves so I've got 'em to replace
147 * corresponding #ifdefs from the previous version;
148 * - I chose to let the redundant switch cases when
149 * sizeof(RC4_CHUNK)!=8 be (were also #ifdefed
150 * before);
151 * - in case you wonder "&(sizeof(RC4_CHUNK)*8-1)" in
152 * [LB]ESHFT guards against "shift is out of range"
153 * warnings when sizeof(RC4_CHUNK)!=8
154 *
155 * <appro@fy.chalmers.se>
156 */
157 if (!is_endian.little)
158 { /* BIG-ENDIAN CASE */
159# define BESHFT(c) (((sizeof(RC4_CHUNK)-(c)-1)*8)&(sizeof(RC4_CHUNK)*8-1))
160 for (;len&(0-sizeof(RC4_CHUNK));len-=sizeof(RC4_CHUNK))
161 {
162 ichunk = *(RC4_CHUNK *)indata;
163 otp = RC4_STEP<<BESHFT(0);
164 otp |= RC4_STEP<<BESHFT(1);
165 otp |= RC4_STEP<<BESHFT(2);
166 otp |= RC4_STEP<<BESHFT(3);
167 if (sizeof(RC4_CHUNK)==8)
168 {
169 otp |= RC4_STEP<<BESHFT(4);
170 otp |= RC4_STEP<<BESHFT(5);
171 otp |= RC4_STEP<<BESHFT(6);
172 otp |= RC4_STEP<<BESHFT(7);
173 }
174 *(RC4_CHUNK *)outdata = otp^ichunk;
175 indata += sizeof(RC4_CHUNK);
176 outdata += sizeof(RC4_CHUNK);
177 }
178 if (len)
179 {
180 RC4_CHUNK mask=(RC4_CHUNK)-1, ochunk;
181
182 ichunk = *(RC4_CHUNK *)indata;
183 ochunk = *(RC4_CHUNK *)outdata;
184 otp = 0;
185 i = BESHFT(0);
186 mask <<= (sizeof(RC4_CHUNK)-len)<<3;
187 switch (len&(sizeof(RC4_CHUNK)-1))
188 {
189 case 7: otp = RC4_STEP<<i, i-=8;
190 case 6: otp |= RC4_STEP<<i, i-=8;
191 case 5: otp |= RC4_STEP<<i, i-=8;
192 case 4: otp |= RC4_STEP<<i, i-=8;
193 case 3: otp |= RC4_STEP<<i, i-=8;
194 case 2: otp |= RC4_STEP<<i, i-=8;
195 case 1: otp |= RC4_STEP<<i, i-=8;
196 case 0: ; /*
197 * it's never the case,
198 * but it has to be here
199 * for ultrix?
200 */
201 }
202 ochunk &= ~mask;
203 ochunk |= (otp^ichunk) & mask;
204 *(RC4_CHUNK *)outdata = ochunk;
205 }
206 key->x=x;
207 key->y=y;
208 return;
209 }
210 else
211 { /* LITTLE-ENDIAN CASE */
212# define LESHFT(c) (((c)*8)&(sizeof(RC4_CHUNK)*8-1))
213 for (;len&(0-sizeof(RC4_CHUNK));len-=sizeof(RC4_CHUNK))
214 {
215 ichunk = *(RC4_CHUNK *)indata;
216 otp = RC4_STEP;
217 otp |= RC4_STEP<<8;
218 otp |= RC4_STEP<<16;
219 otp |= RC4_STEP<<24;
220 if (sizeof(RC4_CHUNK)==8)
221 {
222 otp |= RC4_STEP<<LESHFT(4);
223 otp |= RC4_STEP<<LESHFT(5);
224 otp |= RC4_STEP<<LESHFT(6);
225 otp |= RC4_STEP<<LESHFT(7);
226 }
227 *(RC4_CHUNK *)outdata = otp^ichunk;
228 indata += sizeof(RC4_CHUNK);
229 outdata += sizeof(RC4_CHUNK);
230 }
231 if (len)
232 {
233 RC4_CHUNK mask=(RC4_CHUNK)-1, ochunk;
234
235 ichunk = *(RC4_CHUNK *)indata;
236 ochunk = *(RC4_CHUNK *)outdata;
237 otp = 0;
238 i = 0;
239 mask >>= (sizeof(RC4_CHUNK)-len)<<3;
240 switch (len&(sizeof(RC4_CHUNK)-1))
241 {
242 case 7: otp = RC4_STEP, i+=8;
243 case 6: otp |= RC4_STEP<<i, i+=8;
244 case 5: otp |= RC4_STEP<<i, i+=8;
245 case 4: otp |= RC4_STEP<<i, i+=8;
246 case 3: otp |= RC4_STEP<<i, i+=8;
247 case 2: otp |= RC4_STEP<<i, i+=8;
248 case 1: otp |= RC4_STEP<<i, i+=8;
249 case 0: ; /*
250 * it's never the case,
251 * but it has to be here
252 * for ultrix?
253 */
254 }
255 ochunk &= ~mask;
256 ochunk |= (otp^ichunk) & mask;
257 *(RC4_CHUNK *)outdata = ochunk;
258 }
259 key->x=x;
260 key->y=y;
261 return;
262 }
263 }
264#endif
265#define LOOP(in,out) \
266 x=((x+1)&0xff); \
267 tx=d[x]; \
268 y=(tx+y)&0xff; \
269 d[x]=ty=d[y]; \
270 d[y]=tx; \
271 (out) = d[(tx+ty)&0xff]^ (in);
272
273#ifndef RC4_INDEX
274#define RC4_LOOP(a,b,i) LOOP(*((a)++),*((b)++))
275#else
276#define RC4_LOOP(a,b,i) LOOP(a[i],b[i])
277#endif
278
279 i=len>>3;
280 if (i)
281 {
282 for (;;)
283 {
284 RC4_LOOP(indata,outdata,0);
285 RC4_LOOP(indata,outdata,1);
286 RC4_LOOP(indata,outdata,2);
287 RC4_LOOP(indata,outdata,3);
288 RC4_LOOP(indata,outdata,4);
289 RC4_LOOP(indata,outdata,5);
290 RC4_LOOP(indata,outdata,6);
291 RC4_LOOP(indata,outdata,7);
292#ifdef RC4_INDEX
293 indata+=8;
294 outdata+=8;
295#endif
296 if (--i == 0) break;
297 }
298 }
299 i=len&0x07;
300 if (i)
301 {
302 for (;;)
303 {
304 RC4_LOOP(indata,outdata,0); if (--i == 0) break;
305 RC4_LOOP(indata,outdata,1); if (--i == 0) break;
306 RC4_LOOP(indata,outdata,2); if (--i == 0) break;
307 RC4_LOOP(indata,outdata,3); if (--i == 0) break;
308 RC4_LOOP(indata,outdata,4); if (--i == 0) break;
309 RC4_LOOP(indata,outdata,5); if (--i == 0) break;
310 RC4_LOOP(indata,outdata,6); if (--i == 0) break;
311 }
312 }
313 key->x=x;
314 key->y=y;
315 }
diff --git a/src/lib/libcrypto/rc4/rc4_locl.h b/src/lib/libcrypto/rc4/rc4_locl.h
new file mode 100644
index 0000000000..c712e1632e
--- /dev/null
+++ b/src/lib/libcrypto/rc4/rc4_locl.h
@@ -0,0 +1,5 @@
1#ifndef HEADER_RC4_LOCL_H
2#define HEADER_RC4_LOCL_H
3#include <openssl/opensslconf.h>
4#include <cryptlib.h>
5#endif
diff --git a/src/lib/libcrypto/rc4/rc4_skey.c b/src/lib/libcrypto/rc4/rc4_skey.c
new file mode 100644
index 0000000000..b22c40b0bd
--- /dev/null
+++ b/src/lib/libcrypto/rc4/rc4_skey.c
@@ -0,0 +1,150 @@
1/* crypto/rc4/rc4_skey.c */
2/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3 * All rights reserved.
4 *
5 * This package is an SSL implementation written
6 * by Eric Young (eay@cryptsoft.com).
7 * The implementation was written so as to conform with Netscapes SSL.
8 *
9 * This library is free for commercial and non-commercial use as long as
10 * the following conditions are aheared to. The following conditions
11 * apply to all code found in this distribution, be it the RC4, RSA,
12 * lhash, DES, etc., code; not just the SSL code. The SSL documentation
13 * included with this distribution is covered by the same copyright terms
14 * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15 *
16 * Copyright remains Eric Young's, and as such any Copyright notices in
17 * the code are not to be removed.
18 * If this package is used in a product, Eric Young should be given attribution
19 * as the author of the parts of the library used.
20 * This can be in the form of a textual message at program startup or
21 * in documentation (online or textual) provided with the package.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 * 1. Redistributions of source code must retain the copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in the
30 * documentation and/or other materials provided with the distribution.
31 * 3. All advertising materials mentioning features or use of this software
32 * must display the following acknowledgement:
33 * "This product includes cryptographic software written by
34 * Eric Young (eay@cryptsoft.com)"
35 * The word 'cryptographic' can be left out if the rouines from the library
36 * being used are not cryptographic related :-).
37 * 4. If you include any Windows specific code (or a derivative thereof) from
38 * the apps directory (application code) you must include an acknowledgement:
39 * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40 *
41 * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 *
53 * The licence and distribution terms for any publically available version or
54 * derivative of this code cannot be changed. i.e. this code cannot simply be
55 * copied and put under another distribution licence
56 * [including the GNU Public Licence.]
57 */
58
59#include <openssl/rc4.h>
60#include "rc4_locl.h"
61#include <openssl/opensslv.h>
62
63const char RC4_version[]="RC4" OPENSSL_VERSION_PTEXT;
64
65const char *RC4_options(void)
66 {
67#ifdef RC4_INDEX
68 if (sizeof(RC4_INT) == 1)
69 return("rc4(idx,char)");
70 else
71 return("rc4(idx,int)");
72#else
73 if (sizeof(RC4_INT) == 1)
74 return("rc4(ptr,char)");
75 else
76 return("rc4(ptr,int)");
77#endif
78 }
79
80/* RC4 as implemented from a posting from
81 * Newsgroups: sci.crypt
82 * From: sterndark@netcom.com (David Sterndark)
83 * Subject: RC4 Algorithm revealed.
84 * Message-ID: <sternCvKL4B.Hyy@netcom.com>
85 * Date: Wed, 14 Sep 1994 06:35:31 GMT
86 */
87
88void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data)
89 {
90 register RC4_INT tmp;
91 register int id1,id2;
92 register RC4_INT *d;
93 unsigned int i;
94
95 d= &(key->data[0]);
96 key->x = 0;
97 key->y = 0;
98 id1=id2=0;
99
100#define SK_LOOP(d,n) { \
101 tmp=d[(n)]; \
102 id2 = (data[id1] + tmp + id2) & 0xff; \
103 if (++id1 == len) id1=0; \
104 d[(n)]=d[id2]; \
105 d[id2]=tmp; }
106
107#if defined(OPENSSL_CPUID_OBJ) && !defined(OPENSSL_NO_ASM)
108# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \
109 defined(__INTEL__) || \
110 defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64)
111 if (sizeof(RC4_INT) > 1) {
112 /*
113 * Unlike all other x86 [and x86_64] implementations,
114 * Intel P4 core [including EM64T] was found to perform
115 * poorly with wider RC4_INT. Performance improvement
116 * for IA-32 hand-coded assembler turned out to be 2.8x
117 * if re-coded for RC4_CHAR! It's however inappropriate
118 * to just switch to RC4_CHAR for x86[_64], as non-P4
119 * implementations suffer from significant performance
120 * losses then, e.g. PIII exhibits >2x deterioration,
121 * and so does Opteron. In order to assure optimal
122 * all-round performance, let us [try to] detect P4 at
123 * run-time by checking upon HTT bit in CPU capability
124 * vector and set up compressed key schedule, which is
125 * recognized by correspondingly updated assembler
126 * module...
127 * <appro@fy.chalmers.se>
128 */
129 if (OPENSSL_ia32cap_P & (1<<28)) {
130 unsigned char *cp=(unsigned char *)d;
131
132 for (i=0;i<256;i++) cp[i]=i;
133 for (i=0;i<256;i++) SK_LOOP(cp,i);
134 /* mark schedule as compressed! */
135 d[256/sizeof(RC4_INT)]=-1;
136 return;
137 }
138 }
139# endif
140#endif
141 for (i=0; i < 256; i++) d[i]=i;
142 for (i=0; i < 256; i+=4)
143 {
144 SK_LOOP(d,i+0);
145 SK_LOOP(d,i+1);
146 SK_LOOP(d,i+2);
147 SK_LOOP(d,i+3);
148 }
149 }
150
diff --git a/src/lib/libcrypto/rc4/rc4_fblk.c b/src/lib/libcrypto/rc4/rc4_utl.c
index 1b2a42979b..ab3f02fe6a 100644
--- a/src/lib/libcrypto/rc4/rc4_fblk.c
+++ b/src/lib/libcrypto/rc4/rc4_utl.c
@@ -1,9 +1,6 @@
1/* crypto/rc4/rc4_fblk.c */ 1/* crypto/rc4/rc4_utl.c -*- mode:C; c-file-style: "eay" -*- */
2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3 * project.
4 */
5/* ==================================================================== 2/* ====================================================================
6 * Copyright (c) 2008 The OpenSSL Project. All rights reserved. 3 * Copyright (c) 2011 The OpenSSL Project. All rights reserved.
7 * 4 *
8 * Redistribution and use in source and binary forms, with or without 5 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions 6 * modification, are permitted provided that the following conditions
@@ -20,12 +17,12 @@
20 * 3. All advertising materials mentioning features or use of this 17 * 3. All advertising materials mentioning features or use of this
21 * software must display the following acknowledgment: 18 * software must display the following acknowledgment:
22 * "This product includes software developed by the OpenSSL Project 19 * "This product includes software developed by the OpenSSL Project
23 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" 20 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
24 * 21 *
25 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 22 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26 * endorse or promote products derived from this software without 23 * endorse or promote products derived from this software without
27 * prior written permission. For written permission, please contact 24 * prior written permission. For written permission, please contact
28 * licensing@OpenSSL.org. 25 * openssl-core@openssl.org.
29 * 26 *
30 * 5. Products derived from this software may not be called "OpenSSL" 27 * 5. Products derived from this software may not be called "OpenSSL"
31 * nor may "OpenSSL" appear in their names without prior written 28 * nor may "OpenSSL" appear in their names without prior written
@@ -34,7 +31,7 @@
34 * 6. Redistributions of any form whatsoever must retain the following 31 * 6. Redistributions of any form whatsoever must retain the following
35 * acknowledgment: 32 * acknowledgment:
36 * "This product includes software developed by the OpenSSL Project 33 * "This product includes software developed by the OpenSSL Project
37 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" 34 * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
38 * 35 *
39 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 36 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 37 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -49,27 +46,17 @@
49 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 46 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50 * OF THE POSSIBILITY OF SUCH DAMAGE. 47 * OF THE POSSIBILITY OF SUCH DAMAGE.
51 * ==================================================================== 48 * ====================================================================
49 *
52 */ 50 */
53 51
54
55#include <openssl/rc4.h>
56#include "rc4_locl.h"
57#include <openssl/opensslv.h> 52#include <openssl/opensslv.h>
58#include <openssl/crypto.h> 53#include <openssl/crypto.h>
59#ifdef OPENSSL_FIPS 54#include <openssl/rc4.h>
60#include <openssl/fips.h>
61#endif
62
63/* FIPS mode blocking for RC4 has to be done separately since RC4_set_key
64 * may be implemented in an assembly language file.
65 */
66 55
67#ifdef OPENSSL_FIPS
68void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data) 56void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data)
69 { 57 {
70 if (FIPS_mode()) 58#ifdef OPENSSL_FIPS
71 FIPS_BAD_ABORT(RC4) 59 fips_cipher_abort(RC4);
60#endif
72 private_RC4_set_key(key, len, data); 61 private_RC4_set_key(key, len, data);
73 } 62 }
74#endif
75
diff --git a/src/lib/libcrypto/rc4/rc4test.c b/src/lib/libcrypto/rc4/rc4test.c
index 4312605ccb..633a79e758 100644
--- a/src/lib/libcrypto/rc4/rc4test.c
+++ b/src/lib/libcrypto/rc4/rc4test.c
@@ -120,12 +120,6 @@ int main(int argc, char *argv[])
120 RC4_KEY key; 120 RC4_KEY key;
121 unsigned char obuf[512]; 121 unsigned char obuf[512];
122 122
123#if !defined(OPENSSL_PIC)
124 void OPENSSL_cpuid_setup(void);
125
126 OPENSSL_cpuid_setup();
127#endif
128
129 for (i=0; i<6; i++) 123 for (i=0; i<6; i++)
130 { 124 {
131 RC4_set_key(&key,keys[i][0],&(keys[i][1])); 125 RC4_set_key(&key,keys[i][0],&(keys[i][1]));
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-12-07 22:16:33 +0000