summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/rc4/asm
diff options
context:
space:
mode:
authorcvs2svn <admin@example.com>2012-07-13 17:49:56 +0000
committercvs2svn <admin@example.com>2012-07-13 17:49:56 +0000
commitee04221ea8063435416c7e6369e6eae76843aa71 (patch)
tree821921a1dd0a5a3cece91121e121cc63c4b68128 /src/lib/libcrypto/rc4/asm
parentadf6731f6e1d04718aee00cb93435143046aee9a (diff)
downloadopenbsd-eric_g2k12.tar.gz
openbsd-eric_g2k12.tar.bz2
openbsd-eric_g2k12.zip
This commit was manufactured by cvs2git to create tag 'eric_g2k12'.eric_g2k12
Diffstat (limited to '')
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-586.pl270
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-ia64.pl755
-rw-r--r--src/lib/libcrypto/rc4/asm/rc4-s390x.pl205
-rwxr-xr-xsrc/lib/libcrypto/rc4/asm/rc4-x86_64.pl504
4 files changed, 0 insertions, 1734 deletions
diff --git a/src/lib/libcrypto/rc4/asm/rc4-586.pl b/src/lib/libcrypto/rc4/asm/rc4-586.pl
deleted file mode 100644
index 38a44a70ef..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-586.pl
+++ /dev/null
@@ -1,270 +0,0 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# At some point it became apparent that the original SSLeay RC4
11# assembler implementation performs suboptimally on latest IA-32
12# microarchitectures. After re-tuning performance has changed as
13# following:
14#
15# Pentium -10%
16# Pentium III +12%
17# AMD +50%(*)
18# P4 +250%(**)
19#
20# (*) This number is actually a trade-off:-) It's possible to
21# achieve +72%, but at the cost of -48% off PIII performance.
22# In other words code performing further 13% faster on AMD
23# would perform almost 2 times slower on Intel PIII...
24# For reference! This code delivers ~80% of rc4-amd64.pl
25# performance on the same Opteron machine.
26# (**) This number requires compressed key schedule set up by
27# RC4_set_key [see commentary below for further details].
28#
29# <appro@fy.chalmers.se>
30
31$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
32push(@INC,"${dir}","${dir}../../perlasm");
33require "x86asm.pl";
34
35&asm_init($ARGV[0],"rc4-586.pl");
36
37$xx="eax";
38$yy="ebx";
39$tx="ecx";
40$ty="edx";
41$inp="esi";
42$out="ebp";
43$dat="edi";
44
45sub RC4_loop {
46 my $i=shift;
47 my $func = ($i==0)?*mov:*or;
48
49 &add (&LB($yy),&LB($tx));
50 &mov ($ty,&DWP(0,$dat,$yy,4));
51 &mov (&DWP(0,$dat,$yy,4),$tx);
52 &mov (&DWP(0,$dat,$xx,4),$ty);
53 &add ($ty,$tx);
54 &inc (&LB($xx));
55 &and ($ty,0xff);
56 &ror ($out,8) if ($i!=0);
57 if ($i<3) {
58 &mov ($tx,&DWP(0,$dat,$xx,4));
59 } else {
60 &mov ($tx,&wparam(3)); # reload [re-biased] out
61 }
62 &$func ($out,&DWP(0,$dat,$ty,4));
63}
64
65# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
66&function_begin("RC4");
67 &mov ($dat,&wparam(0)); # load key schedule pointer
68 &mov ($ty, &wparam(1)); # load len
69 &mov ($inp,&wparam(2)); # load inp
70 &mov ($out,&wparam(3)); # load out
71
72 &xor ($xx,$xx); # avoid partial register stalls
73 &xor ($yy,$yy);
74
75 &cmp ($ty,0); # safety net
76 &je (&label("abort"));
77
78 &mov (&LB($xx),&BP(0,$dat)); # load key->x
79 &mov (&LB($yy),&BP(4,$dat)); # load key->y
80 &add ($dat,8);
81
82 &lea ($tx,&DWP(0,$inp,$ty));
83 &sub ($out,$inp); # re-bias out
84 &mov (&wparam(1),$tx); # save input+len
85
86 &inc (&LB($xx));
87
88 # detect compressed key schedule...
89 &cmp (&DWP(256,$dat),-1);
90 &je (&label("RC4_CHAR"));
91
92 &mov ($tx,&DWP(0,$dat,$xx,4));
93
94 &and ($ty,-4); # how many 4-byte chunks?
95 &jz (&label("loop1"));
96
97 &lea ($ty,&DWP(-4,$inp,$ty));
98 &mov (&wparam(2),$ty); # save input+(len/4)*4-4
99 &mov (&wparam(3),$out); # $out as accumulator in this loop
100
101 &set_label("loop4",16);
102 for ($i=0;$i<4;$i++) { RC4_loop($i); }
103 &ror ($out,8);
104 &xor ($out,&DWP(0,$inp));
105 &cmp ($inp,&wparam(2)); # compare to input+(len/4)*4-4
106 &mov (&DWP(0,$tx,$inp),$out);# $tx holds re-biased out here
107 &lea ($inp,&DWP(4,$inp));
108 &mov ($tx,&DWP(0,$dat,$xx,4));
109 &jb (&label("loop4"));
110
111 &cmp ($inp,&wparam(1)); # compare to input+len
112 &je (&label("done"));
113 &mov ($out,&wparam(3)); # restore $out
114
115 &set_label("loop1",16);
116 &add (&LB($yy),&LB($tx));
117 &mov ($ty,&DWP(0,$dat,$yy,4));
118 &mov (&DWP(0,$dat,$yy,4),$tx);
119 &mov (&DWP(0,$dat,$xx,4),$ty);
120 &add ($ty,$tx);
121 &inc (&LB($xx));
122 &and ($ty,0xff);
123 &mov ($ty,&DWP(0,$dat,$ty,4));
124 &xor (&LB($ty),&BP(0,$inp));
125 &lea ($inp,&DWP(1,$inp));
126 &mov ($tx,&DWP(0,$dat,$xx,4));
127 &cmp ($inp,&wparam(1)); # compare to input+len
128 &mov (&BP(-1,$out,$inp),&LB($ty));
129 &jb (&label("loop1"));
130
131 &jmp (&label("done"));
132
133# this is essentially Intel P4 specific codepath...
134&set_label("RC4_CHAR",16);
135 &movz ($tx,&BP(0,$dat,$xx));
136 # strangely enough unrolled loop performs over 20% slower...
137 &set_label("cloop1");
138 &add (&LB($yy),&LB($tx));
139 &movz ($ty,&BP(0,$dat,$yy));
140 &mov (&BP(0,$dat,$yy),&LB($tx));
141 &mov (&BP(0,$dat,$xx),&LB($ty));
142 &add (&LB($ty),&LB($tx));
143 &movz ($ty,&BP(0,$dat,$ty));
144 &add (&LB($xx),1);
145 &xor (&LB($ty),&BP(0,$inp));
146 &lea ($inp,&DWP(1,$inp));
147 &movz ($tx,&BP(0,$dat,$xx));
148 &cmp ($inp,&wparam(1));
149 &mov (&BP(-1,$out,$inp),&LB($ty));
150 &jb (&label("cloop1"));
151
152&set_label("done");
153 &dec (&LB($xx));
154 &mov (&BP(-4,$dat),&LB($yy)); # save key->y
155 &mov (&BP(-8,$dat),&LB($xx)); # save key->x
156&set_label("abort");
157&function_end("RC4");
158
159########################################################################
160
161$inp="esi";
162$out="edi";
163$idi="ebp";
164$ido="ecx";
165$idx="edx";
166
167&external_label("OPENSSL_ia32cap_P");
168
169# void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data);
170&function_begin("RC4_set_key");
171 &mov ($out,&wparam(0)); # load key
172 &mov ($idi,&wparam(1)); # load len
173 &mov ($inp,&wparam(2)); # load data
174 &picmeup($idx,"OPENSSL_ia32cap_P");
175
176 &lea ($out,&DWP(2*4,$out)); # &key->data
177 &lea ($inp,&DWP(0,$inp,$idi)); # $inp to point at the end
178 &neg ($idi);
179 &xor ("eax","eax");
180 &mov (&DWP(-4,$out),$idi); # borrow key->y
181
182 &bt (&DWP(0,$idx),20); # check for bit#20
183 &jc (&label("c1stloop"));
184
185&set_label("w1stloop",16);
186 &mov (&DWP(0,$out,"eax",4),"eax"); # key->data[i]=i;
187 &add (&LB("eax"),1); # i++;
188 &jnc (&label("w1stloop"));
189
190 &xor ($ido,$ido);
191 &xor ($idx,$idx);
192
193&set_label("w2ndloop",16);
194 &mov ("eax",&DWP(0,$out,$ido,4));
195 &add (&LB($idx),&BP(0,$inp,$idi));
196 &add (&LB($idx),&LB("eax"));
197 &add ($idi,1);
198 &mov ("ebx",&DWP(0,$out,$idx,4));
199 &jnz (&label("wnowrap"));
200 &mov ($idi,&DWP(-4,$out));
201 &set_label("wnowrap");
202 &mov (&DWP(0,$out,$idx,4),"eax");
203 &mov (&DWP(0,$out,$ido,4),"ebx");
204 &add (&LB($ido),1);
205 &jnc (&label("w2ndloop"));
206&jmp (&label("exit"));
207
208# Unlike all other x86 [and x86_64] implementations, Intel P4 core
209# [including EM64T] was found to perform poorly with above "32-bit" key
210# schedule, a.k.a. RC4_INT. Performance improvement for IA-32 hand-coded
211# assembler turned out to be 3.5x if re-coded for compressed 8-bit one,
212# a.k.a. RC4_CHAR! It's however inappropriate to just switch to 8-bit
213# schedule for x86[_64], because non-P4 implementations suffer from
214# significant performance losses then, e.g. PIII exhibits >2x
215# deterioration, and so does Opteron. In order to assure optimal
216# all-round performance, we detect P4 at run-time and set up compressed
217# key schedule, which is recognized by RC4 procedure.
218
219&set_label("c1stloop",16);
220 &mov (&BP(0,$out,"eax"),&LB("eax")); # key->data[i]=i;
221 &add (&LB("eax"),1); # i++;
222 &jnc (&label("c1stloop"));
223
224 &xor ($ido,$ido);
225 &xor ($idx,$idx);
226 &xor ("ebx","ebx");
227
228&set_label("c2ndloop",16);
229 &mov (&LB("eax"),&BP(0,$out,$ido));
230 &add (&LB($idx),&BP(0,$inp,$idi));
231 &add (&LB($idx),&LB("eax"));
232 &add ($idi,1);
233 &mov (&LB("ebx"),&BP(0,$out,$idx));
234 &jnz (&label("cnowrap"));
235 &mov ($idi,&DWP(-4,$out));
236 &set_label("cnowrap");
237 &mov (&BP(0,$out,$idx),&LB("eax"));
238 &mov (&BP(0,$out,$ido),&LB("ebx"));
239 &add (&LB($ido),1);
240 &jnc (&label("c2ndloop"));
241
242 &mov (&DWP(256,$out),-1); # mark schedule as compressed
243
244&set_label("exit");
245 &xor ("eax","eax");
246 &mov (&DWP(-8,$out),"eax"); # key->x=0;
247 &mov (&DWP(-4,$out),"eax"); # key->y=0;
248&function_end("RC4_set_key");
249
250# const char *RC4_options(void);
251&function_begin_B("RC4_options");
252 &call (&label("pic_point"));
253&set_label("pic_point");
254 &blindpop("eax");
255 &lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax"));
256 &picmeup("edx","OPENSSL_ia32cap_P");
257 &bt (&DWP(0,"edx"),20);
258 &jnc (&label("skip"));
259 &add ("eax",12);
260 &set_label("skip");
261 &ret ();
262&set_label("opts",64);
263&asciz ("rc4(4x,int)");
264&asciz ("rc4(1x,char)");
265&asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>");
266&align (64);
267&function_end_B("RC4_options");
268
269&asm_finish();
270
diff --git a/src/lib/libcrypto/rc4/asm/rc4-ia64.pl b/src/lib/libcrypto/rc4/asm/rc4-ia64.pl
deleted file mode 100644
index 49cd5b5e69..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-ia64.pl
+++ /dev/null
@@ -1,755 +0,0 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by David Mosberger <David.Mosberger@acm.org> based on the
5# Itanium optimized Crypto code which was released by HP Labs at
6# http://www.hpl.hp.com/research/linux/crypto/.
7#
8# Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
9#
10# Permission is hereby granted, free of charge, to any person obtaining
11# a copy of this software and associated documentation files (the
12# "Software"), to deal in the Software without restriction, including
13# without limitation the rights to use, copy, modify, merge, publish,
14# distribute, sublicense, and/or sell copies of the Software, and to
15# permit persons to whom the Software is furnished to do so, subject to
16# the following conditions:
17#
18# The above copyright notice and this permission notice shall be
19# included in all copies or substantial portions of the Software.
20
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
25# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
26# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
27# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
28
29
30
31# This is a little helper program which generates a software-pipelined
32# for RC4 encryption. The basic algorithm looks like this:
33#
34# for (counter = 0; counter < len; ++counter)
35# {
36# in = inp[counter];
37# SI = S[I];
38# J = (SI + J) & 0xff;
39# SJ = S[J];
40# T = (SI + SJ) & 0xff;
41# S[I] = SJ, S[J] = SI;
42# ST = S[T];
43# outp[counter] = in ^ ST;
44# I = (I + 1) & 0xff;
45# }
46#
47# Pipelining this loop isn't easy, because the stores to the S[] array
48# need to be observed in the right order. The loop generated by the
49# code below has the following pipeline diagram:
50#
51# cycle
52# | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |10 |11 |12 |13 |14 |15 |16 |17 |
53# iter
54# 1: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
55# 2: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
56# 3: xxx LDI xxx xxx xxx LDJ xxx SWP xxx LDT xxx xxx
57#
58# where:
59# LDI = load of S[I]
60# LDJ = load of S[J]
61# SWP = swap of S[I] and S[J]
62# LDT = load of S[T]
63#
64# Note that in the above diagram, the major trouble-spot is that LDI
65# of the 2nd iteration is performed BEFORE the SWP of the first
66# iteration. Fortunately, this is easy to detect (I of the 1st
67# iteration will be equal to J of the 2nd iteration) and when this
68# happens, we simply forward the proper value from the 1st iteration
69# to the 2nd one. The proper value in this case is simply the value
70# of S[I] from the first iteration (thanks to the fact that SWP
71# simply swaps the contents of S[I] and S[J]).
72#
73# Another potential trouble-spot is in cycle 7, where SWP of the 1st
74# iteration issues at the same time as the LDI of the 3rd iteration.
75# However, thanks to IA-64 execution semantics, this can be taken
76# care of simply by placing LDI later in the instruction-group than
77# SWP. IA-64 CPUs will automatically forward the value if they
78# detect that the SWP and LDI are accessing the same memory-location.
79
80# The core-loop that can be pipelined then looks like this (annotated
81# with McKinley/Madison issue port & latency numbers, assuming L1
82# cache hits for the most part):
83
84# operation: instruction: issue-ports: latency
85# ------------------ ----------------------------- ------------- -------
86
87# Data = *inp++ ld1 data = [inp], 1 M0-M1 1 cyc c0
88# shladd Iptr = I, KeyTable, 3 M0-M3, I0, I1 1 cyc
89# I = (I + 1) & 0xff padd1 nextI = I, one M0-M3, I0, I1 3 cyc
90# ;;
91# SI = S[I] ld8 SI = [Iptr] M0-M1 1 cyc c1 * after SWAP!
92# ;;
93# cmp.eq.unc pBypass = I, J * after J is valid!
94# J = SI + J add J = J, SI M0-M3, I0, I1 1 cyc c2
95# (pBypass) br.cond.spnt Bypass
96# ;;
97# ---------------------------------------------------------------------------------------
98# J = J & 0xff zxt1 J = J I0, I1, 1 cyc c3
99# ;;
100# shladd Jptr = J, KeyTable, 3 M0-M3, I0, I1 1 cyc c4
101# ;;
102# SJ = S[J] ld8 SJ = [Jptr] M0-M1 1 cyc c5
103# ;;
104# ---------------------------------------------------------------------------------------
105# T = (SI + SJ) add T = SI, SJ M0-M3, I0, I1 1 cyc c6
106# ;;
107# T = T & 0xff zxt1 T = T I0, I1 1 cyc
108# S[I] = SJ st8 [Iptr] = SJ M2-M3 c7
109# S[J] = SI st8 [Jptr] = SI M2-M3
110# ;;
111# shladd Tptr = T, KeyTable, 3 M0-M3, I0, I1 1 cyc c8
112# ;;
113# ---------------------------------------------------------------------------------------
114# T = S[T] ld8 T = [Tptr] M0-M1 1 cyc c9
115# ;;
116# data ^= T xor data = data, T M0-M3, I0, I1 1 cyc c10
117# ;;
118# *out++ = Data ^ T dep word = word, data, 8, POS I0, I1 1 cyc c11
119# ;;
120# ---------------------------------------------------------------------------------------
121
122# There are several points worth making here:
123
124# - Note that due to the bypass/forwarding-path, the first two
125# phases of the loop are strangly mingled together. In
126# particular, note that the first stage of the pipeline is
127# using the value of "J", as calculated by the second stage.
128# - Each bundle-pair will have exactly 6 instructions.
129# - Pipelined, the loop can execute in 3 cycles/iteration and
130# 4 stages. However, McKinley/Madison can issue "st1" to
131# the same bank at a rate of at most one per 4 cycles. Thus,
132# instead of storing each byte, we accumulate them in a word
133# and then write them back at once with a single "st8" (this
134# implies that the setup code needs to ensure that the output
135# buffer is properly aligned, if need be, by encoding the
136# first few bytes separately).
137# - There is no space for a "br.ctop" instruction. For this
138# reason we can't use module-loop support in IA-64 and have
139# to do a traditional, purely software-pipelined loop.
140# - We can't replace any of the remaining "add/zxt1" pairs with
141# "padd1" because the latency for that instruction is too high
142# and would push the loop to the point where more bypasses
143# would be needed, which we don't have space for.
144# - The above loop runs at around 3.26 cycles/byte, or roughly
145# 440 MByte/sec on a 1.5GHz Madison. This is well below the
146# system bus bandwidth and hence with judicious use of
147# "lfetch" this loop can run at (almost) peak speed even when
148# the input and output data reside in memory. The
149# max. latency that can be tolerated is (PREFETCH_DISTANCE *
150# L2_LINE_SIZE * 3 cyc), or about 384 cycles assuming (at
151# least) 1-ahead prefetching of 128 byte cache-lines. Note
152# that we do NOT prefetch into L1, since that would only
153# interfere with the S[] table values stored there. This is
154# acceptable because there is a 10 cycle latency between
155# load and first use of the input data.
156# - We use a branch to out-of-line bypass-code of cycle-pressure:
157# we calculate the next J, check for the need to activate the
158# bypass path, and activate the bypass path ALL IN THE SAME
159# CYCLE. If we didn't have these constraints, we could do
160# the bypass with a simple conditional move instruction.
161# Fortunately, the bypass paths get activated relatively
162# infrequently, so the extra branches don't cost all that much
163# (about 0.04 cycles/byte, measured on a 16396 byte file with
164# random input data).
165#
166
167$phases = 4; # number of stages/phases in the pipelined-loop
168$unroll_count = 6; # number of times we unrolled it
169$pComI = (1 << 0);
170$pComJ = (1 << 1);
171$pComT = (1 << 2);
172$pOut = (1 << 3);
173
174$NData = 4;
175$NIP = 3;
176$NJP = 2;
177$NI = 2;
178$NSI = 3;
179$NSJ = 2;
180$NT = 2;
181$NOutWord = 2;
182
183#
184# $threshold is the minimum length before we attempt to use the
185# big software-pipelined loop. It MUST be greater-or-equal
186# to:
187# PHASES * (UNROLL_COUNT + 1) + 7
188#
189# The "+ 7" comes from the fact we may have to encode up to
190# 7 bytes separately before the output pointer is aligned.
191#
192$threshold = (3 * ($phases * ($unroll_count + 1)) + 7);
193
194sub I {
195 local *code = shift;
196 local $format = shift;
197 $code .= sprintf ("\t\t".$format."\n", @_);
198}
199
200sub P {
201 local *code = shift;
202 local $format = shift;
203 $code .= sprintf ($format."\n", @_);
204}
205
206sub STOP {
207 local *code = shift;
208 $code .=<<___;
209 ;;
210___
211}
212
213sub emit_body {
214 local *c = shift;
215 local *bypass = shift;
216 local ($iteration, $p) = @_;
217
218 local $i0 = $iteration;
219 local $i1 = $iteration - 1;
220 local $i2 = $iteration - 2;
221 local $i3 = $iteration - 3;
222 local $iw0 = ($iteration - 3) / 8;
223 local $iw1 = ($iteration > 3) ? ($iteration - 4) / 8 : 1;
224 local $byte_num = ($iteration - 3) % 8;
225 local $label = $iteration + 1;
226 local $pAny = ($p & 0xf) == 0xf;
227 local $pByp = (($p & $pComI) && ($iteration > 0));
228
229 $c.=<<___;
230//////////////////////////////////////////////////
231___
232
233 if (($p & 0xf) == 0) {
234 $c.="#ifdef HOST_IS_BIG_ENDIAN\n";
235 &I(\$c,"shr.u OutWord[%u] = OutWord[%u], 32;;",
236 $iw1 % $NOutWord, $iw1 % $NOutWord);
237 $c.="#endif\n";
238 &I(\$c, "st4 [OutPtr] = OutWord[%u], 4", $iw1 % $NOutWord);
239 return;
240 }
241
242 # Cycle 0
243 &I(\$c, "{ .mmi") if ($pAny);
244 &I(\$c, "ld1 Data[%u] = [InPtr], 1", $i0 % $NData) if ($p & $pComI);
245 &I(\$c, "padd1 I[%u] = One, I[%u]", $i0 % $NI, $i1 % $NI)if ($p & $pComI);
246 &I(\$c, "zxt1 J = J") if ($p & $pComJ);
247 &I(\$c, "}") if ($pAny);
248 &I(\$c, "{ .mmi") if ($pAny);
249 &I(\$c, "LKEY T[%u] = [T[%u]]", $i1 % $NT, $i1 % $NT) if ($p & $pOut);
250 &I(\$c, "add T[%u] = SI[%u], SJ[%u]",
251 $i0 % $NT, $i2 % $NSI, $i1 % $NSJ) if ($p & $pComT);
252 &I(\$c, "KEYADDR(IPr[%u], I[%u])", $i0 % $NIP, $i1 % $NI) if ($p & $pComI);
253 &I(\$c, "}") if ($pAny);
254 &STOP(\$c);
255
256 # Cycle 1
257 &I(\$c, "{ .mmi") if ($pAny);
258 &I(\$c, "SKEY [IPr[%u]] = SJ[%u]", $i2 % $NIP, $i1%$NSJ)if ($p & $pComT);
259 &I(\$c, "SKEY [JP[%u]] = SI[%u]", $i1 % $NJP, $i2%$NSI) if ($p & $pComT);
260 &I(\$c, "zxt1 T[%u] = T[%u]", $i0 % $NT, $i0 % $NT) if ($p & $pComT);
261 &I(\$c, "}") if ($pAny);
262 &I(\$c, "{ .mmi") if ($pAny);
263 &I(\$c, "LKEY SI[%u] = [IPr[%u]]", $i0 % $NSI, $i0%$NIP)if ($p & $pComI);
264 &I(\$c, "KEYADDR(JP[%u], J)", $i0 % $NJP) if ($p & $pComJ);
265 &I(\$c, "xor Data[%u] = Data[%u], T[%u]",
266 $i3 % $NData, $i3 % $NData, $i1 % $NT) if ($p & $pOut);
267 &I(\$c, "}") if ($pAny);
268 &STOP(\$c);
269
270 # Cycle 2
271 &I(\$c, "{ .mmi") if ($pAny);
272 &I(\$c, "LKEY SJ[%u] = [JP[%u]]", $i0 % $NSJ, $i0%$NJP) if ($p & $pComJ);
273 &I(\$c, "cmp.eq pBypass, p0 = I[%u], J", $i1 % $NI) if ($pByp);
274 &I(\$c, "dep OutWord[%u] = Data[%u], OutWord[%u], BYTE_POS(%u), 8",
275 $iw0%$NOutWord, $i3%$NData, $iw1%$NOutWord, $byte_num) if ($p & $pOut);
276 &I(\$c, "}") if ($pAny);
277 &I(\$c, "{ .mmb") if ($pAny);
278 &I(\$c, "add J = J, SI[%u]", $i0 % $NSI) if ($p & $pComI);
279 &I(\$c, "KEYADDR(T[%u], T[%u])", $i0 % $NT, $i0 % $NT) if ($p & $pComT);
280 &P(\$c, "(pBypass)\tbr.cond.spnt.many .rc4Bypass%u",$label)if ($pByp);
281 &I(\$c, "}") if ($pAny);
282 &STOP(\$c);
283
284 &P(\$c, ".rc4Resume%u:", $label) if ($pByp);
285 if ($byte_num == 0 && $iteration >= $phases) {
286 &I(\$c, "st8 [OutPtr] = OutWord[%u], 8",
287 $iw1 % $NOutWord) if ($p & $pOut);
288 if ($iteration == (1 + $unroll_count) * $phases - 1) {
289 if ($unroll_count == 6) {
290 &I(\$c, "mov OutWord[%u] = OutWord[%u]",
291 $iw1 % $NOutWord, $iw0 % $NOutWord);
292 }
293 &I(\$c, "lfetch.nt1 [InPrefetch], %u",
294 $unroll_count * $phases);
295 &I(\$c, "lfetch.excl.nt1 [OutPrefetch], %u",
296 $unroll_count * $phases);
297 &I(\$c, "br.cloop.sptk.few .rc4Loop");
298 }
299 }
300
301 if ($pByp) {
302 &P(\$bypass, ".rc4Bypass%u:", $label);
303 &I(\$bypass, "sub J = J, SI[%u]", $i0 % $NSI);
304 &I(\$bypass, "nop 0");
305 &I(\$bypass, "nop 0");
306 &I(\$bypass, ";;");
307 &I(\$bypass, "add J = J, SI[%u]", $i1 % $NSI);
308 &I(\$bypass, "mov SI[%u] = SI[%u]", $i0 % $NSI, $i1 % $NSI);
309 &I(\$bypass, "br.sptk.many .rc4Resume%u\n", $label);
310 &I(\$bypass, ";;");
311 }
312}
313
314$code=<<___;
315.ident \"rc4-ia64.s, version 3.0\"
316.ident \"Copyright (c) 2005 Hewlett-Packard Development Company, L.P.\"
317
318#define LCSave r8
319#define PRSave r9
320
321/* Inputs become invalid once rotation begins! */
322
323#define StateTable in0
324#define DataLen in1
325#define InputBuffer in2
326#define OutputBuffer in3
327
328#define KTable r14
329#define J r15
330#define InPtr r16
331#define OutPtr r17
332#define InPrefetch r18
333#define OutPrefetch r19
334#define One r20
335#define LoopCount r21
336#define Remainder r22
337#define IFinal r23
338#define EndPtr r24
339
340#define tmp0 r25
341#define tmp1 r26
342
343#define pBypass p6
344#define pDone p7
345#define pSmall p8
346#define pAligned p9
347#define pUnaligned p10
348
349#define pComputeI pPhase[0]
350#define pComputeJ pPhase[1]
351#define pComputeT pPhase[2]
352#define pOutput pPhase[3]
353
354#define RetVal r8
355#define L_OK p7
356#define L_NOK p8
357
358#define _NINPUTS 4
359#define _NOUTPUT 0
360
361#define _NROTATE 24
362#define _NLOCALS (_NROTATE - _NINPUTS - _NOUTPUT)
363
364#ifndef SZ
365# define SZ 4 // this must be set to sizeof(RC4_INT)
366#endif
367
368#if SZ == 1
369# define LKEY ld1
370# define SKEY st1
371# define KEYADDR(dst, i) add dst = i, KTable
372#elif SZ == 2
373# define LKEY ld2
374# define SKEY st2
375# define KEYADDR(dst, i) shladd dst = i, 1, KTable
376#elif SZ == 4
377# define LKEY ld4
378# define SKEY st4
379# define KEYADDR(dst, i) shladd dst = i, 2, KTable
380#else
381# define LKEY ld8
382# define SKEY st8
383# define KEYADDR(dst, i) shladd dst = i, 3, KTable
384#endif
385
386#if defined(_HPUX_SOURCE) && !defined(_LP64)
387# define ADDP addp4
388#else
389# define ADDP add
390#endif
391
392/* Define a macro for the bit number of the n-th byte: */
393
394#if defined(_HPUX_SOURCE) || defined(B_ENDIAN)
395# define HOST_IS_BIG_ENDIAN
396# define BYTE_POS(n) (56 - (8 * (n)))
397#else
398# define BYTE_POS(n) (8 * (n))
399#endif
400
401/*
402 We must perform the first phase of the pipeline explicitly since
403 we will always load from the stable the first time. The br.cexit
404 will never be taken since regardless of the number of bytes because
405 the epilogue count is 4.
406*/
407/* MODSCHED_RC4 macro was split to _PROLOGUE and _LOOP, because HP-UX
408 assembler failed on original macro with syntax error. <appro> */
409#define MODSCHED_RC4_PROLOGUE \\
410 { \\
411 ld1 Data[0] = [InPtr], 1; \\
412 add IFinal = 1, I[1]; \\
413 KEYADDR(IPr[0], I[1]); \\
414 } ;; \\
415 { \\
416 LKEY SI[0] = [IPr[0]]; \\
417 mov pr.rot = 0x10000; \\
418 mov ar.ec = 4; \\
419 } ;; \\
420 { \\
421 add J = J, SI[0]; \\
422 zxt1 I[0] = IFinal; \\
423 br.cexit.spnt.few .+16; /* never taken */ \\
424 } ;;
425#define MODSCHED_RC4_LOOP(label) \\
426label: \\
427 { .mmi; \\
428 (pComputeI) ld1 Data[0] = [InPtr], 1; \\
429 (pComputeI) add IFinal = 1, I[1]; \\
430 (pComputeJ) zxt1 J = J; \\
431 }{ .mmi; \\
432 (pOutput) LKEY T[1] = [T[1]]; \\
433 (pComputeT) add T[0] = SI[2], SJ[1]; \\
434 (pComputeI) KEYADDR(IPr[0], I[1]); \\
435 } ;; \\
436 { .mmi; \\
437 (pComputeT) SKEY [IPr[2]] = SJ[1]; \\
438 (pComputeT) SKEY [JP[1]] = SI[2]; \\
439 (pComputeT) zxt1 T[0] = T[0]; \\
440 }{ .mmi; \\
441 (pComputeI) LKEY SI[0] = [IPr[0]]; \\
442 (pComputeJ) KEYADDR(JP[0], J); \\
443 (pComputeI) cmp.eq.unc pBypass, p0 = I[1], J; \\
444 } ;; \\
445 { .mmi; \\
446 (pComputeJ) LKEY SJ[0] = [JP[0]]; \\
447 (pOutput) xor Data[3] = Data[3], T[1]; \\
448 nop 0x0; \\
449 }{ .mmi; \\
450 (pComputeT) KEYADDR(T[0], T[0]); \\
451 (pBypass) mov SI[0] = SI[1]; \\
452 (pComputeI) zxt1 I[0] = IFinal; \\
453 } ;; \\
454 { .mmb; \\
455 (pOutput) st1 [OutPtr] = Data[3], 1; \\
456 (pComputeI) add J = J, SI[0]; \\
457 br.ctop.sptk.few label; \\
458 } ;;
459
460 .text
461
462 .align 32
463
464 .type RC4, \@function
465 .global RC4
466
467 .proc RC4
468 .prologue
469
470RC4:
471 {
472 .mmi
473 alloc r2 = ar.pfs, _NINPUTS, _NLOCALS, _NOUTPUT, _NROTATE
474
475 .rotr Data[4], I[2], IPr[3], SI[3], JP[2], SJ[2], T[2], \\
476 OutWord[2]
477 .rotp pPhase[4]
478
479 ADDP InPrefetch = 0, InputBuffer
480 ADDP KTable = 0, StateTable
481 }
482 {
483 .mmi
484 ADDP InPtr = 0, InputBuffer
485 ADDP OutPtr = 0, OutputBuffer
486 mov RetVal = r0
487 }
488 ;;
489 {
490 .mmi
491 lfetch.nt1 [InPrefetch], 0x80
492 ADDP OutPrefetch = 0, OutputBuffer
493 }
494 { // Return 0 if the input length is nonsensical
495 .mib
496 ADDP StateTable = 0, StateTable
497 cmp.ge.unc L_NOK, L_OK = r0, DataLen
498 (L_NOK) br.ret.sptk.few rp
499 }
500 ;;
501 {
502 .mib
503 cmp.eq.or L_NOK, L_OK = r0, InPtr
504 cmp.eq.or L_NOK, L_OK = r0, OutPtr
505 nop 0x0
506 }
507 {
508 .mib
509 cmp.eq.or L_NOK, L_OK = r0, StateTable
510 nop 0x0
511 (L_NOK) br.ret.sptk.few rp
512 }
513 ;;
514 LKEY I[1] = [KTable], SZ
515/* Prefetch the state-table. It contains 256 elements of size SZ */
516
517#if SZ == 1
518 ADDP tmp0 = 1*128, StateTable
519#elif SZ == 2
520 ADDP tmp0 = 3*128, StateTable
521 ADDP tmp1 = 2*128, StateTable
522#elif SZ == 4
523 ADDP tmp0 = 7*128, StateTable
524 ADDP tmp1 = 6*128, StateTable
525#elif SZ == 8
526 ADDP tmp0 = 15*128, StateTable
527 ADDP tmp1 = 14*128, StateTable
528#endif
529 ;;
530#if SZ >= 8
531 lfetch.fault.nt1 [tmp0], -256 // 15
532 lfetch.fault.nt1 [tmp1], -256;;
533 lfetch.fault.nt1 [tmp0], -256 // 13
534 lfetch.fault.nt1 [tmp1], -256;;
535 lfetch.fault.nt1 [tmp0], -256 // 11
536 lfetch.fault.nt1 [tmp1], -256;;
537 lfetch.fault.nt1 [tmp0], -256 // 9
538 lfetch.fault.nt1 [tmp1], -256;;
539#endif
540#if SZ >= 4
541 lfetch.fault.nt1 [tmp0], -256 // 7
542 lfetch.fault.nt1 [tmp1], -256;;
543 lfetch.fault.nt1 [tmp0], -256 // 5
544 lfetch.fault.nt1 [tmp1], -256;;
545#endif
546#if SZ >= 2
547 lfetch.fault.nt1 [tmp0], -256 // 3
548 lfetch.fault.nt1 [tmp1], -256;;
549#endif
550 {
551 .mii
552 lfetch.fault.nt1 [tmp0] // 1
553 add I[1]=1,I[1];;
554 zxt1 I[1]=I[1]
555 }
556 {
557 .mmi
558 lfetch.nt1 [InPrefetch], 0x80
559 lfetch.excl.nt1 [OutPrefetch], 0x80
560 .save pr, PRSave
561 mov PRSave = pr
562 } ;;
563 {
564 .mmi
565 lfetch.excl.nt1 [OutPrefetch], 0x80
566 LKEY J = [KTable], SZ
567 ADDP EndPtr = DataLen, InPtr
568 } ;;
569 {
570 .mmi
571 ADDP EndPtr = -1, EndPtr // Make it point to
572 // last data byte.
573 mov One = 1
574 .save ar.lc, LCSave
575 mov LCSave = ar.lc
576 .body
577 } ;;
578 {
579 .mmb
580 sub Remainder = 0, OutPtr
581 cmp.gtu pSmall, p0 = $threshold, DataLen
582(pSmall) br.cond.dpnt .rc4Remainder // Data too small for
583 // big loop.
584 } ;;
585 {
586 .mmi
587 and Remainder = 0x7, Remainder
588 ;;
589 cmp.eq pAligned, pUnaligned = Remainder, r0
590 nop 0x0
591 } ;;
592 {
593 .mmb
594.pred.rel "mutex",pUnaligned,pAligned
595(pUnaligned) add Remainder = -1, Remainder
596(pAligned) sub Remainder = EndPtr, InPtr
597(pAligned) br.cond.dptk.many .rc4Aligned
598 } ;;
599 {
600 .mmi
601 nop 0x0
602 nop 0x0
603 mov.i ar.lc = Remainder
604 }
605
606/* Do the initial few bytes via the compact, modulo-scheduled loop
607 until the output pointer is 8-byte-aligned. */
608
609 MODSCHED_RC4_PROLOGUE
610 MODSCHED_RC4_LOOP(.RC4AlignLoop)
611
612 {
613 .mib
614 sub Remainder = EndPtr, InPtr
615 zxt1 IFinal = IFinal
616 clrrrb // Clear CFM.rrb.pr so
617 ;; // next "mov pr.rot = N"
618 // does the right thing.
619 }
620 {
621 .mmi
622 mov I[1] = IFinal
623 nop 0x0
624 nop 0x0
625 } ;;
626
627
628.rc4Aligned:
629
630/*
631 Unrolled loop count = (Remainder - ($unroll_count+1)*$phases)/($unroll_count*$phases)
632 */
633
634 {
635 .mlx
636 add LoopCount = 1 - ($unroll_count + 1)*$phases, Remainder
637 movl Remainder = 0xaaaaaaaaaaaaaaab
638 } ;;
639 {
640 .mmi
641 setf.sig f6 = LoopCount // M2, M3 6 cyc
642 setf.sig f7 = Remainder // M2, M3 6 cyc
643 nop 0x0
644 } ;;
645 {
646 .mfb
647 nop 0x0
648 xmpy.hu f6 = f6, f7
649 nop 0x0
650 } ;;
651 {
652 .mmi
653 getf.sig LoopCount = f6;; // M2 5 cyc
654 nop 0x0
655 shr.u LoopCount = LoopCount, 4
656 } ;;
657 {
658 .mmi
659 nop 0x0
660 nop 0x0
661 mov.i ar.lc = LoopCount
662 } ;;
663
664/* Now comes the unrolled loop: */
665
666.rc4Prologue:
667___
668
669$iteration = 0;
670
671# Generate the prologue:
672$predicates = 1;
673for ($i = 0; $i < $phases; ++$i) {
674 &emit_body (\$code, \$bypass, $iteration++, $predicates);
675 $predicates = ($predicates << 1) | 1;
676}
677
678$code.=<<___;
679.rc4Loop:
680___
681
682# Generate the body:
683for ($i = 0; $i < $unroll_count*$phases; ++$i) {
684 &emit_body (\$code, \$bypass, $iteration++, $predicates);
685}
686
687$code.=<<___;
688.rc4Epilogue:
689___
690
691# Generate the epilogue:
692for ($i = 0; $i < $phases; ++$i) {
693 $predicates <<= 1;
694 &emit_body (\$code, \$bypass, $iteration++, $predicates);
695}
696
697$code.=<<___;
698 {
699 .mmi
700 lfetch.nt1 [EndPtr] // fetch line with last byte
701 mov IFinal = I[1]
702 nop 0x0
703 }
704
705.rc4Remainder:
706 {
707 .mmi
708 sub Remainder = EndPtr, InPtr // Calculate
709 // # of bytes
710 // left - 1
711 nop 0x0
712 nop 0x0
713 } ;;
714 {
715 .mib
716 cmp.eq pDone, p0 = -1, Remainder // done already?
717 mov.i ar.lc = Remainder
718(pDone) br.cond.dptk.few .rc4Complete
719 }
720
721/* Do the remaining bytes via the compact, modulo-scheduled loop */
722
723 MODSCHED_RC4_PROLOGUE
724 MODSCHED_RC4_LOOP(.RC4RestLoop)
725
726.rc4Complete:
727 {
728 .mmi
729 add KTable = -SZ, KTable
730 add IFinal = -1, IFinal
731 mov ar.lc = LCSave
732 } ;;
733 {
734 .mii
735 SKEY [KTable] = J,-SZ
736 zxt1 IFinal = IFinal
737 mov pr = PRSave, 0x1FFFF
738 } ;;
739 {
740 .mib
741 SKEY [KTable] = IFinal
742 add RetVal = 1, r0
743 br.ret.sptk.few rp
744 } ;;
745___
746
747# Last but not least, emit the code for the bypass-code of the unrolled loop:
748
749$code.=$bypass;
750
751$code.=<<___;
752 .endp RC4
753___
754
755print $code;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-s390x.pl b/src/lib/libcrypto/rc4/asm/rc4-s390x.pl
deleted file mode 100644
index 96681fa05e..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-s390x.pl
+++ /dev/null
@@ -1,205 +0,0 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# February 2009
11#
12# Performance is 2x of gcc 3.4.6 on z10. Coding "secret" is to
13# "cluster" Address Generation Interlocks, so that one pipeline stall
14# resolves several dependencies.
15
16$rp="%r14";
17$sp="%r15";
18$code=<<___;
19.text
20
21___
22
23# void RC4(RC4_KEY *key,size_t len,const void *inp,void *out)
24{
25$acc="%r0";
26$cnt="%r1";
27$key="%r2";
28$len="%r3";
29$inp="%r4";
30$out="%r5";
31
32@XX=("%r6","%r7");
33@TX=("%r8","%r9");
34$YY="%r10";
35$TY="%r11";
36
37$code.=<<___;
38.globl RC4
39.type RC4,\@function
40.align 64
41RC4:
42 stmg %r6,%r11,48($sp)
43 llgc $XX[0],0($key)
44 llgc $YY,1($key)
45 la $XX[0],1($XX[0])
46 nill $XX[0],0xff
47 srlg $cnt,$len,3
48 ltgr $cnt,$cnt
49 llgc $TX[0],2($XX[0],$key)
50 jz .Lshort
51 j .Loop8
52
53.align 64
54.Loop8:
55___
56for ($i=0;$i<8;$i++) {
57$code.=<<___;
58 la $YY,0($YY,$TX[0]) # $i
59 nill $YY,255
60 la $XX[1],1($XX[0])
61 nill $XX[1],255
62___
63$code.=<<___ if ($i==1);
64 llgc $acc,2($TY,$key)
65___
66$code.=<<___ if ($i>1);
67 sllg $acc,$acc,8
68 ic $acc,2($TY,$key)
69___
70$code.=<<___;
71 llgc $TY,2($YY,$key)
72 stc $TX[0],2($YY,$key)
73 llgc $TX[1],2($XX[1],$key)
74 stc $TY,2($XX[0],$key)
75 cr $XX[1],$YY
76 jne .Lcmov$i
77 la $TX[1],0($TX[0])
78.Lcmov$i:
79 la $TY,0($TY,$TX[0])
80 nill $TY,255
81___
82push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
83}
84
85$code.=<<___;
86 lg $TX[1],0($inp)
87 sllg $acc,$acc,8
88 la $inp,8($inp)
89 ic $acc,2($TY,$key)
90 xgr $acc,$TX[1]
91 stg $acc,0($out)
92 la $out,8($out)
93 brct $cnt,.Loop8
94
95.Lshort:
96 lghi $acc,7
97 ngr $len,$acc
98 jz .Lexit
99 j .Loop1
100
101.align 16
102.Loop1:
103 la $YY,0($YY,$TX[0])
104 nill $YY,255
105 llgc $TY,2($YY,$key)
106 stc $TX[0],2($YY,$key)
107 stc $TY,2($XX[0],$key)
108 ar $TY,$TX[0]
109 ahi $XX[0],1
110 nill $TY,255
111 nill $XX[0],255
112 llgc $acc,0($inp)
113 la $inp,1($inp)
114 llgc $TY,2($TY,$key)
115 llgc $TX[0],2($XX[0],$key)
116 xr $acc,$TY
117 stc $acc,0($out)
118 la $out,1($out)
119 brct $len,.Loop1
120
121.Lexit:
122 ahi $XX[0],-1
123 stc $XX[0],0($key)
124 stc $YY,1($key)
125 lmg %r6,%r11,48($sp)
126 br $rp
127.size RC4,.-RC4
128.string "RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
129
130___
131}
132
133# void RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)
134{
135$cnt="%r0";
136$idx="%r1";
137$key="%r2";
138$len="%r3";
139$inp="%r4";
140$acc="%r5";
141$dat="%r6";
142$ikey="%r7";
143$iinp="%r8";
144
145$code.=<<___;
146.globl RC4_set_key
147.type RC4_set_key,\@function
148.align 64
149RC4_set_key:
150 stmg %r6,%r8,48($sp)
151 lhi $cnt,256
152 la $idx,0(%r0)
153 sth $idx,0($key)
154.align 4
155.L1stloop:
156 stc $idx,2($idx,$key)
157 la $idx,1($idx)
158 brct $cnt,.L1stloop
159
160 lghi $ikey,-256
161 lr $cnt,$len
162 la $iinp,0(%r0)
163 la $idx,0(%r0)
164.align 16
165.L2ndloop:
166 llgc $acc,2+256($ikey,$key)
167 llgc $dat,0($iinp,$inp)
168 la $idx,0($idx,$acc)
169 la $ikey,1($ikey)
170 la $idx,0($idx,$dat)
171 nill $idx,255
172 la $iinp,1($iinp)
173 tml $ikey,255
174 llgc $dat,2($idx,$key)
175 stc $dat,2+256-1($ikey,$key)
176 stc $acc,2($idx,$key)
177 jz .Ldone
178 brct $cnt,.L2ndloop
179 lr $cnt,$len
180 la $iinp,0(%r0)
181 j .L2ndloop
182.Ldone:
183 lmg %r6,%r8,48($sp)
184 br $rp
185.size RC4_set_key,.-RC4_set_key
186
187___
188}
189
190# const char *RC4_options()
191$code.=<<___;
192.globl RC4_options
193.type RC4_options,\@function
194.align 16
195RC4_options:
196 larl %r2,.Loptions
197 br %r14
198.size RC4_options,.-RC4_options
199.section .rodata
200.Loptions:
201.align 8
202.string "rc4(8x,char)"
203___
204
205print $code;
diff --git a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
deleted file mode 100755
index 544386bf53..0000000000
--- a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
+++ /dev/null
@@ -1,504 +0,0 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
11# "hand-coded assembler"] doesn't stand for the whole improvement
12# coefficient. It turned out that eliminating RC4_CHAR from config
13# line results in ~40% improvement (yes, even for C implementation).
14# Presumably it has everything to do with AMD cache architecture and
15# RAW or whatever penalties. Once again! The module *requires* config
16# line *without* RC4_CHAR! As for coding "secret," I bet on partial
17# register arithmetics. For example instead of 'inc %r8; and $255,%r8'
18# I simply 'inc %r8b'. Even though optimization manual discourages
19# to operate on partial registers, it turned out to be the best bet.
20# At least for AMD... How IA32E would perform remains to be seen...
21
22# As was shown by Marc Bevand reordering of couple of load operations
23# results in even higher performance gain of 3.3x:-) At least on
24# Opteron... For reference, 1x in this case is RC4_CHAR C-code
25# compiled with gcc 3.3.2, which performs at ~54MBps per 1GHz clock.
26# Latter means that if you want to *estimate* what to expect from
27# *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
28
29# Intel P4 EM64T core was found to run the AMD64 code really slow...
30# The only way to achieve comparable performance on P4 was to keep
31# RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
32# compose blended code, which would perform even within 30% marginal
33# on either AMD and Intel platforms, I implement both cases. See
34# rc4_skey.c for further details...
35
36# P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
37# those with add/sub results in 50% performance improvement of folded
38# loop...
39
40# As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
41# performance by >30% [unlike P4 32-bit case that is]. But this is
42# provided that loads are reordered even more aggressively! Both code
43# pathes, AMD64 and EM64T, reorder loads in essentially same manner
44# as my IA-64 implementation. On Opteron this resulted in modest 5%
45# improvement [I had to test it], while final Intel P4 performance
46# achieves respectful 432MBps on 2.8GHz processor now. For reference.
47# If executed on Xeon, current RC4_CHAR code-path is 2.7x faster than
48# RC4_INT code-path. While if executed on Opteron, it's only 25%
49# slower than the RC4_INT one [meaning that if CPU µ-arch detection
50# is not implemented, then this final RC4_CHAR code-path should be
51# preferred, as it provides better *all-round* performance].
52
53# Intel Core2 was observed to perform poorly on both code paths:-( It
54# apparently suffers from some kind of partial register stall, which
55# occurs in 64-bit mode only [as virtually identical 32-bit loop was
56# observed to outperform 64-bit one by almost 50%]. Adding two movzb to
57# cloop1 boosts its performance by 80%! This loop appears to be optimal
58# fit for Core2 and therefore the code was modified to skip cloop8 on
59# this CPU.
60
61$flavour = shift;
62$output = shift;
63if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
64
65$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
66
67$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
68( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
69( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
70die "can't locate x86_64-xlate.pl";
71
72open STDOUT,"| $^X $xlate $flavour $output";
73
74$dat="%rdi"; # arg1
75$len="%rsi"; # arg2
76$inp="%rdx"; # arg3
77$out="%rcx"; # arg4
78
79@XX=("%r8","%r10");
80@TX=("%r9","%r11");
81$YY="%r12";
82$TY="%r13";
83
84$code=<<___;
85.text
86
87.globl RC4
88.type RC4,\@function,4
89.align 16
90RC4: or $len,$len
91 jne .Lentry
92 ret
93.Lentry:
94 push %rbx
95 push %r12
96 push %r13
97.Lprologue:
98
99 add \$8,$dat
100 movl -8($dat),$XX[0]#d
101 movl -4($dat),$YY#d
102 cmpl \$-1,256($dat)
103 je .LRC4_CHAR
104 inc $XX[0]#b
105 movl ($dat,$XX[0],4),$TX[0]#d
106 test \$-8,$len
107 jz .Lloop1
108 jmp .Lloop8
109.align 16
110.Lloop8:
111___
112for ($i=0;$i<8;$i++) {
113$code.=<<___;
114 add $TX[0]#b,$YY#b
115 mov $XX[0],$XX[1]
116 movl ($dat,$YY,4),$TY#d
117 ror \$8,%rax # ror is redundant when $i=0
118 inc $XX[1]#b
119 movl ($dat,$XX[1],4),$TX[1]#d
120 cmp $XX[1],$YY
121 movl $TX[0]#d,($dat,$YY,4)
122 cmove $TX[0],$TX[1]
123 movl $TY#d,($dat,$XX[0],4)
124 add $TX[0]#b,$TY#b
125 movb ($dat,$TY,4),%al
126___
127push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
128}
129$code.=<<___;
130 ror \$8,%rax
131 sub \$8,$len
132
133 xor ($inp),%rax
134 add \$8,$inp
135 mov %rax,($out)
136 add \$8,$out
137
138 test \$-8,$len
139 jnz .Lloop8
140 cmp \$0,$len
141 jne .Lloop1
142 jmp .Lexit
143
144.align 16
145.Lloop1:
146 add $TX[0]#b,$YY#b
147 movl ($dat,$YY,4),$TY#d
148 movl $TX[0]#d,($dat,$YY,4)
149 movl $TY#d,($dat,$XX[0],4)
150 add $TY#b,$TX[0]#b
151 inc $XX[0]#b
152 movl ($dat,$TX[0],4),$TY#d
153 movl ($dat,$XX[0],4),$TX[0]#d
154 xorb ($inp),$TY#b
155 inc $inp
156 movb $TY#b,($out)
157 inc $out
158 dec $len
159 jnz .Lloop1
160 jmp .Lexit
161
162.align 16
163.LRC4_CHAR:
164 add \$1,$XX[0]#b
165 movzb ($dat,$XX[0]),$TX[0]#d
166 test \$-8,$len
167 jz .Lcloop1
168 cmpl \$0,260($dat)
169 jnz .Lcloop1
170 jmp .Lcloop8
171.align 16
172.Lcloop8:
173 mov ($inp),%eax
174 mov 4($inp),%ebx
175___
176# unroll 2x4-wise, because 64-bit rotates kill Intel P4...
177for ($i=0;$i<4;$i++) {
178$code.=<<___;
179 add $TX[0]#b,$YY#b
180 lea 1($XX[0]),$XX[1]
181 movzb ($dat,$YY),$TY#d
182 movzb $XX[1]#b,$XX[1]#d
183 movzb ($dat,$XX[1]),$TX[1]#d
184 movb $TX[0]#b,($dat,$YY)
185 cmp $XX[1],$YY
186 movb $TY#b,($dat,$XX[0])
187 jne .Lcmov$i # Intel cmov is sloooow...
188 mov $TX[0],$TX[1]
189.Lcmov$i:
190 add $TX[0]#b,$TY#b
191 xor ($dat,$TY),%al
192 ror \$8,%eax
193___
194push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
195}
196for ($i=4;$i<8;$i++) {
197$code.=<<___;
198 add $TX[0]#b,$YY#b
199 lea 1($XX[0]),$XX[1]
200 movzb ($dat,$YY),$TY#d
201 movzb $XX[1]#b,$XX[1]#d
202 movzb ($dat,$XX[1]),$TX[1]#d
203 movb $TX[0]#b,($dat,$YY)
204 cmp $XX[1],$YY
205 movb $TY#b,($dat,$XX[0])
206 jne .Lcmov$i # Intel cmov is sloooow...
207 mov $TX[0],$TX[1]
208.Lcmov$i:
209 add $TX[0]#b,$TY#b
210 xor ($dat,$TY),%bl
211 ror \$8,%ebx
212___
213push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
214}
215$code.=<<___;
216 lea -8($len),$len
217 mov %eax,($out)
218 lea 8($inp),$inp
219 mov %ebx,4($out)
220 lea 8($out),$out
221
222 test \$-8,$len
223 jnz .Lcloop8
224 cmp \$0,$len
225 jne .Lcloop1
226 jmp .Lexit
227___
228$code.=<<___;
229.align 16
230.Lcloop1:
231 add $TX[0]#b,$YY#b
232 movzb ($dat,$YY),$TY#d
233 movb $TX[0]#b,($dat,$YY)
234 movb $TY#b,($dat,$XX[0])
235 add $TX[0]#b,$TY#b
236 add \$1,$XX[0]#b
237 movzb $TY#b,$TY#d
238 movzb $XX[0]#b,$XX[0]#d
239 movzb ($dat,$TY),$TY#d
240 movzb ($dat,$XX[0]),$TX[0]#d
241 xorb ($inp),$TY#b
242 lea 1($inp),$inp
243 movb $TY#b,($out)
244 lea 1($out),$out
245 sub \$1,$len
246 jnz .Lcloop1
247 jmp .Lexit
248
249.align 16
250.Lexit:
251 sub \$1,$XX[0]#b
252 movl $XX[0]#d,-8($dat)
253 movl $YY#d,-4($dat)
254
255 mov (%rsp),%r13
256 mov 8(%rsp),%r12
257 mov 16(%rsp),%rbx
258 add \$24,%rsp
259.Lepilogue:
260 ret
261.size RC4,.-RC4
262___
263
264$idx="%r8";
265$ido="%r9";
266
267$code.=<<___;
268.extern OPENSSL_ia32cap_P
269.globl RC4_set_key
270.type RC4_set_key,\@function,3
271.align 16
272RC4_set_key:
273 lea 8($dat),$dat
274 lea ($inp,$len),$inp
275 neg $len
276 mov $len,%rcx
277 xor %eax,%eax
278 xor $ido,$ido
279 xor %r10,%r10
280 xor %r11,%r11
281
282 mov PIC_GOT(OPENSSL_ia32cap_P),$idx#d
283 bt \$20,$idx#d
284 jnc .Lw1stloop
285 bt \$30,$idx#d
286 setc $ido#b
287 mov $ido#d,260($dat)
288 jmp .Lc1stloop
289
290.align 16
291.Lw1stloop:
292 mov %eax,($dat,%rax,4)
293 add \$1,%al
294 jnc .Lw1stloop
295
296 xor $ido,$ido
297 xor $idx,$idx
298.align 16
299.Lw2ndloop:
300 mov ($dat,$ido,4),%r10d
301 add ($inp,$len,1),$idx#b
302 add %r10b,$idx#b
303 add \$1,$len
304 mov ($dat,$idx,4),%r11d
305 cmovz %rcx,$len
306 mov %r10d,($dat,$idx,4)
307 mov %r11d,($dat,$ido,4)
308 add \$1,$ido#b
309 jnc .Lw2ndloop
310 jmp .Lexit_key
311
312.align 16
313.Lc1stloop:
314 mov %al,($dat,%rax)
315 add \$1,%al
316 jnc .Lc1stloop
317
318 xor $ido,$ido
319 xor $idx,$idx
320.align 16
321.Lc2ndloop:
322 mov ($dat,$ido),%r10b
323 add ($inp,$len),$idx#b
324 add %r10b,$idx#b
325 add \$1,$len
326 mov ($dat,$idx),%r11b
327 jnz .Lcnowrap
328 mov %rcx,$len
329.Lcnowrap:
330 mov %r10b,($dat,$idx)
331 mov %r11b,($dat,$ido)
332 add \$1,$ido#b
333 jnc .Lc2ndloop
334 movl \$-1,256($dat)
335
336.align 16
337.Lexit_key:
338 xor %eax,%eax
339 mov %eax,-8($dat)
340 mov %eax,-4($dat)
341 ret
342.size RC4_set_key,.-RC4_set_key
343
344.globl RC4_options
345.type RC4_options,\@abi-omnipotent
346.align 16
347RC4_options:
348 lea .Lopts(%rip),%rax
349 mov PIC_GOT(OPENSSL_ia32cap_P),%edx
350 bt \$20,%edx
351 jnc .Ldone
352 add \$12,%rax
353 bt \$30,%edx
354 jnc .Ldone
355 add \$13,%rax
356.Ldone:
357 ret
358.align 64
359.Lopts:
360.asciz "rc4(8x,int)"
361.asciz "rc4(8x,char)"
362.asciz "rc4(1x,char)"
363.asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
364.align 64
365.size RC4_options,.-RC4_options
366___
367
368# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
369# CONTEXT *context,DISPATCHER_CONTEXT *disp)
370if ($win64) {
371$rec="%rcx";
372$frame="%rdx";
373$context="%r8";
374$disp="%r9";
375
376$code.=<<___;
377.extern __imp_RtlVirtualUnwind
378.type stream_se_handler,\@abi-omnipotent
379.align 16
380stream_se_handler:
381 push %rsi
382 push %rdi
383 push %rbx
384 push %rbp
385 push %r12
386 push %r13
387 push %r14
388 push %r15
389 pushfq
390 sub \$64,%rsp
391
392 mov 120($context),%rax # pull context->Rax
393 mov 248($context),%rbx # pull context->Rip
394
395 lea .Lprologue(%rip),%r10
396 cmp %r10,%rbx # context->Rip<prologue label
397 jb .Lin_prologue
398
399 mov 152($context),%rax # pull context->Rsp
400
401 lea .Lepilogue(%rip),%r10
402 cmp %r10,%rbx # context->Rip>=epilogue label
403 jae .Lin_prologue
404
405 lea 24(%rax),%rax
406
407 mov -8(%rax),%rbx
408 mov -16(%rax),%r12
409 mov -24(%rax),%r13
410 mov %rbx,144($context) # restore context->Rbx
411 mov %r12,216($context) # restore context->R12
412 mov %r13,224($context) # restore context->R13
413
414.Lin_prologue:
415 mov 8(%rax),%rdi
416 mov 16(%rax),%rsi
417 mov %rax,152($context) # restore context->Rsp
418 mov %rsi,168($context) # restore context->Rsi
419 mov %rdi,176($context) # restore context->Rdi
420
421 jmp .Lcommon_seh_exit
422.size stream_se_handler,.-stream_se_handler
423
424.type key_se_handler,\@abi-omnipotent
425.align 16
426key_se_handler:
427 push %rsi
428 push %rdi
429 push %rbx
430 push %rbp
431 push %r12
432 push %r13
433 push %r14
434 push %r15
435 pushfq
436 sub \$64,%rsp
437
438 mov 152($context),%rax # pull context->Rsp
439 mov 8(%rax),%rdi
440 mov 16(%rax),%rsi
441 mov %rsi,168($context) # restore context->Rsi
442 mov %rdi,176($context) # restore context->Rdi
443
444.Lcommon_seh_exit:
445
446 mov 40($disp),%rdi # disp->ContextRecord
447 mov $context,%rsi # context
448 mov \$154,%ecx # sizeof(CONTEXT)
449 .long 0xa548f3fc # cld; rep movsq
450
451 mov $disp,%rsi
452 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
453 mov 8(%rsi),%rdx # arg2, disp->ImageBase
454 mov 0(%rsi),%r8 # arg3, disp->ControlPc
455 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
456 mov 40(%rsi),%r10 # disp->ContextRecord
457 lea 56(%rsi),%r11 # &disp->HandlerData
458 lea 24(%rsi),%r12 # &disp->EstablisherFrame
459 mov %r10,32(%rsp) # arg5
460 mov %r11,40(%rsp) # arg6
461 mov %r12,48(%rsp) # arg7
462 mov %rcx,56(%rsp) # arg8, (NULL)
463 call *__imp_RtlVirtualUnwind(%rip)
464
465 mov \$1,%eax # ExceptionContinueSearch
466 add \$64,%rsp
467 popfq
468 pop %r15
469 pop %r14
470 pop %r13
471 pop %r12
472 pop %rbp
473 pop %rbx
474 pop %rdi
475 pop %rsi
476 ret
477.size key_se_handler,.-key_se_handler
478
479.section .pdata
480.align 4
481 .rva .LSEH_begin_RC4
482 .rva .LSEH_end_RC4
483 .rva .LSEH_info_RC4
484
485 .rva .LSEH_begin_RC4_set_key
486 .rva .LSEH_end_RC4_set_key
487 .rva .LSEH_info_RC4_set_key
488
489.section .xdata
490.align 8
491.LSEH_info_RC4:
492 .byte 9,0,0,0
493 .rva stream_se_handler
494.LSEH_info_RC4_set_key:
495 .byte 9,0,0,0
496 .rva key_se_handler
497___
498}
499
500$code =~ s/#([bwd])/$1/gm;
501
502print $code;
503
504close STDOUT;
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-11-24 03:53:28 +0000