diff options
Diffstat (limited to 'src/lib/libcrypto/sha/asm/sha1-586.pl')
-rw-r--r-- | src/lib/libcrypto/sha/asm/sha1-586.pl | 1225 |
1 files changed, 0 insertions, 1225 deletions
diff --git a/src/lib/libcrypto/sha/asm/sha1-586.pl b/src/lib/libcrypto/sha/asm/sha1-586.pl deleted file mode 100644 index 6fbea34d78..0000000000 --- a/src/lib/libcrypto/sha/asm/sha1-586.pl +++ /dev/null | |||
@@ -1,1225 +0,0 @@ | |||
1 | #!/usr/bin/env perl | ||
2 | |||
3 | # ==================================================================== | ||
4 | # [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
5 | # project. The module is, however, dual licensed under OpenSSL and | ||
6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
8 | # ==================================================================== | ||
9 | |||
10 | # "[Re]written" was achieved in two major overhauls. In 2004 BODY_* | ||
11 | # functions were re-implemented to address P4 performance issue [see | ||
12 | # commentary below], and in 2006 the rest was rewritten in order to | ||
13 | # gain freedom to liberate licensing terms. | ||
14 | |||
15 | # January, September 2004. | ||
16 | # | ||
17 | # It was noted that Intel IA-32 C compiler generates code which | ||
18 | # performs ~30% *faster* on P4 CPU than original *hand-coded* | ||
19 | # SHA1 assembler implementation. To address this problem (and | ||
20 | # prove that humans are still better than machines:-), the | ||
21 | # original code was overhauled, which resulted in following | ||
22 | # performance changes: | ||
23 | # | ||
24 | # compared with original compared with Intel cc | ||
25 | # assembler impl. generated code | ||
26 | # Pentium -16% +48% | ||
27 | # PIII/AMD +8% +16% | ||
28 | # P4 +85%(!) +45% | ||
29 | # | ||
30 | # As you can see Pentium came out as looser:-( Yet I reckoned that | ||
31 | # improvement on P4 outweights the loss and incorporate this | ||
32 | # re-tuned code to 0.9.7 and later. | ||
33 | # ---------------------------------------------------------------- | ||
34 | # <appro@fy.chalmers.se> | ||
35 | |||
36 | # August 2009. | ||
37 | # | ||
38 | # George Spelvin has tipped that F_40_59(b,c,d) can be rewritten as | ||
39 | # '(c&d) + (b&(c^d))', which allows to accumulate partial results | ||
40 | # and lighten "pressure" on scratch registers. This resulted in | ||
41 | # >12% performance improvement on contemporary AMD cores (with no | ||
42 | # degradation on other CPUs:-). Also, the code was revised to maximize | ||
43 | # "distance" between instructions producing input to 'lea' instruction | ||
44 | # and the 'lea' instruction itself, which is essential for Intel Atom | ||
45 | # core and resulted in ~15% improvement. | ||
46 | |||
47 | # October 2010. | ||
48 | # | ||
49 | # Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it | ||
50 | # is to offload message schedule denoted by Wt in NIST specification, | ||
51 | # or Xupdate in OpenSSL source, to SIMD unit. The idea is not novel, | ||
52 | # and in SSE2 context was first explored by Dean Gaudet in 2004, see | ||
53 | # http://arctic.org/~dean/crypto/sha1.html. Since then several things | ||
54 | # have changed that made it interesting again: | ||
55 | # | ||
56 | # a) XMM units became faster and wider; | ||
57 | # b) instruction set became more versatile; | ||
58 | # c) an important observation was made by Max Locktykhin, which made | ||
59 | # it possible to reduce amount of instructions required to perform | ||
60 | # the operation in question, for further details see | ||
61 | # http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/. | ||
62 | |||
63 | # April 2011. | ||
64 | # | ||
65 | # Add AVX code path, probably most controversial... The thing is that | ||
66 | # switch to AVX alone improves performance by as little as 4% in | ||
67 | # comparison to SSSE3 code path. But below result doesn't look like | ||
68 | # 4% improvement... Trouble is that Sandy Bridge decodes 'ro[rl]' as | ||
69 | # pair of µ-ops, and it's the additional µ-ops, two per round, that | ||
70 | # make it run slower than Core2 and Westmere. But 'sh[rl]d' is decoded | ||
71 | # as single µ-op by Sandy Bridge and it's replacing 'ro[rl]' with | ||
72 | # equivalent 'sh[rl]d' that is responsible for the impressive 5.1 | ||
73 | # cycles per processed byte. But 'sh[rl]d' is not something that used | ||
74 | # to be fast, nor does it appear to be fast in upcoming Bulldozer | ||
75 | # [according to its optimization manual]. Which is why AVX code path | ||
76 | # is guarded by *both* AVX and synthetic bit denoting Intel CPUs. | ||
77 | # One can argue that it's unfair to AMD, but without 'sh[rl]d' it | ||
78 | # makes no sense to keep the AVX code path. If somebody feels that | ||
79 | # strongly, it's probably more appropriate to discuss possibility of | ||
80 | # using vector rotate XOP on AMD... | ||
81 | |||
82 | ###################################################################### | ||
83 | # Current performance is summarized in following table. Numbers are | ||
84 | # CPU clock cycles spent to process single byte (less is better). | ||
85 | # | ||
86 | # x86 SSSE3 AVX | ||
87 | # Pentium 15.7 - | ||
88 | # PIII 11.5 - | ||
89 | # P4 10.6 - | ||
90 | # AMD K8 7.1 - | ||
91 | # Core2 7.3 6.1/+20% - | ||
92 | # Atom 12.5 9.5(*)/+32% - | ||
93 | # Westmere 7.3 5.6/+30% - | ||
94 | # Sandy Bridge 8.8 6.2/+40% 5.1(**)/+70% | ||
95 | # | ||
96 | # (*) Loop is 1056 instructions long and expected result is ~8.25. | ||
97 | # It remains mystery [to me] why ILP is limited to 1.7. | ||
98 | # | ||
99 | # (**) As per above comment, the result is for AVX *plus* sh[rl]d. | ||
100 | |||
101 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
102 | push(@INC,"${dir}","${dir}../../perlasm"); | ||
103 | require "x86asm.pl"; | ||
104 | |||
105 | &asm_init($ARGV[0],"sha1-586.pl",$ARGV[$#ARGV] eq "386"); | ||
106 | |||
107 | $xmm=$ymm=0; | ||
108 | for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); } | ||
109 | |||
110 | $ymm=1 if ($xmm && | ||
111 | `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` | ||
112 | =~ /GNU assembler version ([2-9]\.[0-9]+)/ && | ||
113 | $1>=2.19); # first version supporting AVX | ||
114 | |||
115 | &external_label("OPENSSL_ia32cap_P") if ($xmm); | ||
116 | |||
117 | |||
118 | $A="eax"; | ||
119 | $B="ebx"; | ||
120 | $C="ecx"; | ||
121 | $D="edx"; | ||
122 | $E="edi"; | ||
123 | $T="esi"; | ||
124 | $tmp1="ebp"; | ||
125 | |||
126 | @V=($A,$B,$C,$D,$E,$T); | ||
127 | |||
128 | $alt=0; # 1 denotes alternative IALU implementation, which performs | ||
129 | # 8% *worse* on P4, same on Westmere and Atom, 2% better on | ||
130 | # Sandy Bridge... | ||
131 | |||
132 | sub BODY_00_15 | ||
133 | { | ||
134 | local($n,$a,$b,$c,$d,$e,$f)=@_; | ||
135 | |||
136 | &comment("00_15 $n"); | ||
137 | |||
138 | &mov($f,$c); # f to hold F_00_19(b,c,d) | ||
139 | if ($n==0) { &mov($tmp1,$a); } | ||
140 | else { &mov($a,$tmp1); } | ||
141 | &rotl($tmp1,5); # tmp1=ROTATE(a,5) | ||
142 | &xor($f,$d); | ||
143 | &add($tmp1,$e); # tmp1+=e; | ||
144 | &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded | ||
145 | # with xi, also note that e becomes | ||
146 | # f in next round... | ||
147 | &and($f,$b); | ||
148 | &rotr($b,2); # b=ROTATE(b,30) | ||
149 | &xor($f,$d); # f holds F_00_19(b,c,d) | ||
150 | &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi | ||
151 | |||
152 | if ($n==15) { &mov($e,&swtmp(($n+1)%16));# pre-fetch f for next round | ||
153 | &add($f,$tmp1); } # f+=tmp1 | ||
154 | else { &add($tmp1,$f); } # f becomes a in next round | ||
155 | &mov($tmp1,$a) if ($alt && $n==15); | ||
156 | } | ||
157 | |||
158 | sub BODY_16_19 | ||
159 | { | ||
160 | local($n,$a,$b,$c,$d,$e,$f)=@_; | ||
161 | |||
162 | &comment("16_19 $n"); | ||
163 | |||
164 | if ($alt) { | ||
165 | &xor($c,$d); | ||
166 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
167 | &and($tmp1,$c); # tmp1 to hold F_00_19(b,c,d), b&=c^d | ||
168 | &xor($f,&swtmp(($n+8)%16)); | ||
169 | &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) | ||
170 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
171 | &rotl($f,1); # f=ROTATE(f,1) | ||
172 | &add($e,$tmp1); # e+=F_00_19(b,c,d) | ||
173 | &xor($c,$d); # restore $c | ||
174 | &mov($tmp1,$a); # b in next round | ||
175 | &rotr($b,$n==16?2:7); # b=ROTATE(b,30) | ||
176 | &mov(&swtmp($n%16),$f); # xi=f | ||
177 | &rotl($a,5); # ROTATE(a,5) | ||
178 | &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e | ||
179 | &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round | ||
180 | &add($f,$a); # f+=ROTATE(a,5) | ||
181 | } else { | ||
182 | &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d) | ||
183 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
184 | &xor($tmp1,$d); | ||
185 | &xor($f,&swtmp(($n+8)%16)); | ||
186 | &and($tmp1,$b); | ||
187 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
188 | &rotl($f,1); # f=ROTATE(f,1) | ||
189 | &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) | ||
190 | &add($e,$tmp1); # e+=F_00_19(b,c,d) | ||
191 | &mov($tmp1,$a); | ||
192 | &rotr($b,2); # b=ROTATE(b,30) | ||
193 | &mov(&swtmp($n%16),$f); # xi=f | ||
194 | &rotl($tmp1,5); # ROTATE(a,5) | ||
195 | &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e | ||
196 | &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round | ||
197 | &add($f,$tmp1); # f+=ROTATE(a,5) | ||
198 | } | ||
199 | } | ||
200 | |||
201 | sub BODY_20_39 | ||
202 | { | ||
203 | local($n,$a,$b,$c,$d,$e,$f)=@_; | ||
204 | local $K=($n<40)?0x6ed9eba1:0xca62c1d6; | ||
205 | |||
206 | &comment("20_39 $n"); | ||
207 | |||
208 | if ($alt) { | ||
209 | &xor($tmp1,$c); # tmp1 to hold F_20_39(b,c,d), b^=c | ||
210 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
211 | &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) | ||
212 | &xor($f,&swtmp(($n+8)%16)); | ||
213 | &add($e,$tmp1); # e+=F_20_39(b,c,d) | ||
214 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
215 | &rotl($f,1); # f=ROTATE(f,1) | ||
216 | &mov($tmp1,$a); # b in next round | ||
217 | &rotr($b,7); # b=ROTATE(b,30) | ||
218 | &mov(&swtmp($n%16),$f) if($n<77);# xi=f | ||
219 | &rotl($a,5); # ROTATE(a,5) | ||
220 | &xor($b,$c) if($n==39);# warm up for BODY_40_59 | ||
221 | &and($tmp1,$b) if($n==39); | ||
222 | &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY | ||
223 | &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round | ||
224 | &add($f,$a); # f+=ROTATE(a,5) | ||
225 | &rotr($a,5) if ($n==79); | ||
226 | } else { | ||
227 | &mov($tmp1,$b); # tmp1 to hold F_20_39(b,c,d) | ||
228 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
229 | &xor($tmp1,$c); | ||
230 | &xor($f,&swtmp(($n+8)%16)); | ||
231 | &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) | ||
232 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
233 | &rotl($f,1); # f=ROTATE(f,1) | ||
234 | &add($e,$tmp1); # e+=F_20_39(b,c,d) | ||
235 | &rotr($b,2); # b=ROTATE(b,30) | ||
236 | &mov($tmp1,$a); | ||
237 | &rotl($tmp1,5); # ROTATE(a,5) | ||
238 | &mov(&swtmp($n%16),$f) if($n<77);# xi=f | ||
239 | &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY | ||
240 | &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round | ||
241 | &add($f,$tmp1); # f+=ROTATE(a,5) | ||
242 | } | ||
243 | } | ||
244 | |||
245 | sub BODY_40_59 | ||
246 | { | ||
247 | local($n,$a,$b,$c,$d,$e,$f)=@_; | ||
248 | |||
249 | &comment("40_59 $n"); | ||
250 | |||
251 | if ($alt) { | ||
252 | &add($e,$tmp1); # e+=b&(c^d) | ||
253 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
254 | &mov($tmp1,$d); | ||
255 | &xor($f,&swtmp(($n+8)%16)); | ||
256 | &xor($c,$d); # restore $c | ||
257 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
258 | &rotl($f,1); # f=ROTATE(f,1) | ||
259 | &and($tmp1,$c); | ||
260 | &rotr($b,7); # b=ROTATE(b,30) | ||
261 | &add($e,$tmp1); # e+=c&d | ||
262 | &mov($tmp1,$a); # b in next round | ||
263 | &mov(&swtmp($n%16),$f); # xi=f | ||
264 | &rotl($a,5); # ROTATE(a,5) | ||
265 | &xor($b,$c) if ($n<59); | ||
266 | &and($tmp1,$b) if ($n<59);# tmp1 to hold F_40_59(b,c,d) | ||
267 | &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e+(b&(c^d)) | ||
268 | &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round | ||
269 | &add($f,$a); # f+=ROTATE(a,5) | ||
270 | } else { | ||
271 | &mov($tmp1,$c); # tmp1 to hold F_40_59(b,c,d) | ||
272 | &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) | ||
273 | &xor($tmp1,$d); | ||
274 | &xor($f,&swtmp(($n+8)%16)); | ||
275 | &and($tmp1,$b); | ||
276 | &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd | ||
277 | &rotl($f,1); # f=ROTATE(f,1) | ||
278 | &add($tmp1,$e); # b&(c^d)+=e | ||
279 | &rotr($b,2); # b=ROTATE(b,30) | ||
280 | &mov($e,$a); # e becomes volatile | ||
281 | &rotl($e,5); # ROTATE(a,5) | ||
282 | &mov(&swtmp($n%16),$f); # xi=f | ||
283 | &lea($f,&DWP(0x8f1bbcdc,$f,$tmp1));# f+=K_40_59+e+(b&(c^d)) | ||
284 | &mov($tmp1,$c); | ||
285 | &add($f,$e); # f+=ROTATE(a,5) | ||
286 | &and($tmp1,$d); | ||
287 | &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round | ||
288 | &add($f,$tmp1); # f+=c&d | ||
289 | } | ||
290 | } | ||
291 | |||
292 | &function_begin("sha1_block_data_order"); | ||
293 | if ($xmm) { | ||
294 | &static_label("ssse3_shortcut"); | ||
295 | &static_label("avx_shortcut") if ($ymm); | ||
296 | &static_label("K_XX_XX"); | ||
297 | |||
298 | &call (&label("pic_point")); # make it PIC! | ||
299 | &set_label("pic_point"); | ||
300 | &blindpop($tmp1); | ||
301 | &picmeup($T,"OPENSSL_ia32cap_P",$tmp1,&label("pic_point")); | ||
302 | &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); | ||
303 | |||
304 | &mov ($A,&DWP(0,$T)); | ||
305 | &mov ($D,&DWP(4,$T)); | ||
306 | &test ($D,1<<9); # check SSSE3 bit | ||
307 | &jz (&label("x86")); | ||
308 | &test ($A,1<<24); # check FXSR bit | ||
309 | &jz (&label("x86")); | ||
310 | if ($ymm) { | ||
311 | &and ($D,1<<28); # mask AVX bit | ||
312 | &and ($A,1<<30); # mask "Intel CPU" bit | ||
313 | &or ($A,$D); | ||
314 | &cmp ($A,1<<28|1<<30); | ||
315 | &je (&label("avx_shortcut")); | ||
316 | } | ||
317 | &jmp (&label("ssse3_shortcut")); | ||
318 | &set_label("x86",16); | ||
319 | } | ||
320 | &mov($tmp1,&wparam(0)); # SHA_CTX *c | ||
321 | &mov($T,&wparam(1)); # const void *input | ||
322 | &mov($A,&wparam(2)); # size_t num | ||
323 | &stack_push(16+3); # allocate X[16] | ||
324 | &shl($A,6); | ||
325 | &add($A,$T); | ||
326 | &mov(&wparam(2),$A); # pointer beyond the end of input | ||
327 | &mov($E,&DWP(16,$tmp1));# pre-load E | ||
328 | &jmp(&label("loop")); | ||
329 | |||
330 | &set_label("loop",16); | ||
331 | |||
332 | # copy input chunk to X, but reversing byte order! | ||
333 | for ($i=0; $i<16; $i+=4) | ||
334 | { | ||
335 | &mov($A,&DWP(4*($i+0),$T)); | ||
336 | &mov($B,&DWP(4*($i+1),$T)); | ||
337 | &mov($C,&DWP(4*($i+2),$T)); | ||
338 | &mov($D,&DWP(4*($i+3),$T)); | ||
339 | &bswap($A); | ||
340 | &bswap($B); | ||
341 | &bswap($C); | ||
342 | &bswap($D); | ||
343 | &mov(&swtmp($i+0),$A); | ||
344 | &mov(&swtmp($i+1),$B); | ||
345 | &mov(&swtmp($i+2),$C); | ||
346 | &mov(&swtmp($i+3),$D); | ||
347 | } | ||
348 | &mov(&wparam(1),$T); # redundant in 1st spin | ||
349 | |||
350 | &mov($A,&DWP(0,$tmp1)); # load SHA_CTX | ||
351 | &mov($B,&DWP(4,$tmp1)); | ||
352 | &mov($C,&DWP(8,$tmp1)); | ||
353 | &mov($D,&DWP(12,$tmp1)); | ||
354 | # E is pre-loaded | ||
355 | |||
356 | for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } | ||
357 | for(;$i<20;$i++) { &BODY_16_19($i,@V); unshift(@V,pop(@V)); } | ||
358 | for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } | ||
359 | for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } | ||
360 | for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } | ||
361 | |||
362 | (($V[5] eq $D) and ($V[0] eq $E)) or die; # double-check | ||
363 | |||
364 | &mov($tmp1,&wparam(0)); # re-load SHA_CTX* | ||
365 | &mov($D,&wparam(1)); # D is last "T" and is discarded | ||
366 | |||
367 | &add($E,&DWP(0,$tmp1)); # E is last "A"... | ||
368 | &add($T,&DWP(4,$tmp1)); | ||
369 | &add($A,&DWP(8,$tmp1)); | ||
370 | &add($B,&DWP(12,$tmp1)); | ||
371 | &add($C,&DWP(16,$tmp1)); | ||
372 | |||
373 | &mov(&DWP(0,$tmp1),$E); # update SHA_CTX | ||
374 | &add($D,64); # advance input pointer | ||
375 | &mov(&DWP(4,$tmp1),$T); | ||
376 | &cmp($D,&wparam(2)); # have we reached the end yet? | ||
377 | &mov(&DWP(8,$tmp1),$A); | ||
378 | &mov($E,$C); # C is last "E" which needs to be "pre-loaded" | ||
379 | &mov(&DWP(12,$tmp1),$B); | ||
380 | &mov($T,$D); # input pointer | ||
381 | &mov(&DWP(16,$tmp1),$C); | ||
382 | &jb(&label("loop")); | ||
383 | |||
384 | &stack_pop(16+3); | ||
385 | &function_end("sha1_block_data_order"); | ||
386 | |||
387 | if ($xmm) { | ||
388 | ###################################################################### | ||
389 | # The SSSE3 implementation. | ||
390 | # | ||
391 | # %xmm[0-7] are used as ring @X[] buffer containing quadruples of last | ||
392 | # 32 elements of the message schedule or Xupdate outputs. First 4 | ||
393 | # quadruples are simply byte-swapped input, next 4 are calculated | ||
394 | # according to method originally suggested by Dean Gaudet (modulo | ||
395 | # being implemented in SSSE3). Once 8 quadruples or 32 elements are | ||
396 | # collected, it switches to routine proposed by Max Locktyukhin. | ||
397 | # | ||
398 | # Calculations inevitably require temporary reqisters, and there are | ||
399 | # no %xmm registers left to spare. For this reason part of the ring | ||
400 | # buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring | ||
401 | # buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] - | ||
402 | # X[-5], and X[4] - X[-4]... | ||
403 | # | ||
404 | # Another notable optimization is aggressive stack frame compression | ||
405 | # aiming to minimize amount of 9-byte instructions... | ||
406 | # | ||
407 | # Yet another notable optimization is "jumping" $B variable. It means | ||
408 | # that there is no register permanently allocated for $B value. This | ||
409 | # allowed to eliminate one instruction from body_20_39... | ||
410 | # | ||
411 | my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded | ||
412 | my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 | ||
413 | my @V=($A,$B,$C,$D,$E); | ||
414 | my $j=0; # hash round | ||
415 | my @T=($T,$tmp1); | ||
416 | my $inp; | ||
417 | |||
418 | my $_rol=sub { &rol(@_) }; | ||
419 | my $_ror=sub { &ror(@_) }; | ||
420 | |||
421 | &function_begin("_sha1_block_data_order_ssse3"); | ||
422 | &call (&label("pic_point")); # make it PIC! | ||
423 | &set_label("pic_point"); | ||
424 | &blindpop($tmp1); | ||
425 | &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); | ||
426 | &set_label("ssse3_shortcut"); | ||
427 | |||
428 | &movdqa (@X[3],&QWP(0,$tmp1)); # K_00_19 | ||
429 | &movdqa (@X[4],&QWP(16,$tmp1)); # K_20_39 | ||
430 | &movdqa (@X[5],&QWP(32,$tmp1)); # K_40_59 | ||
431 | &movdqa (@X[6],&QWP(48,$tmp1)); # K_60_79 | ||
432 | &movdqa (@X[2],&QWP(64,$tmp1)); # pbswap mask | ||
433 | |||
434 | &mov ($E,&wparam(0)); # load argument block | ||
435 | &mov ($inp=@T[1],&wparam(1)); | ||
436 | &mov ($D,&wparam(2)); | ||
437 | &mov (@T[0],"esp"); | ||
438 | |||
439 | # stack frame layout | ||
440 | # | ||
441 | # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area | ||
442 | # X[4]+K X[5]+K X[6]+K X[7]+K | ||
443 | # X[8]+K X[9]+K X[10]+K X[11]+K | ||
444 | # X[12]+K X[13]+K X[14]+K X[15]+K | ||
445 | # | ||
446 | # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area | ||
447 | # X[4] X[5] X[6] X[7] | ||
448 | # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 | ||
449 | # | ||
450 | # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants | ||
451 | # K_40_59 K_40_59 K_40_59 K_40_59 | ||
452 | # K_60_79 K_60_79 K_60_79 K_60_79 | ||
453 | # K_00_19 K_00_19 K_00_19 K_00_19 | ||
454 | # pbswap mask | ||
455 | # | ||
456 | # +192 ctx # argument block | ||
457 | # +196 inp | ||
458 | # +200 end | ||
459 | # +204 esp | ||
460 | &sub ("esp",208); | ||
461 | &and ("esp",-64); | ||
462 | |||
463 | &movdqa (&QWP(112+0,"esp"),@X[4]); # copy constants | ||
464 | &movdqa (&QWP(112+16,"esp"),@X[5]); | ||
465 | &movdqa (&QWP(112+32,"esp"),@X[6]); | ||
466 | &shl ($D,6); # len*64 | ||
467 | &movdqa (&QWP(112+48,"esp"),@X[3]); | ||
468 | &add ($D,$inp); # end of input | ||
469 | &movdqa (&QWP(112+64,"esp"),@X[2]); | ||
470 | &add ($inp,64); | ||
471 | &mov (&DWP(192+0,"esp"),$E); # save argument block | ||
472 | &mov (&DWP(192+4,"esp"),$inp); | ||
473 | &mov (&DWP(192+8,"esp"),$D); | ||
474 | &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp | ||
475 | |||
476 | &mov ($A,&DWP(0,$E)); # load context | ||
477 | &mov ($B,&DWP(4,$E)); | ||
478 | &mov ($C,&DWP(8,$E)); | ||
479 | &mov ($D,&DWP(12,$E)); | ||
480 | &mov ($E,&DWP(16,$E)); | ||
481 | &mov (@T[0],$B); # magic seed | ||
482 | |||
483 | &movdqu (@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] | ||
484 | &movdqu (@X[-3&7],&QWP(-48,$inp)); | ||
485 | &movdqu (@X[-2&7],&QWP(-32,$inp)); | ||
486 | &movdqu (@X[-1&7],&QWP(-16,$inp)); | ||
487 | &pshufb (@X[-4&7],@X[2]); # byte swap | ||
488 | &pshufb (@X[-3&7],@X[2]); | ||
489 | &pshufb (@X[-2&7],@X[2]); | ||
490 | &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot | ||
491 | &pshufb (@X[-1&7],@X[2]); | ||
492 | &paddd (@X[-4&7],@X[3]); # add K_00_19 | ||
493 | &paddd (@X[-3&7],@X[3]); | ||
494 | &paddd (@X[-2&7],@X[3]); | ||
495 | &movdqa (&QWP(0,"esp"),@X[-4&7]); # X[]+K xfer to IALU | ||
496 | &psubd (@X[-4&7],@X[3]); # restore X[] | ||
497 | &movdqa (&QWP(0+16,"esp"),@X[-3&7]); | ||
498 | &psubd (@X[-3&7],@X[3]); | ||
499 | &movdqa (&QWP(0+32,"esp"),@X[-2&7]); | ||
500 | &psubd (@X[-2&7],@X[3]); | ||
501 | &movdqa (@X[0],@X[-3&7]); | ||
502 | &jmp (&label("loop")); | ||
503 | |||
504 | ###################################################################### | ||
505 | # SSE instruction sequence is first broken to groups of independent | ||
506 | # instructions, independent in respect to their inputs and shifter | ||
507 | # (not all architectures have more than one). Then IALU instructions | ||
508 | # are "knitted in" between the SSE groups. Distance is maintained for | ||
509 | # SSE latency of 2 in hope that it fits better upcoming AMD Bulldozer | ||
510 | # [which allegedly also implements SSSE3]... | ||
511 | # | ||
512 | # Temporary registers usage. X[2] is volatile at the entry and at the | ||
513 | # end is restored from backtrace ring buffer. X[3] is expected to | ||
514 | # contain current K_XX_XX constant and is used to caclulate X[-1]+K | ||
515 | # from previous round, it becomes volatile the moment the value is | ||
516 | # saved to stack for transfer to IALU. X[4] becomes volatile whenever | ||
517 | # X[-4] is accumulated and offloaded to backtrace ring buffer, at the | ||
518 | # end it is loaded with next K_XX_XX [which becomes X[3] in next | ||
519 | # round]... | ||
520 | # | ||
521 | sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 | ||
522 | { use integer; | ||
523 | my $body = shift; | ||
524 | my @insns = (&$body,&$body,&$body,&$body); # 40 instructions | ||
525 | my ($a,$b,$c,$d,$e); | ||
526 | |||
527 | eval(shift(@insns)); | ||
528 | eval(shift(@insns)); | ||
529 | &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" | ||
530 | &movdqa (@X[2],@X[-1&7]); | ||
531 | eval(shift(@insns)); | ||
532 | eval(shift(@insns)); | ||
533 | |||
534 | &paddd (@X[3],@X[-1&7]); | ||
535 | &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer | ||
536 | eval(shift(@insns)); | ||
537 | eval(shift(@insns)); | ||
538 | &psrldq (@X[2],4); # "X[-3]", 3 dwords | ||
539 | eval(shift(@insns)); | ||
540 | eval(shift(@insns)); | ||
541 | &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" | ||
542 | eval(shift(@insns)); | ||
543 | eval(shift(@insns)); | ||
544 | |||
545 | &pxor (@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" | ||
546 | eval(shift(@insns)); | ||
547 | eval(shift(@insns)); | ||
548 | eval(shift(@insns)); | ||
549 | eval(shift(@insns)); | ||
550 | |||
551 | &pxor (@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" | ||
552 | eval(shift(@insns)); | ||
553 | eval(shift(@insns)); | ||
554 | &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU | ||
555 | eval(shift(@insns)); | ||
556 | eval(shift(@insns)); | ||
557 | |||
558 | &movdqa (@X[4],@X[0]); | ||
559 | &movdqa (@X[2],@X[0]); | ||
560 | eval(shift(@insns)); | ||
561 | eval(shift(@insns)); | ||
562 | eval(shift(@insns)); | ||
563 | eval(shift(@insns)); | ||
564 | |||
565 | &pslldq (@X[4],12); # "X[0]"<<96, extract one dword | ||
566 | &paddd (@X[0],@X[0]); | ||
567 | eval(shift(@insns)); | ||
568 | eval(shift(@insns)); | ||
569 | eval(shift(@insns)); | ||
570 | eval(shift(@insns)); | ||
571 | |||
572 | &psrld (@X[2],31); | ||
573 | eval(shift(@insns)); | ||
574 | eval(shift(@insns)); | ||
575 | &movdqa (@X[3],@X[4]); | ||
576 | eval(shift(@insns)); | ||
577 | eval(shift(@insns)); | ||
578 | |||
579 | &psrld (@X[4],30); | ||
580 | &por (@X[0],@X[2]); # "X[0]"<<<=1 | ||
581 | eval(shift(@insns)); | ||
582 | eval(shift(@insns)); | ||
583 | &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer | ||
584 | eval(shift(@insns)); | ||
585 | eval(shift(@insns)); | ||
586 | |||
587 | &pslld (@X[3],2); | ||
588 | &pxor (@X[0],@X[4]); | ||
589 | eval(shift(@insns)); | ||
590 | eval(shift(@insns)); | ||
591 | &movdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX | ||
592 | eval(shift(@insns)); | ||
593 | eval(shift(@insns)); | ||
594 | |||
595 | &pxor (@X[0],@X[3]); # "X[0]"^=("X[0]"<<96)<<<2 | ||
596 | &movdqa (@X[1],@X[-2&7]) if ($Xi<7); | ||
597 | eval(shift(@insns)); | ||
598 | eval(shift(@insns)); | ||
599 | |||
600 | foreach (@insns) { eval; } # remaining instructions [if any] | ||
601 | |||
602 | $Xi++; push(@X,shift(@X)); # "rotate" X[] | ||
603 | } | ||
604 | |||
605 | sub Xupdate_ssse3_32_79() | ||
606 | { use integer; | ||
607 | my $body = shift; | ||
608 | my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions | ||
609 | my ($a,$b,$c,$d,$e); | ||
610 | |||
611 | &movdqa (@X[2],@X[-1&7]) if ($Xi==8); | ||
612 | eval(shift(@insns)); # body_20_39 | ||
613 | &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" | ||
614 | &palignr(@X[2],@X[-2&7],8); # compose "X[-6]" | ||
615 | eval(shift(@insns)); | ||
616 | eval(shift(@insns)); | ||
617 | eval(shift(@insns)); # rol | ||
618 | |||
619 | &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" | ||
620 | &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer | ||
621 | eval(shift(@insns)); | ||
622 | eval(shift(@insns)); | ||
623 | if ($Xi%5) { | ||
624 | &movdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... | ||
625 | } else { # ... or load next one | ||
626 | &movdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); | ||
627 | } | ||
628 | &paddd (@X[3],@X[-1&7]); | ||
629 | eval(shift(@insns)); # ror | ||
630 | eval(shift(@insns)); | ||
631 | |||
632 | &pxor (@X[0],@X[2]); # "X[0]"^="X[-6]" | ||
633 | eval(shift(@insns)); # body_20_39 | ||
634 | eval(shift(@insns)); | ||
635 | eval(shift(@insns)); | ||
636 | eval(shift(@insns)); # rol | ||
637 | |||
638 | &movdqa (@X[2],@X[0]); | ||
639 | &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU | ||
640 | eval(shift(@insns)); | ||
641 | eval(shift(@insns)); | ||
642 | eval(shift(@insns)); # ror | ||
643 | eval(shift(@insns)); | ||
644 | |||
645 | &pslld (@X[0],2); | ||
646 | eval(shift(@insns)); # body_20_39 | ||
647 | eval(shift(@insns)); | ||
648 | &psrld (@X[2],30); | ||
649 | eval(shift(@insns)); | ||
650 | eval(shift(@insns)); # rol | ||
651 | eval(shift(@insns)); | ||
652 | eval(shift(@insns)); | ||
653 | eval(shift(@insns)); # ror | ||
654 | eval(shift(@insns)); | ||
655 | |||
656 | &por (@X[0],@X[2]); # "X[0]"<<<=2 | ||
657 | eval(shift(@insns)); # body_20_39 | ||
658 | eval(shift(@insns)); | ||
659 | &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer | ||
660 | eval(shift(@insns)); | ||
661 | eval(shift(@insns)); # rol | ||
662 | eval(shift(@insns)); | ||
663 | eval(shift(@insns)); | ||
664 | eval(shift(@insns)); # ror | ||
665 | &movdqa (@X[3],@X[0]) if ($Xi<19); | ||
666 | eval(shift(@insns)); | ||
667 | |||
668 | foreach (@insns) { eval; } # remaining instructions | ||
669 | |||
670 | $Xi++; push(@X,shift(@X)); # "rotate" X[] | ||
671 | } | ||
672 | |||
673 | sub Xuplast_ssse3_80() | ||
674 | { use integer; | ||
675 | my $body = shift; | ||
676 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
677 | my ($a,$b,$c,$d,$e); | ||
678 | |||
679 | eval(shift(@insns)); | ||
680 | &paddd (@X[3],@X[-1&7]); | ||
681 | eval(shift(@insns)); | ||
682 | eval(shift(@insns)); | ||
683 | eval(shift(@insns)); | ||
684 | eval(shift(@insns)); | ||
685 | |||
686 | &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU | ||
687 | |||
688 | foreach (@insns) { eval; } # remaining instructions | ||
689 | |||
690 | &mov ($inp=@T[1],&DWP(192+4,"esp")); | ||
691 | &cmp ($inp,&DWP(192+8,"esp")); | ||
692 | &je (&label("done")); | ||
693 | |||
694 | &movdqa (@X[3],&QWP(112+48,"esp")); # K_00_19 | ||
695 | &movdqa (@X[2],&QWP(112+64,"esp")); # pbswap mask | ||
696 | &movdqu (@X[-4&7],&QWP(0,$inp)); # load input | ||
697 | &movdqu (@X[-3&7],&QWP(16,$inp)); | ||
698 | &movdqu (@X[-2&7],&QWP(32,$inp)); | ||
699 | &movdqu (@X[-1&7],&QWP(48,$inp)); | ||
700 | &add ($inp,64); | ||
701 | &pshufb (@X[-4&7],@X[2]); # byte swap | ||
702 | &mov (&DWP(192+4,"esp"),$inp); | ||
703 | &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot | ||
704 | |||
705 | $Xi=0; | ||
706 | } | ||
707 | |||
708 | sub Xloop_ssse3() | ||
709 | { use integer; | ||
710 | my $body = shift; | ||
711 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
712 | my ($a,$b,$c,$d,$e); | ||
713 | |||
714 | eval(shift(@insns)); | ||
715 | eval(shift(@insns)); | ||
716 | &pshufb (@X[($Xi-3)&7],@X[2]); | ||
717 | eval(shift(@insns)); | ||
718 | eval(shift(@insns)); | ||
719 | &paddd (@X[($Xi-4)&7],@X[3]); | ||
720 | eval(shift(@insns)); | ||
721 | eval(shift(@insns)); | ||
722 | eval(shift(@insns)); | ||
723 | eval(shift(@insns)); | ||
724 | &movdqa (&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]); # X[]+K xfer to IALU | ||
725 | eval(shift(@insns)); | ||
726 | eval(shift(@insns)); | ||
727 | &psubd (@X[($Xi-4)&7],@X[3]); | ||
728 | |||
729 | foreach (@insns) { eval; } | ||
730 | $Xi++; | ||
731 | } | ||
732 | |||
733 | sub Xtail_ssse3() | ||
734 | { use integer; | ||
735 | my $body = shift; | ||
736 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
737 | my ($a,$b,$c,$d,$e); | ||
738 | |||
739 | foreach (@insns) { eval; } | ||
740 | } | ||
741 | |||
742 | sub body_00_19 () { | ||
743 | ( | ||
744 | '($a,$b,$c,$d,$e)=@V;'. | ||
745 | '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer | ||
746 | '&xor ($c,$d);', | ||
747 | '&mov (@T[1],$a);', # $b in next round | ||
748 | '&$_rol ($a,5);', | ||
749 | '&and (@T[0],$c);', # ($b&($c^$d)) | ||
750 | '&xor ($c,$d);', # restore $c | ||
751 | '&xor (@T[0],$d);', | ||
752 | '&add ($e,$a);', | ||
753 | '&$_ror ($b,$j?7:2);', # $b>>>2 | ||
754 | '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' | ||
755 | ); | ||
756 | } | ||
757 | |||
758 | sub body_20_39 () { | ||
759 | ( | ||
760 | '($a,$b,$c,$d,$e)=@V;'. | ||
761 | '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer | ||
762 | '&xor (@T[0],$d);', # ($b^$d) | ||
763 | '&mov (@T[1],$a);', # $b in next round | ||
764 | '&$_rol ($a,5);', | ||
765 | '&xor (@T[0],$c);', # ($b^$d^$c) | ||
766 | '&add ($e,$a);', | ||
767 | '&$_ror ($b,7);', # $b>>>2 | ||
768 | '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' | ||
769 | ); | ||
770 | } | ||
771 | |||
772 | sub body_40_59 () { | ||
773 | ( | ||
774 | '($a,$b,$c,$d,$e)=@V;'. | ||
775 | '&mov (@T[1],$c);', | ||
776 | '&xor ($c,$d);', | ||
777 | '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer | ||
778 | '&and (@T[1],$d);', | ||
779 | '&and (@T[0],$c);', # ($b&($c^$d)) | ||
780 | '&$_ror ($b,7);', # $b>>>2 | ||
781 | '&add ($e,@T[1]);', | ||
782 | '&mov (@T[1],$a);', # $b in next round | ||
783 | '&$_rol ($a,5);', | ||
784 | '&add ($e,@T[0]);', | ||
785 | '&xor ($c,$d);', # restore $c | ||
786 | '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' | ||
787 | ); | ||
788 | } | ||
789 | |||
790 | &set_label("loop",16); | ||
791 | &Xupdate_ssse3_16_31(\&body_00_19); | ||
792 | &Xupdate_ssse3_16_31(\&body_00_19); | ||
793 | &Xupdate_ssse3_16_31(\&body_00_19); | ||
794 | &Xupdate_ssse3_16_31(\&body_00_19); | ||
795 | &Xupdate_ssse3_32_79(\&body_00_19); | ||
796 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
797 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
798 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
799 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
800 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
801 | &Xupdate_ssse3_32_79(\&body_40_59); | ||
802 | &Xupdate_ssse3_32_79(\&body_40_59); | ||
803 | &Xupdate_ssse3_32_79(\&body_40_59); | ||
804 | &Xupdate_ssse3_32_79(\&body_40_59); | ||
805 | &Xupdate_ssse3_32_79(\&body_40_59); | ||
806 | &Xupdate_ssse3_32_79(\&body_20_39); | ||
807 | &Xuplast_ssse3_80(\&body_20_39); # can jump to "done" | ||
808 | |||
809 | $saved_j=$j; @saved_V=@V; | ||
810 | |||
811 | &Xloop_ssse3(\&body_20_39); | ||
812 | &Xloop_ssse3(\&body_20_39); | ||
813 | &Xloop_ssse3(\&body_20_39); | ||
814 | |||
815 | &mov (@T[1],&DWP(192,"esp")); # update context | ||
816 | &add ($A,&DWP(0,@T[1])); | ||
817 | &add (@T[0],&DWP(4,@T[1])); # $b | ||
818 | &add ($C,&DWP(8,@T[1])); | ||
819 | &mov (&DWP(0,@T[1]),$A); | ||
820 | &add ($D,&DWP(12,@T[1])); | ||
821 | &mov (&DWP(4,@T[1]),@T[0]); | ||
822 | &add ($E,&DWP(16,@T[1])); | ||
823 | &mov (&DWP(8,@T[1]),$C); | ||
824 | &mov ($B,@T[0]); | ||
825 | &mov (&DWP(12,@T[1]),$D); | ||
826 | &mov (&DWP(16,@T[1]),$E); | ||
827 | &movdqa (@X[0],@X[-3&7]); | ||
828 | |||
829 | &jmp (&label("loop")); | ||
830 | |||
831 | &set_label("done",16); $j=$saved_j; @V=@saved_V; | ||
832 | |||
833 | &Xtail_ssse3(\&body_20_39); | ||
834 | &Xtail_ssse3(\&body_20_39); | ||
835 | &Xtail_ssse3(\&body_20_39); | ||
836 | |||
837 | &mov (@T[1],&DWP(192,"esp")); # update context | ||
838 | &add ($A,&DWP(0,@T[1])); | ||
839 | &mov ("esp",&DWP(192+12,"esp")); # restore %esp | ||
840 | &add (@T[0],&DWP(4,@T[1])); # $b | ||
841 | &add ($C,&DWP(8,@T[1])); | ||
842 | &mov (&DWP(0,@T[1]),$A); | ||
843 | &add ($D,&DWP(12,@T[1])); | ||
844 | &mov (&DWP(4,@T[1]),@T[0]); | ||
845 | &add ($E,&DWP(16,@T[1])); | ||
846 | &mov (&DWP(8,@T[1]),$C); | ||
847 | &mov (&DWP(12,@T[1]),$D); | ||
848 | &mov (&DWP(16,@T[1]),$E); | ||
849 | |||
850 | &function_end("_sha1_block_data_order_ssse3"); | ||
851 | |||
852 | if ($ymm) { | ||
853 | my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded | ||
854 | my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 | ||
855 | my @V=($A,$B,$C,$D,$E); | ||
856 | my $j=0; # hash round | ||
857 | my @T=($T,$tmp1); | ||
858 | my $inp; | ||
859 | |||
860 | my $_rol=sub { &shld(@_[0],@_) }; | ||
861 | my $_ror=sub { &shrd(@_[0],@_) }; | ||
862 | |||
863 | &function_begin("_sha1_block_data_order_avx"); | ||
864 | &call (&label("pic_point")); # make it PIC! | ||
865 | &set_label("pic_point"); | ||
866 | &blindpop($tmp1); | ||
867 | &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); | ||
868 | &set_label("avx_shortcut"); | ||
869 | &vzeroall(); | ||
870 | |||
871 | &vmovdqa(@X[3],&QWP(0,$tmp1)); # K_00_19 | ||
872 | &vmovdqa(@X[4],&QWP(16,$tmp1)); # K_20_39 | ||
873 | &vmovdqa(@X[5],&QWP(32,$tmp1)); # K_40_59 | ||
874 | &vmovdqa(@X[6],&QWP(48,$tmp1)); # K_60_79 | ||
875 | &vmovdqa(@X[2],&QWP(64,$tmp1)); # pbswap mask | ||
876 | |||
877 | &mov ($E,&wparam(0)); # load argument block | ||
878 | &mov ($inp=@T[1],&wparam(1)); | ||
879 | &mov ($D,&wparam(2)); | ||
880 | &mov (@T[0],"esp"); | ||
881 | |||
882 | # stack frame layout | ||
883 | # | ||
884 | # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area | ||
885 | # X[4]+K X[5]+K X[6]+K X[7]+K | ||
886 | # X[8]+K X[9]+K X[10]+K X[11]+K | ||
887 | # X[12]+K X[13]+K X[14]+K X[15]+K | ||
888 | # | ||
889 | # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area | ||
890 | # X[4] X[5] X[6] X[7] | ||
891 | # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 | ||
892 | # | ||
893 | # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants | ||
894 | # K_40_59 K_40_59 K_40_59 K_40_59 | ||
895 | # K_60_79 K_60_79 K_60_79 K_60_79 | ||
896 | # K_00_19 K_00_19 K_00_19 K_00_19 | ||
897 | # pbswap mask | ||
898 | # | ||
899 | # +192 ctx # argument block | ||
900 | # +196 inp | ||
901 | # +200 end | ||
902 | # +204 esp | ||
903 | &sub ("esp",208); | ||
904 | &and ("esp",-64); | ||
905 | |||
906 | &vmovdqa(&QWP(112+0,"esp"),@X[4]); # copy constants | ||
907 | &vmovdqa(&QWP(112+16,"esp"),@X[5]); | ||
908 | &vmovdqa(&QWP(112+32,"esp"),@X[6]); | ||
909 | &shl ($D,6); # len*64 | ||
910 | &vmovdqa(&QWP(112+48,"esp"),@X[3]); | ||
911 | &add ($D,$inp); # end of input | ||
912 | &vmovdqa(&QWP(112+64,"esp"),@X[2]); | ||
913 | &add ($inp,64); | ||
914 | &mov (&DWP(192+0,"esp"),$E); # save argument block | ||
915 | &mov (&DWP(192+4,"esp"),$inp); | ||
916 | &mov (&DWP(192+8,"esp"),$D); | ||
917 | &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp | ||
918 | |||
919 | &mov ($A,&DWP(0,$E)); # load context | ||
920 | &mov ($B,&DWP(4,$E)); | ||
921 | &mov ($C,&DWP(8,$E)); | ||
922 | &mov ($D,&DWP(12,$E)); | ||
923 | &mov ($E,&DWP(16,$E)); | ||
924 | &mov (@T[0],$B); # magic seed | ||
925 | |||
926 | &vmovdqu(@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] | ||
927 | &vmovdqu(@X[-3&7],&QWP(-48,$inp)); | ||
928 | &vmovdqu(@X[-2&7],&QWP(-32,$inp)); | ||
929 | &vmovdqu(@X[-1&7],&QWP(-16,$inp)); | ||
930 | &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap | ||
931 | &vpshufb(@X[-3&7],@X[-3&7],@X[2]); | ||
932 | &vpshufb(@X[-2&7],@X[-2&7],@X[2]); | ||
933 | &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot | ||
934 | &vpshufb(@X[-1&7],@X[-1&7],@X[2]); | ||
935 | &vpaddd (@X[0],@X[-4&7],@X[3]); # add K_00_19 | ||
936 | &vpaddd (@X[1],@X[-3&7],@X[3]); | ||
937 | &vpaddd (@X[2],@X[-2&7],@X[3]); | ||
938 | &vmovdqa(&QWP(0,"esp"),@X[0]); # X[]+K xfer to IALU | ||
939 | &vmovdqa(&QWP(0+16,"esp"),@X[1]); | ||
940 | &vmovdqa(&QWP(0+32,"esp"),@X[2]); | ||
941 | &jmp (&label("loop")); | ||
942 | |||
943 | sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 | ||
944 | { use integer; | ||
945 | my $body = shift; | ||
946 | my @insns = (&$body,&$body,&$body,&$body); # 40 instructions | ||
947 | my ($a,$b,$c,$d,$e); | ||
948 | |||
949 | eval(shift(@insns)); | ||
950 | eval(shift(@insns)); | ||
951 | &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" | ||
952 | eval(shift(@insns)); | ||
953 | eval(shift(@insns)); | ||
954 | |||
955 | &vpaddd (@X[3],@X[3],@X[-1&7]); | ||
956 | &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer | ||
957 | eval(shift(@insns)); | ||
958 | eval(shift(@insns)); | ||
959 | &vpsrldq(@X[2],@X[-1&7],4); # "X[-3]", 3 dwords | ||
960 | eval(shift(@insns)); | ||
961 | eval(shift(@insns)); | ||
962 | &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" | ||
963 | eval(shift(@insns)); | ||
964 | eval(shift(@insns)); | ||
965 | |||
966 | &vpxor (@X[2],@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" | ||
967 | eval(shift(@insns)); | ||
968 | eval(shift(@insns)); | ||
969 | &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU | ||
970 | eval(shift(@insns)); | ||
971 | eval(shift(@insns)); | ||
972 | |||
973 | &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" | ||
974 | eval(shift(@insns)); | ||
975 | eval(shift(@insns)); | ||
976 | eval(shift(@insns)); | ||
977 | eval(shift(@insns)); | ||
978 | |||
979 | &vpsrld (@X[2],@X[0],31); | ||
980 | eval(shift(@insns)); | ||
981 | eval(shift(@insns)); | ||
982 | eval(shift(@insns)); | ||
983 | eval(shift(@insns)); | ||
984 | |||
985 | &vpslldq(@X[4],@X[0],12); # "X[0]"<<96, extract one dword | ||
986 | &vpaddd (@X[0],@X[0],@X[0]); | ||
987 | eval(shift(@insns)); | ||
988 | eval(shift(@insns)); | ||
989 | eval(shift(@insns)); | ||
990 | eval(shift(@insns)); | ||
991 | |||
992 | &vpsrld (@X[3],@X[4],30); | ||
993 | &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=1 | ||
994 | eval(shift(@insns)); | ||
995 | eval(shift(@insns)); | ||
996 | eval(shift(@insns)); | ||
997 | eval(shift(@insns)); | ||
998 | |||
999 | &vpslld (@X[4],@X[4],2); | ||
1000 | &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer | ||
1001 | eval(shift(@insns)); | ||
1002 | eval(shift(@insns)); | ||
1003 | &vpxor (@X[0],@X[0],@X[3]); | ||
1004 | eval(shift(@insns)); | ||
1005 | eval(shift(@insns)); | ||
1006 | eval(shift(@insns)); | ||
1007 | eval(shift(@insns)); | ||
1008 | |||
1009 | &vpxor (@X[0],@X[0],@X[4]); # "X[0]"^=("X[0]"<<96)<<<2 | ||
1010 | eval(shift(@insns)); | ||
1011 | eval(shift(@insns)); | ||
1012 | &vmovdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX | ||
1013 | eval(shift(@insns)); | ||
1014 | eval(shift(@insns)); | ||
1015 | |||
1016 | foreach (@insns) { eval; } # remaining instructions [if any] | ||
1017 | |||
1018 | $Xi++; push(@X,shift(@X)); # "rotate" X[] | ||
1019 | } | ||
1020 | |||
1021 | sub Xupdate_avx_32_79() | ||
1022 | { use integer; | ||
1023 | my $body = shift; | ||
1024 | my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions | ||
1025 | my ($a,$b,$c,$d,$e); | ||
1026 | |||
1027 | &vpalignr(@X[2],@X[-1&7],@X[-2&7],8); # compose "X[-6]" | ||
1028 | &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" | ||
1029 | eval(shift(@insns)); # body_20_39 | ||
1030 | eval(shift(@insns)); | ||
1031 | eval(shift(@insns)); | ||
1032 | eval(shift(@insns)); # rol | ||
1033 | |||
1034 | &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" | ||
1035 | &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer | ||
1036 | eval(shift(@insns)); | ||
1037 | eval(shift(@insns)); | ||
1038 | if ($Xi%5) { | ||
1039 | &vmovdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... | ||
1040 | } else { # ... or load next one | ||
1041 | &vmovdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); | ||
1042 | } | ||
1043 | &vpaddd (@X[3],@X[3],@X[-1&7]); | ||
1044 | eval(shift(@insns)); # ror | ||
1045 | eval(shift(@insns)); | ||
1046 | |||
1047 | &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-6]" | ||
1048 | eval(shift(@insns)); # body_20_39 | ||
1049 | eval(shift(@insns)); | ||
1050 | eval(shift(@insns)); | ||
1051 | eval(shift(@insns)); # rol | ||
1052 | |||
1053 | &vpsrld (@X[2],@X[0],30); | ||
1054 | &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU | ||
1055 | eval(shift(@insns)); | ||
1056 | eval(shift(@insns)); | ||
1057 | eval(shift(@insns)); # ror | ||
1058 | eval(shift(@insns)); | ||
1059 | |||
1060 | &vpslld (@X[0],@X[0],2); | ||
1061 | eval(shift(@insns)); # body_20_39 | ||
1062 | eval(shift(@insns)); | ||
1063 | eval(shift(@insns)); | ||
1064 | eval(shift(@insns)); # rol | ||
1065 | eval(shift(@insns)); | ||
1066 | eval(shift(@insns)); | ||
1067 | eval(shift(@insns)); # ror | ||
1068 | eval(shift(@insns)); | ||
1069 | |||
1070 | &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=2 | ||
1071 | eval(shift(@insns)); # body_20_39 | ||
1072 | eval(shift(@insns)); | ||
1073 | &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer | ||
1074 | eval(shift(@insns)); | ||
1075 | eval(shift(@insns)); # rol | ||
1076 | eval(shift(@insns)); | ||
1077 | eval(shift(@insns)); | ||
1078 | eval(shift(@insns)); # ror | ||
1079 | eval(shift(@insns)); | ||
1080 | |||
1081 | foreach (@insns) { eval; } # remaining instructions | ||
1082 | |||
1083 | $Xi++; push(@X,shift(@X)); # "rotate" X[] | ||
1084 | } | ||
1085 | |||
1086 | sub Xuplast_avx_80() | ||
1087 | { use integer; | ||
1088 | my $body = shift; | ||
1089 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
1090 | my ($a,$b,$c,$d,$e); | ||
1091 | |||
1092 | eval(shift(@insns)); | ||
1093 | &vpaddd (@X[3],@X[3],@X[-1&7]); | ||
1094 | eval(shift(@insns)); | ||
1095 | eval(shift(@insns)); | ||
1096 | eval(shift(@insns)); | ||
1097 | eval(shift(@insns)); | ||
1098 | |||
1099 | &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU | ||
1100 | |||
1101 | foreach (@insns) { eval; } # remaining instructions | ||
1102 | |||
1103 | &mov ($inp=@T[1],&DWP(192+4,"esp")); | ||
1104 | &cmp ($inp,&DWP(192+8,"esp")); | ||
1105 | &je (&label("done")); | ||
1106 | |||
1107 | &vmovdqa(@X[3],&QWP(112+48,"esp")); # K_00_19 | ||
1108 | &vmovdqa(@X[2],&QWP(112+64,"esp")); # pbswap mask | ||
1109 | &vmovdqu(@X[-4&7],&QWP(0,$inp)); # load input | ||
1110 | &vmovdqu(@X[-3&7],&QWP(16,$inp)); | ||
1111 | &vmovdqu(@X[-2&7],&QWP(32,$inp)); | ||
1112 | &vmovdqu(@X[-1&7],&QWP(48,$inp)); | ||
1113 | &add ($inp,64); | ||
1114 | &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap | ||
1115 | &mov (&DWP(192+4,"esp"),$inp); | ||
1116 | &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot | ||
1117 | |||
1118 | $Xi=0; | ||
1119 | } | ||
1120 | |||
1121 | sub Xloop_avx() | ||
1122 | { use integer; | ||
1123 | my $body = shift; | ||
1124 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
1125 | my ($a,$b,$c,$d,$e); | ||
1126 | |||
1127 | eval(shift(@insns)); | ||
1128 | eval(shift(@insns)); | ||
1129 | &vpshufb (@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); | ||
1130 | eval(shift(@insns)); | ||
1131 | eval(shift(@insns)); | ||
1132 | &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@X[3]); | ||
1133 | eval(shift(@insns)); | ||
1134 | eval(shift(@insns)); | ||
1135 | eval(shift(@insns)); | ||
1136 | eval(shift(@insns)); | ||
1137 | &vmovdqa (&QWP(0+16*$Xi,"esp"),@X[$Xi&7]); # X[]+K xfer to IALU | ||
1138 | eval(shift(@insns)); | ||
1139 | eval(shift(@insns)); | ||
1140 | |||
1141 | foreach (@insns) { eval; } | ||
1142 | $Xi++; | ||
1143 | } | ||
1144 | |||
1145 | sub Xtail_avx() | ||
1146 | { use integer; | ||
1147 | my $body = shift; | ||
1148 | my @insns = (&$body,&$body,&$body,&$body); # 32 instructions | ||
1149 | my ($a,$b,$c,$d,$e); | ||
1150 | |||
1151 | foreach (@insns) { eval; } | ||
1152 | } | ||
1153 | |||
1154 | &set_label("loop",16); | ||
1155 | &Xupdate_avx_16_31(\&body_00_19); | ||
1156 | &Xupdate_avx_16_31(\&body_00_19); | ||
1157 | &Xupdate_avx_16_31(\&body_00_19); | ||
1158 | &Xupdate_avx_16_31(\&body_00_19); | ||
1159 | &Xupdate_avx_32_79(\&body_00_19); | ||
1160 | &Xupdate_avx_32_79(\&body_20_39); | ||
1161 | &Xupdate_avx_32_79(\&body_20_39); | ||
1162 | &Xupdate_avx_32_79(\&body_20_39); | ||
1163 | &Xupdate_avx_32_79(\&body_20_39); | ||
1164 | &Xupdate_avx_32_79(\&body_20_39); | ||
1165 | &Xupdate_avx_32_79(\&body_40_59); | ||
1166 | &Xupdate_avx_32_79(\&body_40_59); | ||
1167 | &Xupdate_avx_32_79(\&body_40_59); | ||
1168 | &Xupdate_avx_32_79(\&body_40_59); | ||
1169 | &Xupdate_avx_32_79(\&body_40_59); | ||
1170 | &Xupdate_avx_32_79(\&body_20_39); | ||
1171 | &Xuplast_avx_80(\&body_20_39); # can jump to "done" | ||
1172 | |||
1173 | $saved_j=$j; @saved_V=@V; | ||
1174 | |||
1175 | &Xloop_avx(\&body_20_39); | ||
1176 | &Xloop_avx(\&body_20_39); | ||
1177 | &Xloop_avx(\&body_20_39); | ||
1178 | |||
1179 | &mov (@T[1],&DWP(192,"esp")); # update context | ||
1180 | &add ($A,&DWP(0,@T[1])); | ||
1181 | &add (@T[0],&DWP(4,@T[1])); # $b | ||
1182 | &add ($C,&DWP(8,@T[1])); | ||
1183 | &mov (&DWP(0,@T[1]),$A); | ||
1184 | &add ($D,&DWP(12,@T[1])); | ||
1185 | &mov (&DWP(4,@T[1]),@T[0]); | ||
1186 | &add ($E,&DWP(16,@T[1])); | ||
1187 | &mov (&DWP(8,@T[1]),$C); | ||
1188 | &mov ($B,@T[0]); | ||
1189 | &mov (&DWP(12,@T[1]),$D); | ||
1190 | &mov (&DWP(16,@T[1]),$E); | ||
1191 | |||
1192 | &jmp (&label("loop")); | ||
1193 | |||
1194 | &set_label("done",16); $j=$saved_j; @V=@saved_V; | ||
1195 | |||
1196 | &Xtail_avx(\&body_20_39); | ||
1197 | &Xtail_avx(\&body_20_39); | ||
1198 | &Xtail_avx(\&body_20_39); | ||
1199 | |||
1200 | &vzeroall(); | ||
1201 | |||
1202 | &mov (@T[1],&DWP(192,"esp")); # update context | ||
1203 | &add ($A,&DWP(0,@T[1])); | ||
1204 | &mov ("esp",&DWP(192+12,"esp")); # restore %esp | ||
1205 | &add (@T[0],&DWP(4,@T[1])); # $b | ||
1206 | &add ($C,&DWP(8,@T[1])); | ||
1207 | &mov (&DWP(0,@T[1]),$A); | ||
1208 | &add ($D,&DWP(12,@T[1])); | ||
1209 | &mov (&DWP(4,@T[1]),@T[0]); | ||
1210 | &add ($E,&DWP(16,@T[1])); | ||
1211 | &mov (&DWP(8,@T[1]),$C); | ||
1212 | &mov (&DWP(12,@T[1]),$D); | ||
1213 | &mov (&DWP(16,@T[1]),$E); | ||
1214 | &function_end("_sha1_block_data_order_avx"); | ||
1215 | } | ||
1216 | &set_label("K_XX_XX",64); | ||
1217 | &data_word(0x5a827999,0x5a827999,0x5a827999,0x5a827999); # K_00_19 | ||
1218 | &data_word(0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1); # K_20_39 | ||
1219 | &data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59 | ||
1220 | &data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79 | ||
1221 | &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask | ||
1222 | } | ||
1223 | &asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); | ||
1224 | |||
1225 | &asm_finish(); | ||