diff options
author | thib <> | 2010-07-01 17:44:20 +0000 |
---|---|---|
committer | thib <> | 2010-07-01 17:44:20 +0000 |
commit | aaf95101ecdb076e7121125b5b59b79338f68e63 (patch) | |
tree | e732da07cd98066d0bfc1903e3ee73114f5b295c | |
parent | 1b4deace3d997af54c492a67c6f9468ec87b9b5d (diff) | |
download | openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.tar.gz openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.tar.bz2 openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.zip |
AES-NI engine support for OpenSSL.
This is code mostly picked up from upstream OpenSSL, or to be more exact
a diff from David Woodhouse <dwmw2 at infradead dot org>.
Remember to make includes before doing a build!
no objections from djm@
OK deraadt@, reyk@ (AES is about 4.25x faster on his x201 now)
-rw-r--r-- | src/lib/libcrypto/aes/asm/aesni-x86_64.pl | 992 | ||||
-rw-r--r-- | src/lib/libcrypto/crypto-lib.com | 3 | ||||
-rw-r--r-- | src/lib/libcrypto/engine/Makefile | 21 | ||||
-rw-r--r-- | src/lib/libcrypto/engine/eng_aesni.c | 570 | ||||
-rw-r--r-- | src/lib/libcrypto/engine/eng_all.c | 5 | ||||
-rw-r--r-- | src/lib/libcrypto/engine/engine.h | 1 | ||||
-rw-r--r-- | src/lib/libcrypto/evp/evp.h | 1 | ||||
-rw-r--r-- | src/lib/libcrypto/evp/evp_err.c | 1 | ||||
-rw-r--r-- | src/lib/libcrypto/util/libeay.num | 1 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl | 992 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/crypto-lib.com | 3 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/engine/Makefile | 21 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/engine/eng_aesni.c | 570 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/engine/eng_all.c | 5 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/engine/engine.h | 1 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/evp/evp.h | 1 | ||||
-rw-r--r-- | src/lib/libssl/src/crypto/evp/evp_err.c | 1 | ||||
-rw-r--r-- | src/lib/libssl/src/test/test_aesni | 69 | ||||
-rw-r--r-- | src/lib/libssl/src/util/libeay.num | 1 | ||||
-rw-r--r-- | src/lib/libssl/test/test_aesni | 69 |
20 files changed, 3322 insertions, 6 deletions
diff --git a/src/lib/libcrypto/aes/asm/aesni-x86_64.pl b/src/lib/libcrypto/aes/asm/aesni-x86_64.pl new file mode 100644 index 0000000000..49e0f4b351 --- /dev/null +++ b/src/lib/libcrypto/aes/asm/aesni-x86_64.pl | |||
@@ -0,0 +1,992 @@ | |||
1 | #!/usr/bin/env perl | ||
2 | # | ||
3 | # ==================================================================== | ||
4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
5 | # project. The module is, however, dual licensed under OpenSSL and | ||
6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
8 | # ==================================================================== | ||
9 | # | ||
10 | # This module implements support for Intel AES-NI extension. In | ||
11 | # OpenSSL context it's used with Intel engine, but can also be used as | ||
12 | # drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for | ||
13 | # details]. | ||
14 | |||
15 | $PREFIX="aesni"; # if $PREFIX is set to "AES", the script | ||
16 | # generates drop-in replacement for | ||
17 | # crypto/aes/asm/aes-x86_64.pl:-) | ||
18 | |||
19 | $flavour = shift; | ||
20 | $output = shift; | ||
21 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | ||
22 | |||
23 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | ||
24 | |||
25 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
26 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | ||
27 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | ||
28 | die "can't locate x86_64-xlate.pl"; | ||
29 | |||
30 | open STDOUT,"| $^X $xlate $flavour $output"; | ||
31 | |||
32 | $movkey = $PREFIX eq "aesni" ? "movaps" : "movups"; | ||
33 | @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order | ||
34 | ("%rdi","%rsi","%rdx","%rcx"); # Unix order | ||
35 | |||
36 | $code=".text\n"; | ||
37 | |||
38 | $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! | ||
39 | # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... | ||
40 | $inp="%rdi"; | ||
41 | $out="%rsi"; | ||
42 | $len="%rdx"; | ||
43 | $key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!! | ||
44 | $ivp="%r8"; # cbc | ||
45 | |||
46 | $rnds_="%r10d"; # backup copy for $rounds | ||
47 | $key_="%r11"; # backup copy for $key | ||
48 | |||
49 | # %xmm register layout | ||
50 | $inout0="%xmm0"; $inout1="%xmm1"; | ||
51 | $inout2="%xmm2"; $inout3="%xmm3"; | ||
52 | $rndkey0="%xmm4"; $rndkey1="%xmm5"; | ||
53 | |||
54 | $iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt | ||
55 | $in1="%xmm8"; $in2="%xmm9"; | ||
56 | |||
57 | # Inline version of internal aesni_[en|de]crypt1. | ||
58 | # | ||
59 | # Why folded loop? Because aes[enc|dec] is slow enough to accommodate | ||
60 | # cycles which take care of loop variables... | ||
61 | { my $sn; | ||
62 | sub aesni_generate1 { | ||
63 | my ($p,$key,$rounds)=@_; | ||
64 | ++$sn; | ||
65 | $code.=<<___; | ||
66 | $movkey ($key),$rndkey0 | ||
67 | $movkey 16($key),$rndkey1 | ||
68 | lea 32($key),$key | ||
69 | pxor $rndkey0,$inout0 | ||
70 | .Loop_${p}1_$sn: | ||
71 | aes${p} $rndkey1,$inout0 | ||
72 | dec $rounds | ||
73 | $movkey ($key),$rndkey1 | ||
74 | lea 16($key),$key | ||
75 | jnz .Loop_${p}1_$sn # loop body is 16 bytes | ||
76 | aes${p}last $rndkey1,$inout0 | ||
77 | ___ | ||
78 | }} | ||
79 | # void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key); | ||
80 | # | ||
81 | { my ($inp,$out,$key) = @_4args; | ||
82 | |||
83 | $code.=<<___; | ||
84 | .globl ${PREFIX}_encrypt | ||
85 | .type ${PREFIX}_encrypt,\@abi-omnipotent | ||
86 | .align 16 | ||
87 | ${PREFIX}_encrypt: | ||
88 | movups ($inp),$inout0 # load input | ||
89 | mov 240($key),$rounds # pull $rounds | ||
90 | ___ | ||
91 | &aesni_generate1("enc",$key,$rounds); | ||
92 | $code.=<<___; | ||
93 | movups $inout0,($out) # output | ||
94 | ret | ||
95 | .size ${PREFIX}_encrypt,.-${PREFIX}_encrypt | ||
96 | |||
97 | .globl ${PREFIX}_decrypt | ||
98 | .type ${PREFIX}_decrypt,\@abi-omnipotent | ||
99 | .align 16 | ||
100 | ${PREFIX}_decrypt: | ||
101 | movups ($inp),$inout0 # load input | ||
102 | mov 240($key),$rounds # pull $rounds | ||
103 | ___ | ||
104 | &aesni_generate1("dec",$key,$rounds); | ||
105 | $code.=<<___; | ||
106 | movups $inout0,($out) # output | ||
107 | ret | ||
108 | .size ${PREFIX}_decrypt, .-${PREFIX}_decrypt | ||
109 | ___ | ||
110 | } | ||
111 | |||
112 | # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave | ||
113 | # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec] | ||
114 | # latency is 6, it turned out that it can be scheduled only every | ||
115 | # *second* cycle. Thus 3x interleave is the one providing optimal | ||
116 | # utilization, i.e. when subroutine's throughput is virtually same as | ||
117 | # of non-interleaved subroutine [for number of input blocks up to 3]. | ||
118 | # This is why it makes no sense to implement 2x subroutine. As soon | ||
119 | # as/if Intel improves throughput by making it possible to schedule | ||
120 | # the instructions in question *every* cycles I would have to | ||
121 | # implement 6x interleave and use it in loop... | ||
122 | sub aesni_generate3 { | ||
123 | my $dir=shift; | ||
124 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
125 | # preserved. $inout[0-2] is cipher/clear text... | ||
126 | $code.=<<___; | ||
127 | .type _aesni_${dir}rypt3,\@abi-omnipotent | ||
128 | .align 16 | ||
129 | _aesni_${dir}rypt3: | ||
130 | $movkey ($key),$rndkey0 | ||
131 | shr \$1,$rounds | ||
132 | $movkey 16($key),$rndkey1 | ||
133 | lea 32($key),$key | ||
134 | pxor $rndkey0,$inout0 | ||
135 | pxor $rndkey0,$inout1 | ||
136 | pxor $rndkey0,$inout2 | ||
137 | |||
138 | .L${dir}_loop3: | ||
139 | aes${dir} $rndkey1,$inout0 | ||
140 | $movkey ($key),$rndkey0 | ||
141 | aes${dir} $rndkey1,$inout1 | ||
142 | dec $rounds | ||
143 | aes${dir} $rndkey1,$inout2 | ||
144 | aes${dir} $rndkey0,$inout0 | ||
145 | $movkey 16($key),$rndkey1 | ||
146 | aes${dir} $rndkey0,$inout1 | ||
147 | lea 32($key),$key | ||
148 | aes${dir} $rndkey0,$inout2 | ||
149 | jnz .L${dir}_loop3 | ||
150 | |||
151 | aes${dir} $rndkey1,$inout0 | ||
152 | $movkey ($key),$rndkey0 | ||
153 | aes${dir} $rndkey1,$inout1 | ||
154 | aes${dir} $rndkey1,$inout2 | ||
155 | aes${dir}last $rndkey0,$inout0 | ||
156 | aes${dir}last $rndkey0,$inout1 | ||
157 | aes${dir}last $rndkey0,$inout2 | ||
158 | ret | ||
159 | .size _aesni_${dir}rypt3,.-_aesni_${dir}rypt3 | ||
160 | ___ | ||
161 | } | ||
162 | # 4x interleave is implemented to improve small block performance, | ||
163 | # most notably [and naturally] 4 block by ~30%. One can argue that one | ||
164 | # should have implemented 5x as well, but improvement would be <20%, | ||
165 | # so it's not worth it... | ||
166 | sub aesni_generate4 { | ||
167 | my $dir=shift; | ||
168 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
169 | # preserved. $inout[0-3] is cipher/clear text... | ||
170 | $code.=<<___; | ||
171 | .type _aesni_${dir}rypt4,\@abi-omnipotent | ||
172 | .align 16 | ||
173 | _aesni_${dir}rypt4: | ||
174 | $movkey ($key),$rndkey0 | ||
175 | shr \$1,$rounds | ||
176 | $movkey 16($key),$rndkey1 | ||
177 | lea 32($key),$key | ||
178 | pxor $rndkey0,$inout0 | ||
179 | pxor $rndkey0,$inout1 | ||
180 | pxor $rndkey0,$inout2 | ||
181 | pxor $rndkey0,$inout3 | ||
182 | |||
183 | .L${dir}_loop4: | ||
184 | aes${dir} $rndkey1,$inout0 | ||
185 | $movkey ($key),$rndkey0 | ||
186 | aes${dir} $rndkey1,$inout1 | ||
187 | dec $rounds | ||
188 | aes${dir} $rndkey1,$inout2 | ||
189 | aes${dir} $rndkey1,$inout3 | ||
190 | aes${dir} $rndkey0,$inout0 | ||
191 | $movkey 16($key),$rndkey1 | ||
192 | aes${dir} $rndkey0,$inout1 | ||
193 | lea 32($key),$key | ||
194 | aes${dir} $rndkey0,$inout2 | ||
195 | aes${dir} $rndkey0,$inout3 | ||
196 | jnz .L${dir}_loop4 | ||
197 | |||
198 | aes${dir} $rndkey1,$inout0 | ||
199 | $movkey ($key),$rndkey0 | ||
200 | aes${dir} $rndkey1,$inout1 | ||
201 | aes${dir} $rndkey1,$inout2 | ||
202 | aes${dir} $rndkey1,$inout3 | ||
203 | aes${dir}last $rndkey0,$inout0 | ||
204 | aes${dir}last $rndkey0,$inout1 | ||
205 | aes${dir}last $rndkey0,$inout2 | ||
206 | aes${dir}last $rndkey0,$inout3 | ||
207 | ret | ||
208 | .size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4 | ||
209 | ___ | ||
210 | } | ||
211 | &aesni_generate3("enc") if ($PREFIX eq "aesni"); | ||
212 | &aesni_generate3("dec"); | ||
213 | &aesni_generate4("enc") if ($PREFIX eq "aesni"); | ||
214 | &aesni_generate4("dec"); | ||
215 | |||
216 | if ($PREFIX eq "aesni") { | ||
217 | # void aesni_ecb_encrypt (const void *in, void *out, | ||
218 | # size_t length, const AES_KEY *key, | ||
219 | # int enc); | ||
220 | $code.=<<___; | ||
221 | .globl aesni_ecb_encrypt | ||
222 | .type aesni_ecb_encrypt,\@function,5 | ||
223 | .align 16 | ||
224 | aesni_ecb_encrypt: | ||
225 | cmp \$16,$len # check length | ||
226 | jb .Lecb_ret | ||
227 | |||
228 | mov 240($key),$rounds # pull $rounds | ||
229 | and \$-16,$len | ||
230 | mov $key,$key_ # backup $key | ||
231 | test %r8d,%r8d # 5th argument | ||
232 | mov $rounds,$rnds_ # backup $rounds | ||
233 | jz .Lecb_decrypt | ||
234 | #--------------------------- ECB ENCRYPT ------------------------------# | ||
235 | sub \$0x40,$len | ||
236 | jbe .Lecb_enc_tail | ||
237 | jmp .Lecb_enc_loop3 | ||
238 | .align 16 | ||
239 | .Lecb_enc_loop3: | ||
240 | movups ($inp),$inout0 | ||
241 | movups 0x10($inp),$inout1 | ||
242 | movups 0x20($inp),$inout2 | ||
243 | call _aesni_encrypt3 | ||
244 | sub \$0x30,$len | ||
245 | lea 0x30($inp),$inp | ||
246 | lea 0x30($out),$out | ||
247 | movups $inout0,-0x30($out) | ||
248 | mov $rnds_,$rounds # restore $rounds | ||
249 | movups $inout1,-0x20($out) | ||
250 | mov $key_,$key # restore $key | ||
251 | movups $inout2,-0x10($out) | ||
252 | ja .Lecb_enc_loop3 | ||
253 | |||
254 | .Lecb_enc_tail: | ||
255 | add \$0x40,$len | ||
256 | jz .Lecb_ret | ||
257 | |||
258 | cmp \$0x10,$len | ||
259 | movups ($inp),$inout0 | ||
260 | je .Lecb_enc_one | ||
261 | cmp \$0x20,$len | ||
262 | movups 0x10($inp),$inout1 | ||
263 | je .Lecb_enc_two | ||
264 | cmp \$0x30,$len | ||
265 | movups 0x20($inp),$inout2 | ||
266 | je .Lecb_enc_three | ||
267 | movups 0x30($inp),$inout3 | ||
268 | call _aesni_encrypt4 | ||
269 | movups $inout0,($out) | ||
270 | movups $inout1,0x10($out) | ||
271 | movups $inout2,0x20($out) | ||
272 | movups $inout3,0x30($out) | ||
273 | jmp .Lecb_ret | ||
274 | .align 16 | ||
275 | .Lecb_enc_one: | ||
276 | ___ | ||
277 | &aesni_generate1("enc",$key,$rounds); | ||
278 | $code.=<<___; | ||
279 | movups $inout0,($out) | ||
280 | jmp .Lecb_ret | ||
281 | .align 16 | ||
282 | .Lecb_enc_two: | ||
283 | call _aesni_encrypt3 | ||
284 | movups $inout0,($out) | ||
285 | movups $inout1,0x10($out) | ||
286 | jmp .Lecb_ret | ||
287 | .align 16 | ||
288 | .Lecb_enc_three: | ||
289 | call _aesni_encrypt3 | ||
290 | movups $inout0,($out) | ||
291 | movups $inout1,0x10($out) | ||
292 | movups $inout2,0x20($out) | ||
293 | jmp .Lecb_ret | ||
294 | #--------------------------- ECB DECRYPT ------------------------------# | ||
295 | .align 16 | ||
296 | .Lecb_decrypt: | ||
297 | sub \$0x40,$len | ||
298 | jbe .Lecb_dec_tail | ||
299 | jmp .Lecb_dec_loop3 | ||
300 | .align 16 | ||
301 | .Lecb_dec_loop3: | ||
302 | movups ($inp),$inout0 | ||
303 | movups 0x10($inp),$inout1 | ||
304 | movups 0x20($inp),$inout2 | ||
305 | call _aesni_decrypt3 | ||
306 | sub \$0x30,$len | ||
307 | lea 0x30($inp),$inp | ||
308 | lea 0x30($out),$out | ||
309 | movups $inout0,-0x30($out) | ||
310 | mov $rnds_,$rounds # restore $rounds | ||
311 | movups $inout1,-0x20($out) | ||
312 | mov $key_,$key # restore $key | ||
313 | movups $inout2,-0x10($out) | ||
314 | ja .Lecb_dec_loop3 | ||
315 | |||
316 | .Lecb_dec_tail: | ||
317 | add \$0x40,$len | ||
318 | jz .Lecb_ret | ||
319 | |||
320 | cmp \$0x10,$len | ||
321 | movups ($inp),$inout0 | ||
322 | je .Lecb_dec_one | ||
323 | cmp \$0x20,$len | ||
324 | movups 0x10($inp),$inout1 | ||
325 | je .Lecb_dec_two | ||
326 | cmp \$0x30,$len | ||
327 | movups 0x20($inp),$inout2 | ||
328 | je .Lecb_dec_three | ||
329 | movups 0x30($inp),$inout3 | ||
330 | call _aesni_decrypt4 | ||
331 | movups $inout0,($out) | ||
332 | movups $inout1,0x10($out) | ||
333 | movups $inout2,0x20($out) | ||
334 | movups $inout3,0x30($out) | ||
335 | jmp .Lecb_ret | ||
336 | .align 16 | ||
337 | .Lecb_dec_one: | ||
338 | ___ | ||
339 | &aesni_generate1("dec",$key,$rounds); | ||
340 | $code.=<<___; | ||
341 | movups $inout0,($out) | ||
342 | jmp .Lecb_ret | ||
343 | .align 16 | ||
344 | .Lecb_dec_two: | ||
345 | call _aesni_decrypt3 | ||
346 | movups $inout0,($out) | ||
347 | movups $inout1,0x10($out) | ||
348 | jmp .Lecb_ret | ||
349 | .align 16 | ||
350 | .Lecb_dec_three: | ||
351 | call _aesni_decrypt3 | ||
352 | movups $inout0,($out) | ||
353 | movups $inout1,0x10($out) | ||
354 | movups $inout2,0x20($out) | ||
355 | |||
356 | .Lecb_ret: | ||
357 | ret | ||
358 | .size aesni_ecb_encrypt,.-aesni_ecb_encrypt | ||
359 | ___ | ||
360 | } | ||
361 | |||
362 | # void $PREFIX_cbc_encrypt (const void *inp, void *out, | ||
363 | # size_t length, const AES_KEY *key, | ||
364 | # unsigned char *ivp,const int enc); | ||
365 | $reserved = $win64?0x40:-0x18; # used in decrypt | ||
366 | $code.=<<___; | ||
367 | .globl ${PREFIX}_cbc_encrypt | ||
368 | .type ${PREFIX}_cbc_encrypt,\@function,6 | ||
369 | .align 16 | ||
370 | ${PREFIX}_cbc_encrypt: | ||
371 | test $len,$len # check length | ||
372 | jz .Lcbc_ret | ||
373 | |||
374 | mov 240($key),$rnds_ # pull $rounds | ||
375 | mov $key,$key_ # backup $key | ||
376 | test %r9d,%r9d # 6th argument | ||
377 | jz .Lcbc_decrypt | ||
378 | #--------------------------- CBC ENCRYPT ------------------------------# | ||
379 | movups ($ivp),$inout0 # load iv as initial state | ||
380 | cmp \$16,$len | ||
381 | mov $rnds_,$rounds | ||
382 | jb .Lcbc_enc_tail | ||
383 | sub \$16,$len | ||
384 | jmp .Lcbc_enc_loop | ||
385 | .align 16 | ||
386 | .Lcbc_enc_loop: | ||
387 | movups ($inp),$inout1 # load input | ||
388 | lea 16($inp),$inp | ||
389 | pxor $inout1,$inout0 | ||
390 | ___ | ||
391 | &aesni_generate1("enc",$key,$rounds); | ||
392 | $code.=<<___; | ||
393 | sub \$16,$len | ||
394 | lea 16($out),$out | ||
395 | mov $rnds_,$rounds # restore $rounds | ||
396 | mov $key_,$key # restore $key | ||
397 | movups $inout0,-16($out) # store output | ||
398 | jnc .Lcbc_enc_loop | ||
399 | add \$16,$len | ||
400 | jnz .Lcbc_enc_tail | ||
401 | movups $inout0,($ivp) | ||
402 | jmp .Lcbc_ret | ||
403 | |||
404 | .Lcbc_enc_tail: | ||
405 | mov $len,%rcx # zaps $key | ||
406 | xchg $inp,$out # $inp is %rsi and $out is %rdi now | ||
407 | .long 0x9066A4F3 # rep movsb | ||
408 | mov \$16,%ecx # zero tail | ||
409 | sub $len,%rcx | ||
410 | xor %eax,%eax | ||
411 | .long 0x9066AAF3 # rep stosb | ||
412 | lea -16(%rdi),%rdi # rewind $out by 1 block | ||
413 | mov $rnds_,$rounds # restore $rounds | ||
414 | mov %rdi,%rsi # $inp and $out are the same | ||
415 | mov $key_,$key # restore $key | ||
416 | xor $len,$len # len=16 | ||
417 | jmp .Lcbc_enc_loop # one more spin | ||
418 | #--------------------------- CBC DECRYPT ------------------------------# | ||
419 | .align 16 | ||
420 | .Lcbc_decrypt: | ||
421 | ___ | ||
422 | $code.=<<___ if ($win64); | ||
423 | lea -0x58(%rsp),%rsp | ||
424 | movaps %xmm6,(%rsp) | ||
425 | movaps %xmm7,0x10(%rsp) | ||
426 | movaps %xmm8,0x20(%rsp) | ||
427 | movaps %xmm9,0x30(%rsp) | ||
428 | .Lcbc_decrypt_body: | ||
429 | ___ | ||
430 | $code.=<<___; | ||
431 | movups ($ivp),$iv | ||
432 | sub \$0x40,$len | ||
433 | mov $rnds_,$rounds | ||
434 | jbe .Lcbc_dec_tail | ||
435 | jmp .Lcbc_dec_loop3 | ||
436 | .align 16 | ||
437 | .Lcbc_dec_loop3: | ||
438 | movups ($inp),$inout0 | ||
439 | movups 0x10($inp),$inout1 | ||
440 | movups 0x20($inp),$inout2 | ||
441 | movaps $inout0,$in0 | ||
442 | movaps $inout1,$in1 | ||
443 | movaps $inout2,$in2 | ||
444 | call _aesni_decrypt3 | ||
445 | sub \$0x30,$len | ||
446 | lea 0x30($inp),$inp | ||
447 | lea 0x30($out),$out | ||
448 | pxor $iv,$inout0 | ||
449 | pxor $in0,$inout1 | ||
450 | movaps $in2,$iv | ||
451 | pxor $in1,$inout2 | ||
452 | movups $inout0,-0x30($out) | ||
453 | mov $rnds_,$rounds # restore $rounds | ||
454 | movups $inout1,-0x20($out) | ||
455 | mov $key_,$key # restore $key | ||
456 | movups $inout2,-0x10($out) | ||
457 | ja .Lcbc_dec_loop3 | ||
458 | |||
459 | .Lcbc_dec_tail: | ||
460 | add \$0x40,$len | ||
461 | movups $iv,($ivp) | ||
462 | jz .Lcbc_dec_ret | ||
463 | |||
464 | movups ($inp),$inout0 | ||
465 | cmp \$0x10,$len | ||
466 | movaps $inout0,$in0 | ||
467 | jbe .Lcbc_dec_one | ||
468 | movups 0x10($inp),$inout1 | ||
469 | cmp \$0x20,$len | ||
470 | movaps $inout1,$in1 | ||
471 | jbe .Lcbc_dec_two | ||
472 | movups 0x20($inp),$inout2 | ||
473 | cmp \$0x30,$len | ||
474 | movaps $inout2,$in2 | ||
475 | jbe .Lcbc_dec_three | ||
476 | movups 0x30($inp),$inout3 | ||
477 | call _aesni_decrypt4 | ||
478 | pxor $iv,$inout0 | ||
479 | movups 0x30($inp),$iv | ||
480 | pxor $in0,$inout1 | ||
481 | movups $inout0,($out) | ||
482 | pxor $in1,$inout2 | ||
483 | movups $inout1,0x10($out) | ||
484 | pxor $in2,$inout3 | ||
485 | movups $inout2,0x20($out) | ||
486 | movaps $inout3,$inout0 | ||
487 | lea 0x30($out),$out | ||
488 | jmp .Lcbc_dec_tail_collected | ||
489 | .align 16 | ||
490 | .Lcbc_dec_one: | ||
491 | ___ | ||
492 | &aesni_generate1("dec",$key,$rounds); | ||
493 | $code.=<<___; | ||
494 | pxor $iv,$inout0 | ||
495 | movaps $in0,$iv | ||
496 | jmp .Lcbc_dec_tail_collected | ||
497 | .align 16 | ||
498 | .Lcbc_dec_two: | ||
499 | call _aesni_decrypt3 | ||
500 | pxor $iv,$inout0 | ||
501 | pxor $in0,$inout1 | ||
502 | movups $inout0,($out) | ||
503 | movaps $in1,$iv | ||
504 | movaps $inout1,$inout0 | ||
505 | lea 0x10($out),$out | ||
506 | jmp .Lcbc_dec_tail_collected | ||
507 | .align 16 | ||
508 | .Lcbc_dec_three: | ||
509 | call _aesni_decrypt3 | ||
510 | pxor $iv,$inout0 | ||
511 | pxor $in0,$inout1 | ||
512 | movups $inout0,($out) | ||
513 | pxor $in1,$inout2 | ||
514 | movups $inout1,0x10($out) | ||
515 | movaps $in2,$iv | ||
516 | movaps $inout2,$inout0 | ||
517 | lea 0x20($out),$out | ||
518 | jmp .Lcbc_dec_tail_collected | ||
519 | .align 16 | ||
520 | .Lcbc_dec_tail_collected: | ||
521 | and \$15,$len | ||
522 | movups $iv,($ivp) | ||
523 | jnz .Lcbc_dec_tail_partial | ||
524 | movups $inout0,($out) | ||
525 | jmp .Lcbc_dec_ret | ||
526 | .Lcbc_dec_tail_partial: | ||
527 | movaps $inout0,$reserved(%rsp) | ||
528 | mov $out,%rdi | ||
529 | mov $len,%rcx | ||
530 | lea $reserved(%rsp),%rsi | ||
531 | .long 0x9066A4F3 # rep movsb | ||
532 | |||
533 | .Lcbc_dec_ret: | ||
534 | ___ | ||
535 | $code.=<<___ if ($win64); | ||
536 | movaps (%rsp),%xmm6 | ||
537 | movaps 0x10(%rsp),%xmm7 | ||
538 | movaps 0x20(%rsp),%xmm8 | ||
539 | movaps 0x30(%rsp),%xmm9 | ||
540 | lea 0x58(%rsp),%rsp | ||
541 | ___ | ||
542 | $code.=<<___; | ||
543 | .Lcbc_ret: | ||
544 | ret | ||
545 | .size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt | ||
546 | ___ | ||
547 | |||
548 | # int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey, | ||
549 | # int bits, AES_KEY *key) | ||
550 | { my ($inp,$bits,$key) = @_4args; | ||
551 | $bits =~ s/%r/%e/; | ||
552 | |||
553 | $code.=<<___; | ||
554 | .globl ${PREFIX}_set_decrypt_key | ||
555 | .type ${PREFIX}_set_decrypt_key,\@abi-omnipotent | ||
556 | .align 16 | ||
557 | ${PREFIX}_set_decrypt_key: | ||
558 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
559 | call _aesni_set_encrypt_key | ||
560 | shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key | ||
561 | test %eax,%eax | ||
562 | jnz .Ldec_key_ret | ||
563 | lea 16($key,$bits),$inp # points at the end of key schedule | ||
564 | |||
565 | $movkey ($key),%xmm0 # just swap | ||
566 | $movkey ($inp),%xmm1 | ||
567 | $movkey %xmm0,($inp) | ||
568 | $movkey %xmm1,($key) | ||
569 | lea 16($key),$key | ||
570 | lea -16($inp),$inp | ||
571 | |||
572 | .Ldec_key_inverse: | ||
573 | $movkey ($key),%xmm0 # swap and inverse | ||
574 | $movkey ($inp),%xmm1 | ||
575 | aesimc %xmm0,%xmm0 | ||
576 | aesimc %xmm1,%xmm1 | ||
577 | lea 16($key),$key | ||
578 | lea -16($inp),$inp | ||
579 | cmp $key,$inp | ||
580 | $movkey %xmm0,16($inp) | ||
581 | $movkey %xmm1,-16($key) | ||
582 | ja .Ldec_key_inverse | ||
583 | |||
584 | $movkey ($key),%xmm0 # inverse middle | ||
585 | aesimc %xmm0,%xmm0 | ||
586 | $movkey %xmm0,($inp) | ||
587 | .Ldec_key_ret: | ||
588 | add \$8,%rsp | ||
589 | ret | ||
590 | .LSEH_end_set_decrypt_key: | ||
591 | .size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key | ||
592 | ___ | ||
593 | |||
594 | # This is based on submission by | ||
595 | # | ||
596 | # Huang Ying <ying.huang@intel.com> | ||
597 | # Vinodh Gopal <vinodh.gopal@intel.com> | ||
598 | # Kahraman Akdemir | ||
599 | # | ||
600 | # Agressively optimized in respect to aeskeygenassist's critical path | ||
601 | # and is contained in %xmm0-5 to meet Win64 ABI requirement. | ||
602 | # | ||
603 | $code.=<<___; | ||
604 | .globl ${PREFIX}_set_encrypt_key | ||
605 | .type ${PREFIX}_set_encrypt_key,\@abi-omnipotent | ||
606 | .align 16 | ||
607 | ${PREFIX}_set_encrypt_key: | ||
608 | _aesni_set_encrypt_key: | ||
609 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
610 | test $inp,$inp | ||
611 | mov \$-1,%rax | ||
612 | jz .Lenc_key_ret | ||
613 | test $key,$key | ||
614 | jz .Lenc_key_ret | ||
615 | |||
616 | movups ($inp),%xmm0 # pull first 128 bits of *userKey | ||
617 | pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0 | ||
618 | lea 16($key),%rax | ||
619 | cmp \$256,$bits | ||
620 | je .L14rounds | ||
621 | cmp \$192,$bits | ||
622 | je .L12rounds | ||
623 | cmp \$128,$bits | ||
624 | jne .Lbad_keybits | ||
625 | |||
626 | .L10rounds: | ||
627 | mov \$9,$bits # 10 rounds for 128-bit key | ||
628 | $movkey %xmm0,($key) # round 0 | ||
629 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 | ||
630 | call .Lkey_expansion_128_cold | ||
631 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 2 | ||
632 | call .Lkey_expansion_128 | ||
633 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 3 | ||
634 | call .Lkey_expansion_128 | ||
635 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 4 | ||
636 | call .Lkey_expansion_128 | ||
637 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 5 | ||
638 | call .Lkey_expansion_128 | ||
639 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 6 | ||
640 | call .Lkey_expansion_128 | ||
641 | aeskeygenassist \$0x40,%xmm0,%xmm1 # round 7 | ||
642 | call .Lkey_expansion_128 | ||
643 | aeskeygenassist \$0x80,%xmm0,%xmm1 # round 8 | ||
644 | call .Lkey_expansion_128 | ||
645 | aeskeygenassist \$0x1b,%xmm0,%xmm1 # round 9 | ||
646 | call .Lkey_expansion_128 | ||
647 | aeskeygenassist \$0x36,%xmm0,%xmm1 # round 10 | ||
648 | call .Lkey_expansion_128 | ||
649 | $movkey %xmm0,(%rax) | ||
650 | mov $bits,80(%rax) # 240(%rdx) | ||
651 | xor %eax,%eax | ||
652 | jmp .Lenc_key_ret | ||
653 | |||
654 | .align 16 | ||
655 | .L12rounds: | ||
656 | movq 16($inp),%xmm2 # remaining 1/3 of *userKey | ||
657 | mov \$11,$bits # 12 rounds for 192 | ||
658 | $movkey %xmm0,($key) # round 0 | ||
659 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 | ||
660 | call .Lkey_expansion_192a_cold | ||
661 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 2,3 | ||
662 | call .Lkey_expansion_192b | ||
663 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 4,5 | ||
664 | call .Lkey_expansion_192a | ||
665 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 5,6 | ||
666 | call .Lkey_expansion_192b | ||
667 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 7,8 | ||
668 | call .Lkey_expansion_192a | ||
669 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 8,9 | ||
670 | call .Lkey_expansion_192b | ||
671 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 10,11 | ||
672 | call .Lkey_expansion_192a | ||
673 | aeskeygenassist \$0x80,%xmm2,%xmm1 # round 11,12 | ||
674 | call .Lkey_expansion_192b | ||
675 | $movkey %xmm0,(%rax) | ||
676 | mov $bits,48(%rax) # 240(%rdx) | ||
677 | xor %rax, %rax | ||
678 | jmp .Lenc_key_ret | ||
679 | |||
680 | .align 16 | ||
681 | .L14rounds: | ||
682 | movups 16($inp),%xmm2 # remaning half of *userKey | ||
683 | mov \$13,$bits # 14 rounds for 256 | ||
684 | lea 16(%rax),%rax | ||
685 | $movkey %xmm0,($key) # round 0 | ||
686 | $movkey %xmm2,16($key) # round 1 | ||
687 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2 | ||
688 | call .Lkey_expansion_256a_cold | ||
689 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 3 | ||
690 | call .Lkey_expansion_256b | ||
691 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 4 | ||
692 | call .Lkey_expansion_256a | ||
693 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 5 | ||
694 | call .Lkey_expansion_256b | ||
695 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 6 | ||
696 | call .Lkey_expansion_256a | ||
697 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 7 | ||
698 | call .Lkey_expansion_256b | ||
699 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 8 | ||
700 | call .Lkey_expansion_256a | ||
701 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 9 | ||
702 | call .Lkey_expansion_256b | ||
703 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 10 | ||
704 | call .Lkey_expansion_256a | ||
705 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 11 | ||
706 | call .Lkey_expansion_256b | ||
707 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 12 | ||
708 | call .Lkey_expansion_256a | ||
709 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 13 | ||
710 | call .Lkey_expansion_256b | ||
711 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 14 | ||
712 | call .Lkey_expansion_256a | ||
713 | $movkey %xmm0,(%rax) | ||
714 | mov $bits,16(%rax) # 240(%rdx) | ||
715 | xor %rax,%rax | ||
716 | jmp .Lenc_key_ret | ||
717 | |||
718 | .align 16 | ||
719 | .Lbad_keybits: | ||
720 | mov \$-2,%rax | ||
721 | .Lenc_key_ret: | ||
722 | add \$8,%rsp | ||
723 | ret | ||
724 | .LSEH_end_set_encrypt_key: | ||
725 | |||
726 | .align 16 | ||
727 | .Lkey_expansion_128: | ||
728 | $movkey %xmm0,(%rax) | ||
729 | lea 16(%rax),%rax | ||
730 | .Lkey_expansion_128_cold: | ||
731 | shufps \$0b00010000,%xmm0,%xmm4 | ||
732 | pxor %xmm4, %xmm0 | ||
733 | shufps \$0b10001100,%xmm0,%xmm4 | ||
734 | pxor %xmm4, %xmm0 | ||
735 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
736 | pxor %xmm1,%xmm0 | ||
737 | ret | ||
738 | |||
739 | .align 16 | ||
740 | .Lkey_expansion_192a: | ||
741 | $movkey %xmm0,(%rax) | ||
742 | lea 16(%rax),%rax | ||
743 | .Lkey_expansion_192a_cold: | ||
744 | movaps %xmm2, %xmm5 | ||
745 | .Lkey_expansion_192b_warm: | ||
746 | shufps \$0b00010000,%xmm0,%xmm4 | ||
747 | movaps %xmm2,%xmm3 | ||
748 | pxor %xmm4,%xmm0 | ||
749 | shufps \$0b10001100,%xmm0,%xmm4 | ||
750 | pslldq \$4,%xmm3 | ||
751 | pxor %xmm4,%xmm0 | ||
752 | pshufd \$0b01010101,%xmm1,%xmm1 # critical path | ||
753 | pxor %xmm3,%xmm2 | ||
754 | pxor %xmm1,%xmm0 | ||
755 | pshufd \$0b11111111,%xmm0,%xmm3 | ||
756 | pxor %xmm3,%xmm2 | ||
757 | ret | ||
758 | |||
759 | .align 16 | ||
760 | .Lkey_expansion_192b: | ||
761 | movaps %xmm0,%xmm3 | ||
762 | shufps \$0b01000100,%xmm0,%xmm5 | ||
763 | $movkey %xmm5,(%rax) | ||
764 | shufps \$0b01001110,%xmm2,%xmm3 | ||
765 | $movkey %xmm3,16(%rax) | ||
766 | lea 32(%rax),%rax | ||
767 | jmp .Lkey_expansion_192b_warm | ||
768 | |||
769 | .align 16 | ||
770 | .Lkey_expansion_256a: | ||
771 | $movkey %xmm2,(%rax) | ||
772 | lea 16(%rax),%rax | ||
773 | .Lkey_expansion_256a_cold: | ||
774 | shufps \$0b00010000,%xmm0,%xmm4 | ||
775 | pxor %xmm4,%xmm0 | ||
776 | shufps \$0b10001100,%xmm0,%xmm4 | ||
777 | pxor %xmm4,%xmm0 | ||
778 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
779 | pxor %xmm1,%xmm0 | ||
780 | ret | ||
781 | |||
782 | .align 16 | ||
783 | .Lkey_expansion_256b: | ||
784 | $movkey %xmm0,(%rax) | ||
785 | lea 16(%rax),%rax | ||
786 | |||
787 | shufps \$0b00010000,%xmm2,%xmm4 | ||
788 | pxor %xmm4,%xmm2 | ||
789 | shufps \$0b10001100,%xmm2,%xmm4 | ||
790 | pxor %xmm4,%xmm2 | ||
791 | pshufd \$0b10101010,%xmm1,%xmm1 # critical path | ||
792 | pxor %xmm1,%xmm2 | ||
793 | ret | ||
794 | .size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key | ||
795 | ___ | ||
796 | } | ||
797 | |||
798 | $code.=<<___; | ||
799 | .asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>" | ||
800 | .align 64 | ||
801 | ___ | ||
802 | |||
803 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | ||
804 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | ||
805 | if ($win64) { | ||
806 | $rec="%rcx"; | ||
807 | $frame="%rdx"; | ||
808 | $context="%r8"; | ||
809 | $disp="%r9"; | ||
810 | |||
811 | $code.=<<___; | ||
812 | .extern __imp_RtlVirtualUnwind | ||
813 | .type cbc_se_handler,\@abi-omnipotent | ||
814 | .align 16 | ||
815 | cbc_se_handler: | ||
816 | push %rsi | ||
817 | push %rdi | ||
818 | push %rbx | ||
819 | push %rbp | ||
820 | push %r12 | ||
821 | push %r13 | ||
822 | push %r14 | ||
823 | push %r15 | ||
824 | pushfq | ||
825 | sub \$64,%rsp | ||
826 | |||
827 | mov 152($context),%rax # pull context->Rsp | ||
828 | mov 248($context),%rbx # pull context->Rip | ||
829 | |||
830 | lea .Lcbc_decrypt(%rip),%r10 | ||
831 | cmp %r10,%rbx # context->Rip<"prologue" label | ||
832 | jb .Lin_prologue | ||
833 | |||
834 | lea .Lcbc_decrypt_body(%rip),%r10 | ||
835 | cmp %r10,%rbx # context->Rip<cbc_decrypt_body | ||
836 | jb .Lrestore_rax | ||
837 | |||
838 | lea .Lcbc_ret(%rip),%r10 | ||
839 | cmp %r10,%rbx # context->Rip>="epilogue" label | ||
840 | jae .Lin_prologue | ||
841 | |||
842 | lea 0(%rax),%rsi # top of stack | ||
843 | lea 512($context),%rdi # &context.Xmm6 | ||
844 | mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax) | ||
845 | .long 0xa548f3fc # cld; rep movsq | ||
846 | lea 0x58(%rax),%rax # adjust stack pointer | ||
847 | jmp .Lin_prologue | ||
848 | |||
849 | .Lrestore_rax: | ||
850 | mov 120($context),%rax | ||
851 | .Lin_prologue: | ||
852 | mov 8(%rax),%rdi | ||
853 | mov 16(%rax),%rsi | ||
854 | mov %rax,152($context) # restore context->Rsp | ||
855 | mov %rsi,168($context) # restore context->Rsi | ||
856 | mov %rdi,176($context) # restore context->Rdi | ||
857 | |||
858 | jmp .Lcommon_seh_exit | ||
859 | .size cbc_se_handler,.-cbc_se_handler | ||
860 | |||
861 | .type ecb_se_handler,\@abi-omnipotent | ||
862 | .align 16 | ||
863 | ecb_se_handler: | ||
864 | push %rsi | ||
865 | push %rdi | ||
866 | push %rbx | ||
867 | push %rbp | ||
868 | push %r12 | ||
869 | push %r13 | ||
870 | push %r14 | ||
871 | push %r15 | ||
872 | pushfq | ||
873 | sub \$64,%rsp | ||
874 | |||
875 | mov 152($context),%rax # pull context->Rsp | ||
876 | mov 8(%rax),%rdi | ||
877 | mov 16(%rax),%rsi | ||
878 | mov %rsi,168($context) # restore context->Rsi | ||
879 | mov %rdi,176($context) # restore context->Rdi | ||
880 | |||
881 | .Lcommon_seh_exit: | ||
882 | |||
883 | mov 40($disp),%rdi # disp->ContextRecord | ||
884 | mov $context,%rsi # context | ||
885 | mov \$154,%ecx # sizeof(CONTEXT) | ||
886 | .long 0xa548f3fc # cld; rep movsq | ||
887 | |||
888 | mov $disp,%rsi | ||
889 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | ||
890 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | ||
891 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | ||
892 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | ||
893 | mov 40(%rsi),%r10 # disp->ContextRecord | ||
894 | lea 56(%rsi),%r11 # &disp->HandlerData | ||
895 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | ||
896 | mov %r10,32(%rsp) # arg5 | ||
897 | mov %r11,40(%rsp) # arg6 | ||
898 | mov %r12,48(%rsp) # arg7 | ||
899 | mov %rcx,56(%rsp) # arg8, (NULL) | ||
900 | call *__imp_RtlVirtualUnwind(%rip) | ||
901 | |||
902 | mov \$1,%eax # ExceptionContinueSearch | ||
903 | add \$64,%rsp | ||
904 | popfq | ||
905 | pop %r15 | ||
906 | pop %r14 | ||
907 | pop %r13 | ||
908 | pop %r12 | ||
909 | pop %rbp | ||
910 | pop %rbx | ||
911 | pop %rdi | ||
912 | pop %rsi | ||
913 | ret | ||
914 | .size cbc_se_handler,.-cbc_se_handler | ||
915 | |||
916 | .section .pdata | ||
917 | .align 4 | ||
918 | .rva .LSEH_begin_${PREFIX}_ecb_encrypt | ||
919 | .rva .LSEH_end_${PREFIX}_ecb_encrypt | ||
920 | .rva .LSEH_info_ecb | ||
921 | |||
922 | .rva .LSEH_begin_${PREFIX}_cbc_encrypt | ||
923 | .rva .LSEH_end_${PREFIX}_cbc_encrypt | ||
924 | .rva .LSEH_info_cbc | ||
925 | |||
926 | .rva ${PREFIX}_set_decrypt_key | ||
927 | .rva .LSEH_end_set_decrypt_key | ||
928 | .rva .LSEH_info_key | ||
929 | |||
930 | .rva ${PREFIX}_set_encrypt_key | ||
931 | .rva .LSEH_end_set_encrypt_key | ||
932 | .rva .LSEH_info_key | ||
933 | .section .xdata | ||
934 | .align 8 | ||
935 | .LSEH_info_ecb: | ||
936 | .byte 9,0,0,0 | ||
937 | .rva ecb_se_handler | ||
938 | .LSEH_info_cbc: | ||
939 | .byte 9,0,0,0 | ||
940 | .rva cbc_se_handler | ||
941 | .LSEH_info_key: | ||
942 | .byte 0x01,0x04,0x01,0x00 | ||
943 | .byte 0x04,0x02,0x00,0x00 | ||
944 | ___ | ||
945 | } | ||
946 | |||
947 | sub rex { | ||
948 | local *opcode=shift; | ||
949 | my ($dst,$src)=@_; | ||
950 | |||
951 | if ($dst>=8 || $src>=8) { | ||
952 | $rex=0x40; | ||
953 | $rex|=0x04 if($dst>=8); | ||
954 | $rex|=0x01 if($src>=8); | ||
955 | push @opcode,$rex; | ||
956 | } | ||
957 | } | ||
958 | |||
959 | sub aesni { | ||
960 | my $line=shift; | ||
961 | my @opcode=(0x66); | ||
962 | |||
963 | if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
964 | rex(\@opcode,$4,$3); | ||
965 | push @opcode,0x0f,0x3a,0xdf; | ||
966 | push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M | ||
967 | my $c=$2; | ||
968 | push @opcode,$c=~/^0/?oct($c):$c; | ||
969 | return ".byte\t".join(',',@opcode); | ||
970 | } | ||
971 | elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
972 | my %opcodelet = ( | ||
973 | "aesimc" => 0xdb, | ||
974 | "aesenc" => 0xdc, "aesenclast" => 0xdd, | ||
975 | "aesdec" => 0xde, "aesdeclast" => 0xdf | ||
976 | ); | ||
977 | return undef if (!defined($opcodelet{$1})); | ||
978 | rex(\@opcode,$3,$2); | ||
979 | push @opcode,0x0f,0x38,$opcodelet{$1}; | ||
980 | push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M | ||
981 | return ".byte\t".join(',',@opcode); | ||
982 | } | ||
983 | return $line; | ||
984 | } | ||
985 | |||
986 | $code =~ s/\`([^\`]*)\`/eval($1)/gem; | ||
987 | $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; | ||
988 | |||
989 | print $code; | ||
990 | |||
991 | close STDOUT; | ||
992 | |||
diff --git a/src/lib/libcrypto/crypto-lib.com b/src/lib/libcrypto/crypto-lib.com index e72af90822..8f1c1d5a80 100644 --- a/src/lib/libcrypto/crypto-lib.com +++ b/src/lib/libcrypto/crypto-lib.com | |||
@@ -209,7 +209,8 @@ $ LIB_ENGINE = "eng_err,eng_lib,eng_list,eng_init,eng_ctrl,"+ - | |||
209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - | 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - |
210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - | 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - |
211 | "tb_cipher,tb_digest,"+ - | 211 | "tb_cipher,tb_digest,"+ - |
212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock" | 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock"+ - |
213 | "eng_aesni" | ||
213 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - | 214 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - |
214 | "aes_ctr,aes_ige,aes_wrap" | 215 | "aes_ctr,aes_ige,aes_wrap" |
215 | $ LIB_BUFFER = "buffer,buf_str,buf_err" | 216 | $ LIB_BUFFER = "buffer,buf_str,buf_err" |
diff --git a/src/lib/libcrypto/engine/Makefile b/src/lib/libcrypto/engine/Makefile index 0cc3722089..0736153075 100644 --- a/src/lib/libcrypto/engine/Makefile +++ b/src/lib/libcrypto/engine/Makefile | |||
@@ -21,12 +21,14 @@ LIBSRC= eng_err.c eng_lib.c eng_list.c eng_init.c eng_ctrl.c \ | |||
21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ | 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ |
22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ | 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ |
23 | tb_cipher.c tb_digest.c \ | 23 | tb_cipher.c tb_digest.c \ |
24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c | 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c \ |
25 | eng_aesni.c | ||
25 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ | 26 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ |
26 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ | 27 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ |
27 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ | 28 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ |
28 | tb_cipher.o tb_digest.o \ | 29 | tb_cipher.o tb_digest.o \ |
29 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o | 30 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o \ |
31 | eng_aesni.o | ||
30 | 32 | ||
31 | SRC= $(LIBSRC) | 33 | SRC= $(LIBSRC) |
32 | 34 | ||
@@ -82,6 +84,21 @@ clean: | |||
82 | 84 | ||
83 | # DO NOT DELETE THIS LINE -- make depend depends on it. | 85 | # DO NOT DELETE THIS LINE -- make depend depends on it. |
84 | 86 | ||
87 | eng_aesni.o: ../../e_os.h ../../include/openssl/aes.h | ||
88 | eng_aesni.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h | ||
89 | eng_aesni.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h | ||
90 | eng_aesni.o: ../../include/openssl/dso.h ../../include/openssl/e_os2.h | ||
91 | eng_aesni.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h | ||
92 | eng_aesni.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h | ||
93 | eng_aesni.o: ../../include/openssl/err.h ../../include/openssl/evp.h | ||
94 | eng_aesni.o: ../../include/openssl/lhash.h ../../include/openssl/modes.h | ||
95 | eng_aesni.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h | ||
96 | eng_aesni.o: ../../include/openssl/opensslconf.h | ||
97 | eng_aesni.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h | ||
98 | eng_aesni.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h | ||
99 | eng_aesni.o: ../../include/openssl/sha.h ../../include/openssl/stack.h | ||
100 | eng_aesni.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h | ||
101 | eng_aesni.o: ../../include/openssl/x509_vfy.h ../cryptlib.h eng_aesni.c | ||
85 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h | 102 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h |
86 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h | 103 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h |
87 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h | 104 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h |
diff --git a/src/lib/libcrypto/engine/eng_aesni.c b/src/lib/libcrypto/engine/eng_aesni.c new file mode 100644 index 0000000000..eeded1eefa --- /dev/null +++ b/src/lib/libcrypto/engine/eng_aesni.c | |||
@@ -0,0 +1,570 @@ | |||
1 | /* | ||
2 | * Support for Intel AES-NI intruction set | ||
3 | * Author: Huang Ying <ying.huang@intel.com> | ||
4 | * | ||
5 | * Intel AES-NI is a new set of Single Instruction Multiple Data | ||
6 | * (SIMD) instructions that are going to be introduced in the next | ||
7 | * generation of Intel processor, as of 2009. These instructions | ||
8 | * enable fast and secure data encryption and decryption, using the | ||
9 | * Advanced Encryption Standard (AES), defined by FIPS Publication | ||
10 | * number 197. The architecture introduces six instructions that | ||
11 | * offer full hardware support for AES. Four of them support high | ||
12 | * performance data encryption and decryption, and the other two | ||
13 | * instructions support the AES key expansion procedure. | ||
14 | * | ||
15 | * The white paper can be downloaded from: | ||
16 | * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf | ||
17 | * | ||
18 | * This file is based on engines/e_padlock.c | ||
19 | */ | ||
20 | |||
21 | /* ==================================================================== | ||
22 | * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved. | ||
23 | * | ||
24 | * Redistribution and use in source and binary forms, with or without | ||
25 | * modification, are permitted provided that the following conditions | ||
26 | * are met: | ||
27 | * | ||
28 | * 1. Redistributions of source code must retain the above copyright | ||
29 | * notice, this list of conditions and the following disclaimer. | ||
30 | * | ||
31 | * 2. Redistributions in binary form must reproduce the above copyright | ||
32 | * notice, this list of conditions and the following disclaimer in | ||
33 | * the documentation and/or other materials provided with the | ||
34 | * distribution. | ||
35 | * | ||
36 | * 3. All advertising materials mentioning features or use of this | ||
37 | * software must display the following acknowledgment: | ||
38 | * "This product includes software developed by the OpenSSL Project | ||
39 | * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" | ||
40 | * | ||
41 | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to | ||
42 | * endorse or promote products derived from this software without | ||
43 | * prior written permission. For written permission, please contact | ||
44 | * licensing@OpenSSL.org. | ||
45 | * | ||
46 | * 5. Products derived from this software may not be called "OpenSSL" | ||
47 | * nor may "OpenSSL" appear in their names without prior written | ||
48 | * permission of the OpenSSL Project. | ||
49 | * | ||
50 | * 6. Redistributions of any form whatsoever must retain the following | ||
51 | * acknowledgment: | ||
52 | * "This product includes software developed by the OpenSSL Project | ||
53 | * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" | ||
54 | * | ||
55 | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY | ||
56 | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
57 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
58 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR | ||
59 | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
60 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | ||
61 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | ||
62 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||
63 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | ||
64 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
65 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED | ||
66 | * OF THE POSSIBILITY OF SUCH DAMAGE. | ||
67 | * ==================================================================== | ||
68 | * | ||
69 | * This product includes cryptographic software written by Eric Young | ||
70 | * (eay@cryptsoft.com). This product includes software written by Tim | ||
71 | * Hudson (tjh@cryptsoft.com). | ||
72 | * | ||
73 | */ | ||
74 | |||
75 | |||
76 | #include <openssl/opensslconf.h> | ||
77 | |||
78 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AES_NI) && !defined(OPENSSL_NO_AES) | ||
79 | |||
80 | #include <stdio.h> | ||
81 | #include <assert.h> | ||
82 | #include "cryptlib.h" | ||
83 | #include <openssl/dso.h> | ||
84 | #include <openssl/engine.h> | ||
85 | #include <openssl/evp.h> | ||
86 | #include <openssl/aes.h> | ||
87 | #include <openssl/err.h> | ||
88 | |||
89 | /* AES-NI is available *ONLY* on some x86 CPUs. Not only that it | ||
90 | doesn't exist elsewhere, but it even can't be compiled on other | ||
91 | platforms! */ | ||
92 | #undef COMPILE_HW_AESNI | ||
93 | #if (defined(__x86_64) || defined(__x86_64__) || \ | ||
94 | defined(_M_AMD64) || defined(_M_X64) || \ | ||
95 | defined(OPENSSL_IA32_SSE2)) && !defined(OPENSSL_NO_ASM) && !defined(__i386__) | ||
96 | #define COMPILE_HW_AESNI | ||
97 | #endif | ||
98 | static ENGINE *ENGINE_aesni (void); | ||
99 | |||
100 | void ENGINE_load_aesni (void) | ||
101 | { | ||
102 | /* On non-x86 CPUs it just returns. */ | ||
103 | #ifdef COMPILE_HW_AESNI | ||
104 | ENGINE *toadd = ENGINE_aesni(); | ||
105 | if (!toadd) return; | ||
106 | ENGINE_add (toadd); | ||
107 | ENGINE_register_complete (toadd); | ||
108 | ENGINE_free (toadd); | ||
109 | ERR_clear_error (); | ||
110 | #endif | ||
111 | } | ||
112 | |||
113 | #ifdef COMPILE_HW_AESNI | ||
114 | int aesni_set_encrypt_key(const unsigned char *userKey, int bits, | ||
115 | AES_KEY *key); | ||
116 | int aesni_set_decrypt_key(const unsigned char *userKey, int bits, | ||
117 | AES_KEY *key); | ||
118 | |||
119 | void aesni_encrypt(const unsigned char *in, unsigned char *out, | ||
120 | const AES_KEY *key); | ||
121 | void aesni_decrypt(const unsigned char *in, unsigned char *out, | ||
122 | const AES_KEY *key); | ||
123 | |||
124 | void aesni_ecb_encrypt(const unsigned char *in, | ||
125 | unsigned char *out, | ||
126 | size_t length, | ||
127 | const AES_KEY *key, | ||
128 | int enc); | ||
129 | void aesni_cbc_encrypt(const unsigned char *in, | ||
130 | unsigned char *out, | ||
131 | size_t length, | ||
132 | const AES_KEY *key, | ||
133 | unsigned char *ivec, int enc); | ||
134 | |||
135 | /* Function for ENGINE detection and control */ | ||
136 | static int aesni_init(ENGINE *e); | ||
137 | |||
138 | /* Cipher Stuff */ | ||
139 | static int aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | ||
140 | const int **nids, int nid); | ||
141 | |||
142 | #define AESNI_MIN_ALIGN 16 | ||
143 | #define AESNI_ALIGN(x) \ | ||
144 | ((void *)(((unsigned long)(x)+AESNI_MIN_ALIGN-1)&~(AESNI_MIN_ALIGN-1))) | ||
145 | |||
146 | /* Engine names */ | ||
147 | static const char aesni_id[] = "aesni", | ||
148 | aesni_name[] = "Intel AES-NI engine", | ||
149 | no_aesni_name[] = "Intel AES-NI engine (no-aesni)"; | ||
150 | |||
151 | |||
152 | /* The input and output encrypted as though 128bit cfb mode is being | ||
153 | * used. The extra state information to record how much of the | ||
154 | * 128bit block we have used is contained in *num; | ||
155 | */ | ||
156 | static void aesni_cfb128_encrypt(const unsigned char *in, unsigned char *out, | ||
157 | unsigned int len, const void *key, | ||
158 | unsigned char ivec[16], int *num, | ||
159 | int enc) | ||
160 | { | ||
161 | unsigned int n; | ||
162 | size_t l = 0; | ||
163 | |||
164 | assert(in && out && key && ivec && num); | ||
165 | |||
166 | n = *num; | ||
167 | |||
168 | if (enc) { | ||
169 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
170 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
171 | while (n && len) { | ||
172 | *(out++) = ivec[n] ^= *(in++); | ||
173 | --len; | ||
174 | n = (n+1) % 16; | ||
175 | } | ||
176 | while (len>=16) { | ||
177 | aesni_encrypt(ivec, ivec, key); | ||
178 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
179 | *(size_t*)(out+n) = | ||
180 | *(size_t*)(ivec+n) ^= *(size_t*)(in+n); | ||
181 | } | ||
182 | len -= 16; | ||
183 | out += 16; | ||
184 | in += 16; | ||
185 | } | ||
186 | n = 0; | ||
187 | if (len) { | ||
188 | aesni_encrypt(ivec, ivec, key); | ||
189 | while (len--) { | ||
190 | out[n] = ivec[n] ^= in[n]; | ||
191 | ++n; | ||
192 | } | ||
193 | } | ||
194 | *num = n; | ||
195 | return; | ||
196 | } while (0); | ||
197 | /* the rest would be commonly eliminated by x86* compiler */ | ||
198 | #endif | ||
199 | while (l<len) { | ||
200 | if (n == 0) { | ||
201 | aesni_encrypt(ivec, ivec, key); | ||
202 | } | ||
203 | out[l] = ivec[n] ^= in[l]; | ||
204 | ++l; | ||
205 | n = (n+1) % 16; | ||
206 | } | ||
207 | *num = n; | ||
208 | } else { | ||
209 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
210 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
211 | while (n && len) { | ||
212 | unsigned char c; | ||
213 | *(out++) = ivec[n] ^ (c = *(in++)); ivec[n] = c; | ||
214 | --len; | ||
215 | n = (n+1) % 16; | ||
216 | } | ||
217 | while (len>=16) { | ||
218 | aesni_encrypt(ivec, ivec, key); | ||
219 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
220 | size_t t = *(size_t*)(in+n); | ||
221 | *(size_t*)(out+n) = *(size_t*)(ivec+n) ^ t; | ||
222 | *(size_t*)(ivec+n) = t; | ||
223 | } | ||
224 | len -= 16; | ||
225 | out += 16; | ||
226 | in += 16; | ||
227 | } | ||
228 | n = 0; | ||
229 | if (len) { | ||
230 | aesni_encrypt(ivec, ivec, key); | ||
231 | while (len--) { | ||
232 | unsigned char c; | ||
233 | out[n] = ivec[n] ^ (c = in[n]); ivec[n] = c; | ||
234 | ++n; | ||
235 | } | ||
236 | } | ||
237 | *num = n; | ||
238 | return; | ||
239 | } while (0); | ||
240 | /* the rest would be commonly eliminated by x86* compiler */ | ||
241 | #endif | ||
242 | while (l<len) { | ||
243 | unsigned char c; | ||
244 | if (n == 0) { | ||
245 | aesni_encrypt(ivec, ivec, key); | ||
246 | } | ||
247 | out[l] = ivec[n] ^ (c = in[l]); ivec[n] = c; | ||
248 | ++l; | ||
249 | n = (n+1) % 16; | ||
250 | } | ||
251 | *num=n; | ||
252 | } | ||
253 | } | ||
254 | |||
255 | /* The input and output encrypted as though 128bit ofb mode is being | ||
256 | * used. The extra state information to record how much of the | ||
257 | * 128bit block we have used is contained in *num; | ||
258 | */ | ||
259 | static void aesni_ofb128_encrypt(const unsigned char *in, unsigned char *out, | ||
260 | unsigned int len, const void *key, | ||
261 | unsigned char ivec[16], int *num) | ||
262 | { | ||
263 | unsigned int n; | ||
264 | size_t l=0; | ||
265 | |||
266 | assert(in && out && key && ivec && num); | ||
267 | |||
268 | n = *num; | ||
269 | |||
270 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
271 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
272 | while (n && len) { | ||
273 | *(out++) = *(in++) ^ ivec[n]; | ||
274 | --len; | ||
275 | n = (n+1) % 16; | ||
276 | } | ||
277 | while (len>=16) { | ||
278 | aesni_encrypt(ivec, ivec, key); | ||
279 | for (n=0; n<16; n+=sizeof(size_t)) | ||
280 | *(size_t*)(out+n) = | ||
281 | *(size_t*)(in+n) ^ *(size_t*)(ivec+n); | ||
282 | len -= 16; | ||
283 | out += 16; | ||
284 | in += 16; | ||
285 | } | ||
286 | n = 0; | ||
287 | if (len) { | ||
288 | aesni_encrypt(ivec, ivec, key); | ||
289 | while (len--) { | ||
290 | out[n] = in[n] ^ ivec[n]; | ||
291 | ++n; | ||
292 | } | ||
293 | } | ||
294 | *num = n; | ||
295 | return; | ||
296 | } while(0); | ||
297 | /* the rest would be commonly eliminated by x86* compiler */ | ||
298 | #endif | ||
299 | while (l<len) { | ||
300 | if (n==0) { | ||
301 | aesni_encrypt(ivec, ivec, key); | ||
302 | } | ||
303 | out[l] = in[l] ^ ivec[n]; | ||
304 | ++l; | ||
305 | n = (n+1) % 16; | ||
306 | } | ||
307 | |||
308 | *num=n; | ||
309 | } | ||
310 | /* ===== Engine "management" functions ===== */ | ||
311 | |||
312 | #if defined(_WIN32) | ||
313 | typedef unsigned __int64 IA32CAP; | ||
314 | #else | ||
315 | typedef unsigned long long IA32CAP; | ||
316 | #endif | ||
317 | |||
318 | /* Prepare the ENGINE structure for registration */ | ||
319 | static int | ||
320 | aesni_bind_helper(ENGINE *e) | ||
321 | { | ||
322 | int engage; | ||
323 | if (sizeof(OPENSSL_ia32cap_P) > 4) { | ||
324 | engage = ((IA32CAP)OPENSSL_ia32cap_P >> 57) & 1; | ||
325 | } else { | ||
326 | IA32CAP OPENSSL_ia32_cpuid(void); | ||
327 | engage = (OPENSSL_ia32_cpuid() >> 57) & 1; | ||
328 | } | ||
329 | |||
330 | /* Register everything or return with an error */ | ||
331 | if (!ENGINE_set_id(e, aesni_id) || | ||
332 | !ENGINE_set_name(e, engage ? aesni_name : no_aesni_name) || | ||
333 | |||
334 | !ENGINE_set_init_function(e, aesni_init) || | ||
335 | (engage && !ENGINE_set_ciphers (e, aesni_ciphers)) | ||
336 | ) | ||
337 | return 0; | ||
338 | |||
339 | /* Everything looks good */ | ||
340 | return 1; | ||
341 | } | ||
342 | |||
343 | /* Constructor */ | ||
344 | static ENGINE * | ||
345 | ENGINE_aesni(void) | ||
346 | { | ||
347 | ENGINE *eng = ENGINE_new(); | ||
348 | |||
349 | if (!eng) { | ||
350 | return NULL; | ||
351 | } | ||
352 | |||
353 | if (!aesni_bind_helper(eng)) { | ||
354 | ENGINE_free(eng); | ||
355 | return NULL; | ||
356 | } | ||
357 | |||
358 | return eng; | ||
359 | } | ||
360 | |||
361 | /* Check availability of the engine */ | ||
362 | static int | ||
363 | aesni_init(ENGINE *e) | ||
364 | { | ||
365 | return 1; | ||
366 | } | ||
367 | |||
368 | #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb) | ||
369 | #define NID_aes_128_cfb NID_aes_128_cfb128 | ||
370 | #endif | ||
371 | |||
372 | #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb) | ||
373 | #define NID_aes_128_ofb NID_aes_128_ofb128 | ||
374 | #endif | ||
375 | |||
376 | #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb) | ||
377 | #define NID_aes_192_cfb NID_aes_192_cfb128 | ||
378 | #endif | ||
379 | |||
380 | #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb) | ||
381 | #define NID_aes_192_ofb NID_aes_192_ofb128 | ||
382 | #endif | ||
383 | |||
384 | #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb) | ||
385 | #define NID_aes_256_cfb NID_aes_256_cfb128 | ||
386 | #endif | ||
387 | |||
388 | #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb) | ||
389 | #define NID_aes_256_ofb NID_aes_256_ofb128 | ||
390 | #endif | ||
391 | |||
392 | /* List of supported ciphers. */ | ||
393 | static int aesni_cipher_nids[] = { | ||
394 | NID_aes_128_ecb, | ||
395 | NID_aes_128_cbc, | ||
396 | NID_aes_128_cfb, | ||
397 | NID_aes_128_ofb, | ||
398 | |||
399 | NID_aes_192_ecb, | ||
400 | NID_aes_192_cbc, | ||
401 | NID_aes_192_cfb, | ||
402 | NID_aes_192_ofb, | ||
403 | |||
404 | NID_aes_256_ecb, | ||
405 | NID_aes_256_cbc, | ||
406 | NID_aes_256_cfb, | ||
407 | NID_aes_256_ofb, | ||
408 | }; | ||
409 | static int aesni_cipher_nids_num = | ||
410 | (sizeof(aesni_cipher_nids)/sizeof(aesni_cipher_nids[0])); | ||
411 | |||
412 | typedef struct | ||
413 | { | ||
414 | AES_KEY ks; | ||
415 | unsigned int _pad1[3]; | ||
416 | } AESNI_KEY; | ||
417 | |||
418 | static int | ||
419 | aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *user_key, | ||
420 | const unsigned char *iv, int enc) | ||
421 | { | ||
422 | int ret; | ||
423 | AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
424 | |||
425 | if ((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CFB_MODE | ||
426 | || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_OFB_MODE | ||
427 | || enc) | ||
428 | ret=aesni_set_encrypt_key(user_key, ctx->key_len * 8, key); | ||
429 | else | ||
430 | ret=aesni_set_decrypt_key(user_key, ctx->key_len * 8, key); | ||
431 | |||
432 | if(ret < 0) { | ||
433 | EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED); | ||
434 | return 0; | ||
435 | } | ||
436 | |||
437 | return 1; | ||
438 | } | ||
439 | |||
440 | static int aesni_cipher_ecb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
441 | const unsigned char *in, unsigned int inl) | ||
442 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
443 | aesni_ecb_encrypt(in, out, inl, key, ctx->encrypt); | ||
444 | return 1; | ||
445 | } | ||
446 | static int aesni_cipher_cbc(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
447 | const unsigned char *in, unsigned int inl) | ||
448 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
449 | aesni_cbc_encrypt(in, out, inl, key, | ||
450 | ctx->iv, ctx->encrypt); | ||
451 | return 1; | ||
452 | } | ||
453 | static int aesni_cipher_cfb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
454 | const unsigned char *in, unsigned int inl) | ||
455 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
456 | |||
457 | aesni_cfb128_encrypt(in, out, inl, key, ctx->iv, | ||
458 | &ctx->num, ctx->encrypt); | ||
459 | return 1; | ||
460 | } | ||
461 | static int aesni_cipher_ofb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
462 | const unsigned char *in, unsigned int inl) | ||
463 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
464 | aesni_ofb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num); | ||
465 | return 1; | ||
466 | } | ||
467 | |||
468 | #define AES_BLOCK_SIZE 16 | ||
469 | |||
470 | #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE | ||
471 | #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE | ||
472 | #define EVP_CIPHER_block_size_OFB 1 | ||
473 | #define EVP_CIPHER_block_size_CFB 1 | ||
474 | |||
475 | /* Declaring so many ciphers by hand would be a pain. | ||
476 | Instead introduce a bit of preprocessor magic :-) */ | ||
477 | #define DECLARE_AES_EVP(ksize,lmode,umode) \ | ||
478 | static const EVP_CIPHER aesni_##ksize##_##lmode = { \ | ||
479 | NID_aes_##ksize##_##lmode, \ | ||
480 | EVP_CIPHER_block_size_##umode, \ | ||
481 | ksize / 8, \ | ||
482 | AES_BLOCK_SIZE, \ | ||
483 | 0 | EVP_CIPH_##umode##_MODE, \ | ||
484 | aesni_init_key, \ | ||
485 | aesni_cipher_##lmode, \ | ||
486 | NULL, \ | ||
487 | sizeof(AESNI_KEY), \ | ||
488 | EVP_CIPHER_set_asn1_iv, \ | ||
489 | EVP_CIPHER_get_asn1_iv, \ | ||
490 | NULL, \ | ||
491 | NULL \ | ||
492 | } | ||
493 | |||
494 | DECLARE_AES_EVP(128,ecb,ECB); | ||
495 | DECLARE_AES_EVP(128,cbc,CBC); | ||
496 | DECLARE_AES_EVP(128,cfb,CFB); | ||
497 | DECLARE_AES_EVP(128,ofb,OFB); | ||
498 | |||
499 | DECLARE_AES_EVP(192,ecb,ECB); | ||
500 | DECLARE_AES_EVP(192,cbc,CBC); | ||
501 | DECLARE_AES_EVP(192,cfb,CFB); | ||
502 | DECLARE_AES_EVP(192,ofb,OFB); | ||
503 | |||
504 | DECLARE_AES_EVP(256,ecb,ECB); | ||
505 | DECLARE_AES_EVP(256,cbc,CBC); | ||
506 | DECLARE_AES_EVP(256,cfb,CFB); | ||
507 | DECLARE_AES_EVP(256,ofb,OFB); | ||
508 | |||
509 | static int | ||
510 | aesni_ciphers (ENGINE *e, const EVP_CIPHER **cipher, | ||
511 | const int **nids, int nid) | ||
512 | { | ||
513 | /* No specific cipher => return a list of supported nids ... */ | ||
514 | if (!cipher) { | ||
515 | *nids = aesni_cipher_nids; | ||
516 | return aesni_cipher_nids_num; | ||
517 | } | ||
518 | |||
519 | /* ... or the requested "cipher" otherwise */ | ||
520 | switch (nid) { | ||
521 | case NID_aes_128_ecb: | ||
522 | *cipher = &aesni_128_ecb; | ||
523 | break; | ||
524 | case NID_aes_128_cbc: | ||
525 | *cipher = &aesni_128_cbc; | ||
526 | break; | ||
527 | case NID_aes_128_cfb: | ||
528 | *cipher = &aesni_128_cfb; | ||
529 | break; | ||
530 | case NID_aes_128_ofb: | ||
531 | *cipher = &aesni_128_ofb; | ||
532 | break; | ||
533 | |||
534 | case NID_aes_192_ecb: | ||
535 | *cipher = &aesni_192_ecb; | ||
536 | break; | ||
537 | case NID_aes_192_cbc: | ||
538 | *cipher = &aesni_192_cbc; | ||
539 | break; | ||
540 | case NID_aes_192_cfb: | ||
541 | *cipher = &aesni_192_cfb; | ||
542 | break; | ||
543 | case NID_aes_192_ofb: | ||
544 | *cipher = &aesni_192_ofb; | ||
545 | break; | ||
546 | |||
547 | case NID_aes_256_ecb: | ||
548 | *cipher = &aesni_256_ecb; | ||
549 | break; | ||
550 | case NID_aes_256_cbc: | ||
551 | *cipher = &aesni_256_cbc; | ||
552 | break; | ||
553 | case NID_aes_256_cfb: | ||
554 | *cipher = &aesni_256_cfb; | ||
555 | break; | ||
556 | case NID_aes_256_ofb: | ||
557 | *cipher = &aesni_256_ofb; | ||
558 | break; | ||
559 | |||
560 | default: | ||
561 | /* Sorry, we don't support this NID */ | ||
562 | *cipher = NULL; | ||
563 | return 0; | ||
564 | } | ||
565 | return 1; | ||
566 | } | ||
567 | |||
568 | #endif /* COMPILE_HW_AESNI */ | ||
569 | #endif /* !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) && !defined(OPENSSL_NO_AES) */ | ||
570 | |||
diff --git a/src/lib/libcrypto/engine/eng_all.c b/src/lib/libcrypto/engine/eng_all.c index d29cd57dc2..69e54f461e 100644 --- a/src/lib/libcrypto/engine/eng_all.c +++ b/src/lib/libcrypto/engine/eng_all.c | |||
@@ -71,6 +71,11 @@ void ENGINE_load_builtin_engines(void) | |||
71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) | 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) |
72 | ENGINE_load_padlock(); | 72 | ENGINE_load_padlock(); |
73 | #endif | 73 | #endif |
74 | |||
75 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) | ||
76 | ENGINE_load_aesni(); | ||
77 | #endif | ||
78 | |||
74 | ENGINE_load_dynamic(); | 79 | ENGINE_load_dynamic(); |
75 | #ifndef OPENSSL_NO_STATIC_ENGINE | 80 | #ifndef OPENSSL_NO_STATIC_ENGINE |
76 | #ifndef OPENSSL_NO_HW | 81 | #ifndef OPENSSL_NO_HW |
diff --git a/src/lib/libcrypto/engine/engine.h b/src/lib/libcrypto/engine/engine.h index f503595ece..ecab9453ba 100644 --- a/src/lib/libcrypto/engine/engine.h +++ b/src/lib/libcrypto/engine/engine.h | |||
@@ -337,6 +337,7 @@ void ENGINE_load_sureware(void); | |||
337 | void ENGINE_load_ubsec(void); | 337 | void ENGINE_load_ubsec(void); |
338 | #endif | 338 | #endif |
339 | void ENGINE_load_cryptodev(void); | 339 | void ENGINE_load_cryptodev(void); |
340 | void ENGINE_load_aesni(void); | ||
340 | void ENGINE_load_padlock(void); | 341 | void ENGINE_load_padlock(void); |
341 | void ENGINE_load_builtin_engines(void); | 342 | void ENGINE_load_builtin_engines(void); |
342 | #ifndef OPENSSL_NO_CAPIENG | 343 | #ifndef OPENSSL_NO_CAPIENG |
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h index 51011f2b14..82b5862ce7 100644 --- a/src/lib/libcrypto/evp/evp.h +++ b/src/lib/libcrypto/evp/evp.h | |||
@@ -959,6 +959,7 @@ void ERR_load_EVP_strings(void); | |||
959 | /* Error codes for the EVP functions. */ | 959 | /* Error codes for the EVP functions. */ |
960 | 960 | ||
961 | /* Function codes. */ | 961 | /* Function codes. */ |
962 | #define EVP_F_AESNI_INIT_KEY 165 | ||
962 | #define EVP_F_AES_INIT_KEY 133 | 963 | #define EVP_F_AES_INIT_KEY 133 |
963 | #define EVP_F_ALG_MODULE_INIT 138 | 964 | #define EVP_F_ALG_MODULE_INIT 138 |
964 | #define EVP_F_CAMELLIA_INIT_KEY 159 | 965 | #define EVP_F_CAMELLIA_INIT_KEY 159 |
diff --git a/src/lib/libcrypto/evp/evp_err.c b/src/lib/libcrypto/evp/evp_err.c index b5b900d4fe..ec2d127cd8 100644 --- a/src/lib/libcrypto/evp/evp_err.c +++ b/src/lib/libcrypto/evp/evp_err.c | |||
@@ -70,6 +70,7 @@ | |||
70 | 70 | ||
71 | static ERR_STRING_DATA EVP_str_functs[]= | 71 | static ERR_STRING_DATA EVP_str_functs[]= |
72 | { | 72 | { |
73 | {ERR_FUNC(EVP_F_AESNI_INIT_KEY), "AESNI_INIT_KEY"}, | ||
73 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, | 74 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, |
74 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, | 75 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, |
75 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, | 76 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, |
diff --git a/src/lib/libcrypto/util/libeay.num b/src/lib/libcrypto/util/libeay.num index 74eb337227..0bad595233 100644 --- a/src/lib/libcrypto/util/libeay.num +++ b/src/lib/libcrypto/util/libeay.num | |||
@@ -3725,3 +3725,4 @@ JPAKE_STEP2_release 4110 EXIST::FUNCTION:JPAKE | |||
3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE | 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE |
3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE | 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE |
3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE | 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE |
3728 | ENGINE_load_aesni 4114 EXIST::FUNCTION:ENGINE | ||
diff --git a/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl b/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl new file mode 100644 index 0000000000..49e0f4b351 --- /dev/null +++ b/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl | |||
@@ -0,0 +1,992 @@ | |||
1 | #!/usr/bin/env perl | ||
2 | # | ||
3 | # ==================================================================== | ||
4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
5 | # project. The module is, however, dual licensed under OpenSSL and | ||
6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
8 | # ==================================================================== | ||
9 | # | ||
10 | # This module implements support for Intel AES-NI extension. In | ||
11 | # OpenSSL context it's used with Intel engine, but can also be used as | ||
12 | # drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for | ||
13 | # details]. | ||
14 | |||
15 | $PREFIX="aesni"; # if $PREFIX is set to "AES", the script | ||
16 | # generates drop-in replacement for | ||
17 | # crypto/aes/asm/aes-x86_64.pl:-) | ||
18 | |||
19 | $flavour = shift; | ||
20 | $output = shift; | ||
21 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | ||
22 | |||
23 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | ||
24 | |||
25 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
26 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | ||
27 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | ||
28 | die "can't locate x86_64-xlate.pl"; | ||
29 | |||
30 | open STDOUT,"| $^X $xlate $flavour $output"; | ||
31 | |||
32 | $movkey = $PREFIX eq "aesni" ? "movaps" : "movups"; | ||
33 | @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order | ||
34 | ("%rdi","%rsi","%rdx","%rcx"); # Unix order | ||
35 | |||
36 | $code=".text\n"; | ||
37 | |||
38 | $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! | ||
39 | # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... | ||
40 | $inp="%rdi"; | ||
41 | $out="%rsi"; | ||
42 | $len="%rdx"; | ||
43 | $key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!! | ||
44 | $ivp="%r8"; # cbc | ||
45 | |||
46 | $rnds_="%r10d"; # backup copy for $rounds | ||
47 | $key_="%r11"; # backup copy for $key | ||
48 | |||
49 | # %xmm register layout | ||
50 | $inout0="%xmm0"; $inout1="%xmm1"; | ||
51 | $inout2="%xmm2"; $inout3="%xmm3"; | ||
52 | $rndkey0="%xmm4"; $rndkey1="%xmm5"; | ||
53 | |||
54 | $iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt | ||
55 | $in1="%xmm8"; $in2="%xmm9"; | ||
56 | |||
57 | # Inline version of internal aesni_[en|de]crypt1. | ||
58 | # | ||
59 | # Why folded loop? Because aes[enc|dec] is slow enough to accommodate | ||
60 | # cycles which take care of loop variables... | ||
61 | { my $sn; | ||
62 | sub aesni_generate1 { | ||
63 | my ($p,$key,$rounds)=@_; | ||
64 | ++$sn; | ||
65 | $code.=<<___; | ||
66 | $movkey ($key),$rndkey0 | ||
67 | $movkey 16($key),$rndkey1 | ||
68 | lea 32($key),$key | ||
69 | pxor $rndkey0,$inout0 | ||
70 | .Loop_${p}1_$sn: | ||
71 | aes${p} $rndkey1,$inout0 | ||
72 | dec $rounds | ||
73 | $movkey ($key),$rndkey1 | ||
74 | lea 16($key),$key | ||
75 | jnz .Loop_${p}1_$sn # loop body is 16 bytes | ||
76 | aes${p}last $rndkey1,$inout0 | ||
77 | ___ | ||
78 | }} | ||
79 | # void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key); | ||
80 | # | ||
81 | { my ($inp,$out,$key) = @_4args; | ||
82 | |||
83 | $code.=<<___; | ||
84 | .globl ${PREFIX}_encrypt | ||
85 | .type ${PREFIX}_encrypt,\@abi-omnipotent | ||
86 | .align 16 | ||
87 | ${PREFIX}_encrypt: | ||
88 | movups ($inp),$inout0 # load input | ||
89 | mov 240($key),$rounds # pull $rounds | ||
90 | ___ | ||
91 | &aesni_generate1("enc",$key,$rounds); | ||
92 | $code.=<<___; | ||
93 | movups $inout0,($out) # output | ||
94 | ret | ||
95 | .size ${PREFIX}_encrypt,.-${PREFIX}_encrypt | ||
96 | |||
97 | .globl ${PREFIX}_decrypt | ||
98 | .type ${PREFIX}_decrypt,\@abi-omnipotent | ||
99 | .align 16 | ||
100 | ${PREFIX}_decrypt: | ||
101 | movups ($inp),$inout0 # load input | ||
102 | mov 240($key),$rounds # pull $rounds | ||
103 | ___ | ||
104 | &aesni_generate1("dec",$key,$rounds); | ||
105 | $code.=<<___; | ||
106 | movups $inout0,($out) # output | ||
107 | ret | ||
108 | .size ${PREFIX}_decrypt, .-${PREFIX}_decrypt | ||
109 | ___ | ||
110 | } | ||
111 | |||
112 | # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave | ||
113 | # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec] | ||
114 | # latency is 6, it turned out that it can be scheduled only every | ||
115 | # *second* cycle. Thus 3x interleave is the one providing optimal | ||
116 | # utilization, i.e. when subroutine's throughput is virtually same as | ||
117 | # of non-interleaved subroutine [for number of input blocks up to 3]. | ||
118 | # This is why it makes no sense to implement 2x subroutine. As soon | ||
119 | # as/if Intel improves throughput by making it possible to schedule | ||
120 | # the instructions in question *every* cycles I would have to | ||
121 | # implement 6x interleave and use it in loop... | ||
122 | sub aesni_generate3 { | ||
123 | my $dir=shift; | ||
124 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
125 | # preserved. $inout[0-2] is cipher/clear text... | ||
126 | $code.=<<___; | ||
127 | .type _aesni_${dir}rypt3,\@abi-omnipotent | ||
128 | .align 16 | ||
129 | _aesni_${dir}rypt3: | ||
130 | $movkey ($key),$rndkey0 | ||
131 | shr \$1,$rounds | ||
132 | $movkey 16($key),$rndkey1 | ||
133 | lea 32($key),$key | ||
134 | pxor $rndkey0,$inout0 | ||
135 | pxor $rndkey0,$inout1 | ||
136 | pxor $rndkey0,$inout2 | ||
137 | |||
138 | .L${dir}_loop3: | ||
139 | aes${dir} $rndkey1,$inout0 | ||
140 | $movkey ($key),$rndkey0 | ||
141 | aes${dir} $rndkey1,$inout1 | ||
142 | dec $rounds | ||
143 | aes${dir} $rndkey1,$inout2 | ||
144 | aes${dir} $rndkey0,$inout0 | ||
145 | $movkey 16($key),$rndkey1 | ||
146 | aes${dir} $rndkey0,$inout1 | ||
147 | lea 32($key),$key | ||
148 | aes${dir} $rndkey0,$inout2 | ||
149 | jnz .L${dir}_loop3 | ||
150 | |||
151 | aes${dir} $rndkey1,$inout0 | ||
152 | $movkey ($key),$rndkey0 | ||
153 | aes${dir} $rndkey1,$inout1 | ||
154 | aes${dir} $rndkey1,$inout2 | ||
155 | aes${dir}last $rndkey0,$inout0 | ||
156 | aes${dir}last $rndkey0,$inout1 | ||
157 | aes${dir}last $rndkey0,$inout2 | ||
158 | ret | ||
159 | .size _aesni_${dir}rypt3,.-_aesni_${dir}rypt3 | ||
160 | ___ | ||
161 | } | ||
162 | # 4x interleave is implemented to improve small block performance, | ||
163 | # most notably [and naturally] 4 block by ~30%. One can argue that one | ||
164 | # should have implemented 5x as well, but improvement would be <20%, | ||
165 | # so it's not worth it... | ||
166 | sub aesni_generate4 { | ||
167 | my $dir=shift; | ||
168 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
169 | # preserved. $inout[0-3] is cipher/clear text... | ||
170 | $code.=<<___; | ||
171 | .type _aesni_${dir}rypt4,\@abi-omnipotent | ||
172 | .align 16 | ||
173 | _aesni_${dir}rypt4: | ||
174 | $movkey ($key),$rndkey0 | ||
175 | shr \$1,$rounds | ||
176 | $movkey 16($key),$rndkey1 | ||
177 | lea 32($key),$key | ||
178 | pxor $rndkey0,$inout0 | ||
179 | pxor $rndkey0,$inout1 | ||
180 | pxor $rndkey0,$inout2 | ||
181 | pxor $rndkey0,$inout3 | ||
182 | |||
183 | .L${dir}_loop4: | ||
184 | aes${dir} $rndkey1,$inout0 | ||
185 | $movkey ($key),$rndkey0 | ||
186 | aes${dir} $rndkey1,$inout1 | ||
187 | dec $rounds | ||
188 | aes${dir} $rndkey1,$inout2 | ||
189 | aes${dir} $rndkey1,$inout3 | ||
190 | aes${dir} $rndkey0,$inout0 | ||
191 | $movkey 16($key),$rndkey1 | ||
192 | aes${dir} $rndkey0,$inout1 | ||
193 | lea 32($key),$key | ||
194 | aes${dir} $rndkey0,$inout2 | ||
195 | aes${dir} $rndkey0,$inout3 | ||
196 | jnz .L${dir}_loop4 | ||
197 | |||
198 | aes${dir} $rndkey1,$inout0 | ||
199 | $movkey ($key),$rndkey0 | ||
200 | aes${dir} $rndkey1,$inout1 | ||
201 | aes${dir} $rndkey1,$inout2 | ||
202 | aes${dir} $rndkey1,$inout3 | ||
203 | aes${dir}last $rndkey0,$inout0 | ||
204 | aes${dir}last $rndkey0,$inout1 | ||
205 | aes${dir}last $rndkey0,$inout2 | ||
206 | aes${dir}last $rndkey0,$inout3 | ||
207 | ret | ||
208 | .size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4 | ||
209 | ___ | ||
210 | } | ||
211 | &aesni_generate3("enc") if ($PREFIX eq "aesni"); | ||
212 | &aesni_generate3("dec"); | ||
213 | &aesni_generate4("enc") if ($PREFIX eq "aesni"); | ||
214 | &aesni_generate4("dec"); | ||
215 | |||
216 | if ($PREFIX eq "aesni") { | ||
217 | # void aesni_ecb_encrypt (const void *in, void *out, | ||
218 | # size_t length, const AES_KEY *key, | ||
219 | # int enc); | ||
220 | $code.=<<___; | ||
221 | .globl aesni_ecb_encrypt | ||
222 | .type aesni_ecb_encrypt,\@function,5 | ||
223 | .align 16 | ||
224 | aesni_ecb_encrypt: | ||
225 | cmp \$16,$len # check length | ||
226 | jb .Lecb_ret | ||
227 | |||
228 | mov 240($key),$rounds # pull $rounds | ||
229 | and \$-16,$len | ||
230 | mov $key,$key_ # backup $key | ||
231 | test %r8d,%r8d # 5th argument | ||
232 | mov $rounds,$rnds_ # backup $rounds | ||
233 | jz .Lecb_decrypt | ||
234 | #--------------------------- ECB ENCRYPT ------------------------------# | ||
235 | sub \$0x40,$len | ||
236 | jbe .Lecb_enc_tail | ||
237 | jmp .Lecb_enc_loop3 | ||
238 | .align 16 | ||
239 | .Lecb_enc_loop3: | ||
240 | movups ($inp),$inout0 | ||
241 | movups 0x10($inp),$inout1 | ||
242 | movups 0x20($inp),$inout2 | ||
243 | call _aesni_encrypt3 | ||
244 | sub \$0x30,$len | ||
245 | lea 0x30($inp),$inp | ||
246 | lea 0x30($out),$out | ||
247 | movups $inout0,-0x30($out) | ||
248 | mov $rnds_,$rounds # restore $rounds | ||
249 | movups $inout1,-0x20($out) | ||
250 | mov $key_,$key # restore $key | ||
251 | movups $inout2,-0x10($out) | ||
252 | ja .Lecb_enc_loop3 | ||
253 | |||
254 | .Lecb_enc_tail: | ||
255 | add \$0x40,$len | ||
256 | jz .Lecb_ret | ||
257 | |||
258 | cmp \$0x10,$len | ||
259 | movups ($inp),$inout0 | ||
260 | je .Lecb_enc_one | ||
261 | cmp \$0x20,$len | ||
262 | movups 0x10($inp),$inout1 | ||
263 | je .Lecb_enc_two | ||
264 | cmp \$0x30,$len | ||
265 | movups 0x20($inp),$inout2 | ||
266 | je .Lecb_enc_three | ||
267 | movups 0x30($inp),$inout3 | ||
268 | call _aesni_encrypt4 | ||
269 | movups $inout0,($out) | ||
270 | movups $inout1,0x10($out) | ||
271 | movups $inout2,0x20($out) | ||
272 | movups $inout3,0x30($out) | ||
273 | jmp .Lecb_ret | ||
274 | .align 16 | ||
275 | .Lecb_enc_one: | ||
276 | ___ | ||
277 | &aesni_generate1("enc",$key,$rounds); | ||
278 | $code.=<<___; | ||
279 | movups $inout0,($out) | ||
280 | jmp .Lecb_ret | ||
281 | .align 16 | ||
282 | .Lecb_enc_two: | ||
283 | call _aesni_encrypt3 | ||
284 | movups $inout0,($out) | ||
285 | movups $inout1,0x10($out) | ||
286 | jmp .Lecb_ret | ||
287 | .align 16 | ||
288 | .Lecb_enc_three: | ||
289 | call _aesni_encrypt3 | ||
290 | movups $inout0,($out) | ||
291 | movups $inout1,0x10($out) | ||
292 | movups $inout2,0x20($out) | ||
293 | jmp .Lecb_ret | ||
294 | #--------------------------- ECB DECRYPT ------------------------------# | ||
295 | .align 16 | ||
296 | .Lecb_decrypt: | ||
297 | sub \$0x40,$len | ||
298 | jbe .Lecb_dec_tail | ||
299 | jmp .Lecb_dec_loop3 | ||
300 | .align 16 | ||
301 | .Lecb_dec_loop3: | ||
302 | movups ($inp),$inout0 | ||
303 | movups 0x10($inp),$inout1 | ||
304 | movups 0x20($inp),$inout2 | ||
305 | call _aesni_decrypt3 | ||
306 | sub \$0x30,$len | ||
307 | lea 0x30($inp),$inp | ||
308 | lea 0x30($out),$out | ||
309 | movups $inout0,-0x30($out) | ||
310 | mov $rnds_,$rounds # restore $rounds | ||
311 | movups $inout1,-0x20($out) | ||
312 | mov $key_,$key # restore $key | ||
313 | movups $inout2,-0x10($out) | ||
314 | ja .Lecb_dec_loop3 | ||
315 | |||
316 | .Lecb_dec_tail: | ||
317 | add \$0x40,$len | ||
318 | jz .Lecb_ret | ||
319 | |||
320 | cmp \$0x10,$len | ||
321 | movups ($inp),$inout0 | ||
322 | je .Lecb_dec_one | ||
323 | cmp \$0x20,$len | ||
324 | movups 0x10($inp),$inout1 | ||
325 | je .Lecb_dec_two | ||
326 | cmp \$0x30,$len | ||
327 | movups 0x20($inp),$inout2 | ||
328 | je .Lecb_dec_three | ||
329 | movups 0x30($inp),$inout3 | ||
330 | call _aesni_decrypt4 | ||
331 | movups $inout0,($out) | ||
332 | movups $inout1,0x10($out) | ||
333 | movups $inout2,0x20($out) | ||
334 | movups $inout3,0x30($out) | ||
335 | jmp .Lecb_ret | ||
336 | .align 16 | ||
337 | .Lecb_dec_one: | ||
338 | ___ | ||
339 | &aesni_generate1("dec",$key,$rounds); | ||
340 | $code.=<<___; | ||
341 | movups $inout0,($out) | ||
342 | jmp .Lecb_ret | ||
343 | .align 16 | ||
344 | .Lecb_dec_two: | ||
345 | call _aesni_decrypt3 | ||
346 | movups $inout0,($out) | ||
347 | movups $inout1,0x10($out) | ||
348 | jmp .Lecb_ret | ||
349 | .align 16 | ||
350 | .Lecb_dec_three: | ||
351 | call _aesni_decrypt3 | ||
352 | movups $inout0,($out) | ||
353 | movups $inout1,0x10($out) | ||
354 | movups $inout2,0x20($out) | ||
355 | |||
356 | .Lecb_ret: | ||
357 | ret | ||
358 | .size aesni_ecb_encrypt,.-aesni_ecb_encrypt | ||
359 | ___ | ||
360 | } | ||
361 | |||
362 | # void $PREFIX_cbc_encrypt (const void *inp, void *out, | ||
363 | # size_t length, const AES_KEY *key, | ||
364 | # unsigned char *ivp,const int enc); | ||
365 | $reserved = $win64?0x40:-0x18; # used in decrypt | ||
366 | $code.=<<___; | ||
367 | .globl ${PREFIX}_cbc_encrypt | ||
368 | .type ${PREFIX}_cbc_encrypt,\@function,6 | ||
369 | .align 16 | ||
370 | ${PREFIX}_cbc_encrypt: | ||
371 | test $len,$len # check length | ||
372 | jz .Lcbc_ret | ||
373 | |||
374 | mov 240($key),$rnds_ # pull $rounds | ||
375 | mov $key,$key_ # backup $key | ||
376 | test %r9d,%r9d # 6th argument | ||
377 | jz .Lcbc_decrypt | ||
378 | #--------------------------- CBC ENCRYPT ------------------------------# | ||
379 | movups ($ivp),$inout0 # load iv as initial state | ||
380 | cmp \$16,$len | ||
381 | mov $rnds_,$rounds | ||
382 | jb .Lcbc_enc_tail | ||
383 | sub \$16,$len | ||
384 | jmp .Lcbc_enc_loop | ||
385 | .align 16 | ||
386 | .Lcbc_enc_loop: | ||
387 | movups ($inp),$inout1 # load input | ||
388 | lea 16($inp),$inp | ||
389 | pxor $inout1,$inout0 | ||
390 | ___ | ||
391 | &aesni_generate1("enc",$key,$rounds); | ||
392 | $code.=<<___; | ||
393 | sub \$16,$len | ||
394 | lea 16($out),$out | ||
395 | mov $rnds_,$rounds # restore $rounds | ||
396 | mov $key_,$key # restore $key | ||
397 | movups $inout0,-16($out) # store output | ||
398 | jnc .Lcbc_enc_loop | ||
399 | add \$16,$len | ||
400 | jnz .Lcbc_enc_tail | ||
401 | movups $inout0,($ivp) | ||
402 | jmp .Lcbc_ret | ||
403 | |||
404 | .Lcbc_enc_tail: | ||
405 | mov $len,%rcx # zaps $key | ||
406 | xchg $inp,$out # $inp is %rsi and $out is %rdi now | ||
407 | .long 0x9066A4F3 # rep movsb | ||
408 | mov \$16,%ecx # zero tail | ||
409 | sub $len,%rcx | ||
410 | xor %eax,%eax | ||
411 | .long 0x9066AAF3 # rep stosb | ||
412 | lea -16(%rdi),%rdi # rewind $out by 1 block | ||
413 | mov $rnds_,$rounds # restore $rounds | ||
414 | mov %rdi,%rsi # $inp and $out are the same | ||
415 | mov $key_,$key # restore $key | ||
416 | xor $len,$len # len=16 | ||
417 | jmp .Lcbc_enc_loop # one more spin | ||
418 | #--------------------------- CBC DECRYPT ------------------------------# | ||
419 | .align 16 | ||
420 | .Lcbc_decrypt: | ||
421 | ___ | ||
422 | $code.=<<___ if ($win64); | ||
423 | lea -0x58(%rsp),%rsp | ||
424 | movaps %xmm6,(%rsp) | ||
425 | movaps %xmm7,0x10(%rsp) | ||
426 | movaps %xmm8,0x20(%rsp) | ||
427 | movaps %xmm9,0x30(%rsp) | ||
428 | .Lcbc_decrypt_body: | ||
429 | ___ | ||
430 | $code.=<<___; | ||
431 | movups ($ivp),$iv | ||
432 | sub \$0x40,$len | ||
433 | mov $rnds_,$rounds | ||
434 | jbe .Lcbc_dec_tail | ||
435 | jmp .Lcbc_dec_loop3 | ||
436 | .align 16 | ||
437 | .Lcbc_dec_loop3: | ||
438 | movups ($inp),$inout0 | ||
439 | movups 0x10($inp),$inout1 | ||
440 | movups 0x20($inp),$inout2 | ||
441 | movaps $inout0,$in0 | ||
442 | movaps $inout1,$in1 | ||
443 | movaps $inout2,$in2 | ||
444 | call _aesni_decrypt3 | ||
445 | sub \$0x30,$len | ||
446 | lea 0x30($inp),$inp | ||
447 | lea 0x30($out),$out | ||
448 | pxor $iv,$inout0 | ||
449 | pxor $in0,$inout1 | ||
450 | movaps $in2,$iv | ||
451 | pxor $in1,$inout2 | ||
452 | movups $inout0,-0x30($out) | ||
453 | mov $rnds_,$rounds # restore $rounds | ||
454 | movups $inout1,-0x20($out) | ||
455 | mov $key_,$key # restore $key | ||
456 | movups $inout2,-0x10($out) | ||
457 | ja .Lcbc_dec_loop3 | ||
458 | |||
459 | .Lcbc_dec_tail: | ||
460 | add \$0x40,$len | ||
461 | movups $iv,($ivp) | ||
462 | jz .Lcbc_dec_ret | ||
463 | |||
464 | movups ($inp),$inout0 | ||
465 | cmp \$0x10,$len | ||
466 | movaps $inout0,$in0 | ||
467 | jbe .Lcbc_dec_one | ||
468 | movups 0x10($inp),$inout1 | ||
469 | cmp \$0x20,$len | ||
470 | movaps $inout1,$in1 | ||
471 | jbe .Lcbc_dec_two | ||
472 | movups 0x20($inp),$inout2 | ||
473 | cmp \$0x30,$len | ||
474 | movaps $inout2,$in2 | ||
475 | jbe .Lcbc_dec_three | ||
476 | movups 0x30($inp),$inout3 | ||
477 | call _aesni_decrypt4 | ||
478 | pxor $iv,$inout0 | ||
479 | movups 0x30($inp),$iv | ||
480 | pxor $in0,$inout1 | ||
481 | movups $inout0,($out) | ||
482 | pxor $in1,$inout2 | ||
483 | movups $inout1,0x10($out) | ||
484 | pxor $in2,$inout3 | ||
485 | movups $inout2,0x20($out) | ||
486 | movaps $inout3,$inout0 | ||
487 | lea 0x30($out),$out | ||
488 | jmp .Lcbc_dec_tail_collected | ||
489 | .align 16 | ||
490 | .Lcbc_dec_one: | ||
491 | ___ | ||
492 | &aesni_generate1("dec",$key,$rounds); | ||
493 | $code.=<<___; | ||
494 | pxor $iv,$inout0 | ||
495 | movaps $in0,$iv | ||
496 | jmp .Lcbc_dec_tail_collected | ||
497 | .align 16 | ||
498 | .Lcbc_dec_two: | ||
499 | call _aesni_decrypt3 | ||
500 | pxor $iv,$inout0 | ||
501 | pxor $in0,$inout1 | ||
502 | movups $inout0,($out) | ||
503 | movaps $in1,$iv | ||
504 | movaps $inout1,$inout0 | ||
505 | lea 0x10($out),$out | ||
506 | jmp .Lcbc_dec_tail_collected | ||
507 | .align 16 | ||
508 | .Lcbc_dec_three: | ||
509 | call _aesni_decrypt3 | ||
510 | pxor $iv,$inout0 | ||
511 | pxor $in0,$inout1 | ||
512 | movups $inout0,($out) | ||
513 | pxor $in1,$inout2 | ||
514 | movups $inout1,0x10($out) | ||
515 | movaps $in2,$iv | ||
516 | movaps $inout2,$inout0 | ||
517 | lea 0x20($out),$out | ||
518 | jmp .Lcbc_dec_tail_collected | ||
519 | .align 16 | ||
520 | .Lcbc_dec_tail_collected: | ||
521 | and \$15,$len | ||
522 | movups $iv,($ivp) | ||
523 | jnz .Lcbc_dec_tail_partial | ||
524 | movups $inout0,($out) | ||
525 | jmp .Lcbc_dec_ret | ||
526 | .Lcbc_dec_tail_partial: | ||
527 | movaps $inout0,$reserved(%rsp) | ||
528 | mov $out,%rdi | ||
529 | mov $len,%rcx | ||
530 | lea $reserved(%rsp),%rsi | ||
531 | .long 0x9066A4F3 # rep movsb | ||
532 | |||
533 | .Lcbc_dec_ret: | ||
534 | ___ | ||
535 | $code.=<<___ if ($win64); | ||
536 | movaps (%rsp),%xmm6 | ||
537 | movaps 0x10(%rsp),%xmm7 | ||
538 | movaps 0x20(%rsp),%xmm8 | ||
539 | movaps 0x30(%rsp),%xmm9 | ||
540 | lea 0x58(%rsp),%rsp | ||
541 | ___ | ||
542 | $code.=<<___; | ||
543 | .Lcbc_ret: | ||
544 | ret | ||
545 | .size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt | ||
546 | ___ | ||
547 | |||
548 | # int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey, | ||
549 | # int bits, AES_KEY *key) | ||
550 | { my ($inp,$bits,$key) = @_4args; | ||
551 | $bits =~ s/%r/%e/; | ||
552 | |||
553 | $code.=<<___; | ||
554 | .globl ${PREFIX}_set_decrypt_key | ||
555 | .type ${PREFIX}_set_decrypt_key,\@abi-omnipotent | ||
556 | .align 16 | ||
557 | ${PREFIX}_set_decrypt_key: | ||
558 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
559 | call _aesni_set_encrypt_key | ||
560 | shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key | ||
561 | test %eax,%eax | ||
562 | jnz .Ldec_key_ret | ||
563 | lea 16($key,$bits),$inp # points at the end of key schedule | ||
564 | |||
565 | $movkey ($key),%xmm0 # just swap | ||
566 | $movkey ($inp),%xmm1 | ||
567 | $movkey %xmm0,($inp) | ||
568 | $movkey %xmm1,($key) | ||
569 | lea 16($key),$key | ||
570 | lea -16($inp),$inp | ||
571 | |||
572 | .Ldec_key_inverse: | ||
573 | $movkey ($key),%xmm0 # swap and inverse | ||
574 | $movkey ($inp),%xmm1 | ||
575 | aesimc %xmm0,%xmm0 | ||
576 | aesimc %xmm1,%xmm1 | ||
577 | lea 16($key),$key | ||
578 | lea -16($inp),$inp | ||
579 | cmp $key,$inp | ||
580 | $movkey %xmm0,16($inp) | ||
581 | $movkey %xmm1,-16($key) | ||
582 | ja .Ldec_key_inverse | ||
583 | |||
584 | $movkey ($key),%xmm0 # inverse middle | ||
585 | aesimc %xmm0,%xmm0 | ||
586 | $movkey %xmm0,($inp) | ||
587 | .Ldec_key_ret: | ||
588 | add \$8,%rsp | ||
589 | ret | ||
590 | .LSEH_end_set_decrypt_key: | ||
591 | .size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key | ||
592 | ___ | ||
593 | |||
594 | # This is based on submission by | ||
595 | # | ||
596 | # Huang Ying <ying.huang@intel.com> | ||
597 | # Vinodh Gopal <vinodh.gopal@intel.com> | ||
598 | # Kahraman Akdemir | ||
599 | # | ||
600 | # Agressively optimized in respect to aeskeygenassist's critical path | ||
601 | # and is contained in %xmm0-5 to meet Win64 ABI requirement. | ||
602 | # | ||
603 | $code.=<<___; | ||
604 | .globl ${PREFIX}_set_encrypt_key | ||
605 | .type ${PREFIX}_set_encrypt_key,\@abi-omnipotent | ||
606 | .align 16 | ||
607 | ${PREFIX}_set_encrypt_key: | ||
608 | _aesni_set_encrypt_key: | ||
609 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
610 | test $inp,$inp | ||
611 | mov \$-1,%rax | ||
612 | jz .Lenc_key_ret | ||
613 | test $key,$key | ||
614 | jz .Lenc_key_ret | ||
615 | |||
616 | movups ($inp),%xmm0 # pull first 128 bits of *userKey | ||
617 | pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0 | ||
618 | lea 16($key),%rax | ||
619 | cmp \$256,$bits | ||
620 | je .L14rounds | ||
621 | cmp \$192,$bits | ||
622 | je .L12rounds | ||
623 | cmp \$128,$bits | ||
624 | jne .Lbad_keybits | ||
625 | |||
626 | .L10rounds: | ||
627 | mov \$9,$bits # 10 rounds for 128-bit key | ||
628 | $movkey %xmm0,($key) # round 0 | ||
629 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 | ||
630 | call .Lkey_expansion_128_cold | ||
631 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 2 | ||
632 | call .Lkey_expansion_128 | ||
633 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 3 | ||
634 | call .Lkey_expansion_128 | ||
635 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 4 | ||
636 | call .Lkey_expansion_128 | ||
637 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 5 | ||
638 | call .Lkey_expansion_128 | ||
639 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 6 | ||
640 | call .Lkey_expansion_128 | ||
641 | aeskeygenassist \$0x40,%xmm0,%xmm1 # round 7 | ||
642 | call .Lkey_expansion_128 | ||
643 | aeskeygenassist \$0x80,%xmm0,%xmm1 # round 8 | ||
644 | call .Lkey_expansion_128 | ||
645 | aeskeygenassist \$0x1b,%xmm0,%xmm1 # round 9 | ||
646 | call .Lkey_expansion_128 | ||
647 | aeskeygenassist \$0x36,%xmm0,%xmm1 # round 10 | ||
648 | call .Lkey_expansion_128 | ||
649 | $movkey %xmm0,(%rax) | ||
650 | mov $bits,80(%rax) # 240(%rdx) | ||
651 | xor %eax,%eax | ||
652 | jmp .Lenc_key_ret | ||
653 | |||
654 | .align 16 | ||
655 | .L12rounds: | ||
656 | movq 16($inp),%xmm2 # remaining 1/3 of *userKey | ||
657 | mov \$11,$bits # 12 rounds for 192 | ||
658 | $movkey %xmm0,($key) # round 0 | ||
659 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 | ||
660 | call .Lkey_expansion_192a_cold | ||
661 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 2,3 | ||
662 | call .Lkey_expansion_192b | ||
663 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 4,5 | ||
664 | call .Lkey_expansion_192a | ||
665 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 5,6 | ||
666 | call .Lkey_expansion_192b | ||
667 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 7,8 | ||
668 | call .Lkey_expansion_192a | ||
669 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 8,9 | ||
670 | call .Lkey_expansion_192b | ||
671 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 10,11 | ||
672 | call .Lkey_expansion_192a | ||
673 | aeskeygenassist \$0x80,%xmm2,%xmm1 # round 11,12 | ||
674 | call .Lkey_expansion_192b | ||
675 | $movkey %xmm0,(%rax) | ||
676 | mov $bits,48(%rax) # 240(%rdx) | ||
677 | xor %rax, %rax | ||
678 | jmp .Lenc_key_ret | ||
679 | |||
680 | .align 16 | ||
681 | .L14rounds: | ||
682 | movups 16($inp),%xmm2 # remaning half of *userKey | ||
683 | mov \$13,$bits # 14 rounds for 256 | ||
684 | lea 16(%rax),%rax | ||
685 | $movkey %xmm0,($key) # round 0 | ||
686 | $movkey %xmm2,16($key) # round 1 | ||
687 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2 | ||
688 | call .Lkey_expansion_256a_cold | ||
689 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 3 | ||
690 | call .Lkey_expansion_256b | ||
691 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 4 | ||
692 | call .Lkey_expansion_256a | ||
693 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 5 | ||
694 | call .Lkey_expansion_256b | ||
695 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 6 | ||
696 | call .Lkey_expansion_256a | ||
697 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 7 | ||
698 | call .Lkey_expansion_256b | ||
699 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 8 | ||
700 | call .Lkey_expansion_256a | ||
701 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 9 | ||
702 | call .Lkey_expansion_256b | ||
703 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 10 | ||
704 | call .Lkey_expansion_256a | ||
705 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 11 | ||
706 | call .Lkey_expansion_256b | ||
707 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 12 | ||
708 | call .Lkey_expansion_256a | ||
709 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 13 | ||
710 | call .Lkey_expansion_256b | ||
711 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 14 | ||
712 | call .Lkey_expansion_256a | ||
713 | $movkey %xmm0,(%rax) | ||
714 | mov $bits,16(%rax) # 240(%rdx) | ||
715 | xor %rax,%rax | ||
716 | jmp .Lenc_key_ret | ||
717 | |||
718 | .align 16 | ||
719 | .Lbad_keybits: | ||
720 | mov \$-2,%rax | ||
721 | .Lenc_key_ret: | ||
722 | add \$8,%rsp | ||
723 | ret | ||
724 | .LSEH_end_set_encrypt_key: | ||
725 | |||
726 | .align 16 | ||
727 | .Lkey_expansion_128: | ||
728 | $movkey %xmm0,(%rax) | ||
729 | lea 16(%rax),%rax | ||
730 | .Lkey_expansion_128_cold: | ||
731 | shufps \$0b00010000,%xmm0,%xmm4 | ||
732 | pxor %xmm4, %xmm0 | ||
733 | shufps \$0b10001100,%xmm0,%xmm4 | ||
734 | pxor %xmm4, %xmm0 | ||
735 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
736 | pxor %xmm1,%xmm0 | ||
737 | ret | ||
738 | |||
739 | .align 16 | ||
740 | .Lkey_expansion_192a: | ||
741 | $movkey %xmm0,(%rax) | ||
742 | lea 16(%rax),%rax | ||
743 | .Lkey_expansion_192a_cold: | ||
744 | movaps %xmm2, %xmm5 | ||
745 | .Lkey_expansion_192b_warm: | ||
746 | shufps \$0b00010000,%xmm0,%xmm4 | ||
747 | movaps %xmm2,%xmm3 | ||
748 | pxor %xmm4,%xmm0 | ||
749 | shufps \$0b10001100,%xmm0,%xmm4 | ||
750 | pslldq \$4,%xmm3 | ||
751 | pxor %xmm4,%xmm0 | ||
752 | pshufd \$0b01010101,%xmm1,%xmm1 # critical path | ||
753 | pxor %xmm3,%xmm2 | ||
754 | pxor %xmm1,%xmm0 | ||
755 | pshufd \$0b11111111,%xmm0,%xmm3 | ||
756 | pxor %xmm3,%xmm2 | ||
757 | ret | ||
758 | |||
759 | .align 16 | ||
760 | .Lkey_expansion_192b: | ||
761 | movaps %xmm0,%xmm3 | ||
762 | shufps \$0b01000100,%xmm0,%xmm5 | ||
763 | $movkey %xmm5,(%rax) | ||
764 | shufps \$0b01001110,%xmm2,%xmm3 | ||
765 | $movkey %xmm3,16(%rax) | ||
766 | lea 32(%rax),%rax | ||
767 | jmp .Lkey_expansion_192b_warm | ||
768 | |||
769 | .align 16 | ||
770 | .Lkey_expansion_256a: | ||
771 | $movkey %xmm2,(%rax) | ||
772 | lea 16(%rax),%rax | ||
773 | .Lkey_expansion_256a_cold: | ||
774 | shufps \$0b00010000,%xmm0,%xmm4 | ||
775 | pxor %xmm4,%xmm0 | ||
776 | shufps \$0b10001100,%xmm0,%xmm4 | ||
777 | pxor %xmm4,%xmm0 | ||
778 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
779 | pxor %xmm1,%xmm0 | ||
780 | ret | ||
781 | |||
782 | .align 16 | ||
783 | .Lkey_expansion_256b: | ||
784 | $movkey %xmm0,(%rax) | ||
785 | lea 16(%rax),%rax | ||
786 | |||
787 | shufps \$0b00010000,%xmm2,%xmm4 | ||
788 | pxor %xmm4,%xmm2 | ||
789 | shufps \$0b10001100,%xmm2,%xmm4 | ||
790 | pxor %xmm4,%xmm2 | ||
791 | pshufd \$0b10101010,%xmm1,%xmm1 # critical path | ||
792 | pxor %xmm1,%xmm2 | ||
793 | ret | ||
794 | .size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key | ||
795 | ___ | ||
796 | } | ||
797 | |||
798 | $code.=<<___; | ||
799 | .asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>" | ||
800 | .align 64 | ||
801 | ___ | ||
802 | |||
803 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | ||
804 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | ||
805 | if ($win64) { | ||
806 | $rec="%rcx"; | ||
807 | $frame="%rdx"; | ||
808 | $context="%r8"; | ||
809 | $disp="%r9"; | ||
810 | |||
811 | $code.=<<___; | ||
812 | .extern __imp_RtlVirtualUnwind | ||
813 | .type cbc_se_handler,\@abi-omnipotent | ||
814 | .align 16 | ||
815 | cbc_se_handler: | ||
816 | push %rsi | ||
817 | push %rdi | ||
818 | push %rbx | ||
819 | push %rbp | ||
820 | push %r12 | ||
821 | push %r13 | ||
822 | push %r14 | ||
823 | push %r15 | ||
824 | pushfq | ||
825 | sub \$64,%rsp | ||
826 | |||
827 | mov 152($context),%rax # pull context->Rsp | ||
828 | mov 248($context),%rbx # pull context->Rip | ||
829 | |||
830 | lea .Lcbc_decrypt(%rip),%r10 | ||
831 | cmp %r10,%rbx # context->Rip<"prologue" label | ||
832 | jb .Lin_prologue | ||
833 | |||
834 | lea .Lcbc_decrypt_body(%rip),%r10 | ||
835 | cmp %r10,%rbx # context->Rip<cbc_decrypt_body | ||
836 | jb .Lrestore_rax | ||
837 | |||
838 | lea .Lcbc_ret(%rip),%r10 | ||
839 | cmp %r10,%rbx # context->Rip>="epilogue" label | ||
840 | jae .Lin_prologue | ||
841 | |||
842 | lea 0(%rax),%rsi # top of stack | ||
843 | lea 512($context),%rdi # &context.Xmm6 | ||
844 | mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax) | ||
845 | .long 0xa548f3fc # cld; rep movsq | ||
846 | lea 0x58(%rax),%rax # adjust stack pointer | ||
847 | jmp .Lin_prologue | ||
848 | |||
849 | .Lrestore_rax: | ||
850 | mov 120($context),%rax | ||
851 | .Lin_prologue: | ||
852 | mov 8(%rax),%rdi | ||
853 | mov 16(%rax),%rsi | ||
854 | mov %rax,152($context) # restore context->Rsp | ||
855 | mov %rsi,168($context) # restore context->Rsi | ||
856 | mov %rdi,176($context) # restore context->Rdi | ||
857 | |||
858 | jmp .Lcommon_seh_exit | ||
859 | .size cbc_se_handler,.-cbc_se_handler | ||
860 | |||
861 | .type ecb_se_handler,\@abi-omnipotent | ||
862 | .align 16 | ||
863 | ecb_se_handler: | ||
864 | push %rsi | ||
865 | push %rdi | ||
866 | push %rbx | ||
867 | push %rbp | ||
868 | push %r12 | ||
869 | push %r13 | ||
870 | push %r14 | ||
871 | push %r15 | ||
872 | pushfq | ||
873 | sub \$64,%rsp | ||
874 | |||
875 | mov 152($context),%rax # pull context->Rsp | ||
876 | mov 8(%rax),%rdi | ||
877 | mov 16(%rax),%rsi | ||
878 | mov %rsi,168($context) # restore context->Rsi | ||
879 | mov %rdi,176($context) # restore context->Rdi | ||
880 | |||
881 | .Lcommon_seh_exit: | ||
882 | |||
883 | mov 40($disp),%rdi # disp->ContextRecord | ||
884 | mov $context,%rsi # context | ||
885 | mov \$154,%ecx # sizeof(CONTEXT) | ||
886 | .long 0xa548f3fc # cld; rep movsq | ||
887 | |||
888 | mov $disp,%rsi | ||
889 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | ||
890 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | ||
891 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | ||
892 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | ||
893 | mov 40(%rsi),%r10 # disp->ContextRecord | ||
894 | lea 56(%rsi),%r11 # &disp->HandlerData | ||
895 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | ||
896 | mov %r10,32(%rsp) # arg5 | ||
897 | mov %r11,40(%rsp) # arg6 | ||
898 | mov %r12,48(%rsp) # arg7 | ||
899 | mov %rcx,56(%rsp) # arg8, (NULL) | ||
900 | call *__imp_RtlVirtualUnwind(%rip) | ||
901 | |||
902 | mov \$1,%eax # ExceptionContinueSearch | ||
903 | add \$64,%rsp | ||
904 | popfq | ||
905 | pop %r15 | ||
906 | pop %r14 | ||
907 | pop %r13 | ||
908 | pop %r12 | ||
909 | pop %rbp | ||
910 | pop %rbx | ||
911 | pop %rdi | ||
912 | pop %rsi | ||
913 | ret | ||
914 | .size cbc_se_handler,.-cbc_se_handler | ||
915 | |||
916 | .section .pdata | ||
917 | .align 4 | ||
918 | .rva .LSEH_begin_${PREFIX}_ecb_encrypt | ||
919 | .rva .LSEH_end_${PREFIX}_ecb_encrypt | ||
920 | .rva .LSEH_info_ecb | ||
921 | |||
922 | .rva .LSEH_begin_${PREFIX}_cbc_encrypt | ||
923 | .rva .LSEH_end_${PREFIX}_cbc_encrypt | ||
924 | .rva .LSEH_info_cbc | ||
925 | |||
926 | .rva ${PREFIX}_set_decrypt_key | ||
927 | .rva .LSEH_end_set_decrypt_key | ||
928 | .rva .LSEH_info_key | ||
929 | |||
930 | .rva ${PREFIX}_set_encrypt_key | ||
931 | .rva .LSEH_end_set_encrypt_key | ||
932 | .rva .LSEH_info_key | ||
933 | .section .xdata | ||
934 | .align 8 | ||
935 | .LSEH_info_ecb: | ||
936 | .byte 9,0,0,0 | ||
937 | .rva ecb_se_handler | ||
938 | .LSEH_info_cbc: | ||
939 | .byte 9,0,0,0 | ||
940 | .rva cbc_se_handler | ||
941 | .LSEH_info_key: | ||
942 | .byte 0x01,0x04,0x01,0x00 | ||
943 | .byte 0x04,0x02,0x00,0x00 | ||
944 | ___ | ||
945 | } | ||
946 | |||
947 | sub rex { | ||
948 | local *opcode=shift; | ||
949 | my ($dst,$src)=@_; | ||
950 | |||
951 | if ($dst>=8 || $src>=8) { | ||
952 | $rex=0x40; | ||
953 | $rex|=0x04 if($dst>=8); | ||
954 | $rex|=0x01 if($src>=8); | ||
955 | push @opcode,$rex; | ||
956 | } | ||
957 | } | ||
958 | |||
959 | sub aesni { | ||
960 | my $line=shift; | ||
961 | my @opcode=(0x66); | ||
962 | |||
963 | if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
964 | rex(\@opcode,$4,$3); | ||
965 | push @opcode,0x0f,0x3a,0xdf; | ||
966 | push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M | ||
967 | my $c=$2; | ||
968 | push @opcode,$c=~/^0/?oct($c):$c; | ||
969 | return ".byte\t".join(',',@opcode); | ||
970 | } | ||
971 | elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
972 | my %opcodelet = ( | ||
973 | "aesimc" => 0xdb, | ||
974 | "aesenc" => 0xdc, "aesenclast" => 0xdd, | ||
975 | "aesdec" => 0xde, "aesdeclast" => 0xdf | ||
976 | ); | ||
977 | return undef if (!defined($opcodelet{$1})); | ||
978 | rex(\@opcode,$3,$2); | ||
979 | push @opcode,0x0f,0x38,$opcodelet{$1}; | ||
980 | push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M | ||
981 | return ".byte\t".join(',',@opcode); | ||
982 | } | ||
983 | return $line; | ||
984 | } | ||
985 | |||
986 | $code =~ s/\`([^\`]*)\`/eval($1)/gem; | ||
987 | $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; | ||
988 | |||
989 | print $code; | ||
990 | |||
991 | close STDOUT; | ||
992 | |||
diff --git a/src/lib/libssl/src/crypto/crypto-lib.com b/src/lib/libssl/src/crypto/crypto-lib.com index e72af90822..8f1c1d5a80 100644 --- a/src/lib/libssl/src/crypto/crypto-lib.com +++ b/src/lib/libssl/src/crypto/crypto-lib.com | |||
@@ -209,7 +209,8 @@ $ LIB_ENGINE = "eng_err,eng_lib,eng_list,eng_init,eng_ctrl,"+ - | |||
209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - | 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - |
210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - | 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - |
211 | "tb_cipher,tb_digest,"+ - | 211 | "tb_cipher,tb_digest,"+ - |
212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock" | 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock"+ - |
213 | "eng_aesni" | ||
213 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - | 214 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - |
214 | "aes_ctr,aes_ige,aes_wrap" | 215 | "aes_ctr,aes_ige,aes_wrap" |
215 | $ LIB_BUFFER = "buffer,buf_str,buf_err" | 216 | $ LIB_BUFFER = "buffer,buf_str,buf_err" |
diff --git a/src/lib/libssl/src/crypto/engine/Makefile b/src/lib/libssl/src/crypto/engine/Makefile index 0cc3722089..0736153075 100644 --- a/src/lib/libssl/src/crypto/engine/Makefile +++ b/src/lib/libssl/src/crypto/engine/Makefile | |||
@@ -21,12 +21,14 @@ LIBSRC= eng_err.c eng_lib.c eng_list.c eng_init.c eng_ctrl.c \ | |||
21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ | 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ |
22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ | 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ |
23 | tb_cipher.c tb_digest.c \ | 23 | tb_cipher.c tb_digest.c \ |
24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c | 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c \ |
25 | eng_aesni.c | ||
25 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ | 26 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ |
26 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ | 27 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ |
27 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ | 28 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ |
28 | tb_cipher.o tb_digest.o \ | 29 | tb_cipher.o tb_digest.o \ |
29 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o | 30 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o \ |
31 | eng_aesni.o | ||
30 | 32 | ||
31 | SRC= $(LIBSRC) | 33 | SRC= $(LIBSRC) |
32 | 34 | ||
@@ -82,6 +84,21 @@ clean: | |||
82 | 84 | ||
83 | # DO NOT DELETE THIS LINE -- make depend depends on it. | 85 | # DO NOT DELETE THIS LINE -- make depend depends on it. |
84 | 86 | ||
87 | eng_aesni.o: ../../e_os.h ../../include/openssl/aes.h | ||
88 | eng_aesni.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h | ||
89 | eng_aesni.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h | ||
90 | eng_aesni.o: ../../include/openssl/dso.h ../../include/openssl/e_os2.h | ||
91 | eng_aesni.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h | ||
92 | eng_aesni.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h | ||
93 | eng_aesni.o: ../../include/openssl/err.h ../../include/openssl/evp.h | ||
94 | eng_aesni.o: ../../include/openssl/lhash.h ../../include/openssl/modes.h | ||
95 | eng_aesni.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h | ||
96 | eng_aesni.o: ../../include/openssl/opensslconf.h | ||
97 | eng_aesni.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h | ||
98 | eng_aesni.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h | ||
99 | eng_aesni.o: ../../include/openssl/sha.h ../../include/openssl/stack.h | ||
100 | eng_aesni.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h | ||
101 | eng_aesni.o: ../../include/openssl/x509_vfy.h ../cryptlib.h eng_aesni.c | ||
85 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h | 102 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h |
86 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h | 103 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h |
87 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h | 104 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h |
diff --git a/src/lib/libssl/src/crypto/engine/eng_aesni.c b/src/lib/libssl/src/crypto/engine/eng_aesni.c new file mode 100644 index 0000000000..eeded1eefa --- /dev/null +++ b/src/lib/libssl/src/crypto/engine/eng_aesni.c | |||
@@ -0,0 +1,570 @@ | |||
1 | /* | ||
2 | * Support for Intel AES-NI intruction set | ||
3 | * Author: Huang Ying <ying.huang@intel.com> | ||
4 | * | ||
5 | * Intel AES-NI is a new set of Single Instruction Multiple Data | ||
6 | * (SIMD) instructions that are going to be introduced in the next | ||
7 | * generation of Intel processor, as of 2009. These instructions | ||
8 | * enable fast and secure data encryption and decryption, using the | ||
9 | * Advanced Encryption Standard (AES), defined by FIPS Publication | ||
10 | * number 197. The architecture introduces six instructions that | ||
11 | * offer full hardware support for AES. Four of them support high | ||
12 | * performance data encryption and decryption, and the other two | ||
13 | * instructions support the AES key expansion procedure. | ||
14 | * | ||
15 | * The white paper can be downloaded from: | ||
16 | * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf | ||
17 | * | ||
18 | * This file is based on engines/e_padlock.c | ||
19 | */ | ||
20 | |||
21 | /* ==================================================================== | ||
22 | * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved. | ||
23 | * | ||
24 | * Redistribution and use in source and binary forms, with or without | ||
25 | * modification, are permitted provided that the following conditions | ||
26 | * are met: | ||
27 | * | ||
28 | * 1. Redistributions of source code must retain the above copyright | ||
29 | * notice, this list of conditions and the following disclaimer. | ||
30 | * | ||
31 | * 2. Redistributions in binary form must reproduce the above copyright | ||
32 | * notice, this list of conditions and the following disclaimer in | ||
33 | * the documentation and/or other materials provided with the | ||
34 | * distribution. | ||
35 | * | ||
36 | * 3. All advertising materials mentioning features or use of this | ||
37 | * software must display the following acknowledgment: | ||
38 | * "This product includes software developed by the OpenSSL Project | ||
39 | * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" | ||
40 | * | ||
41 | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to | ||
42 | * endorse or promote products derived from this software without | ||
43 | * prior written permission. For written permission, please contact | ||
44 | * licensing@OpenSSL.org. | ||
45 | * | ||
46 | * 5. Products derived from this software may not be called "OpenSSL" | ||
47 | * nor may "OpenSSL" appear in their names without prior written | ||
48 | * permission of the OpenSSL Project. | ||
49 | * | ||
50 | * 6. Redistributions of any form whatsoever must retain the following | ||
51 | * acknowledgment: | ||
52 | * "This product includes software developed by the OpenSSL Project | ||
53 | * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" | ||
54 | * | ||
55 | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY | ||
56 | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
57 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
58 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR | ||
59 | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
60 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | ||
61 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | ||
62 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||
63 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | ||
64 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
65 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED | ||
66 | * OF THE POSSIBILITY OF SUCH DAMAGE. | ||
67 | * ==================================================================== | ||
68 | * | ||
69 | * This product includes cryptographic software written by Eric Young | ||
70 | * (eay@cryptsoft.com). This product includes software written by Tim | ||
71 | * Hudson (tjh@cryptsoft.com). | ||
72 | * | ||
73 | */ | ||
74 | |||
75 | |||
76 | #include <openssl/opensslconf.h> | ||
77 | |||
78 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AES_NI) && !defined(OPENSSL_NO_AES) | ||
79 | |||
80 | #include <stdio.h> | ||
81 | #include <assert.h> | ||
82 | #include "cryptlib.h" | ||
83 | #include <openssl/dso.h> | ||
84 | #include <openssl/engine.h> | ||
85 | #include <openssl/evp.h> | ||
86 | #include <openssl/aes.h> | ||
87 | #include <openssl/err.h> | ||
88 | |||
89 | /* AES-NI is available *ONLY* on some x86 CPUs. Not only that it | ||
90 | doesn't exist elsewhere, but it even can't be compiled on other | ||
91 | platforms! */ | ||
92 | #undef COMPILE_HW_AESNI | ||
93 | #if (defined(__x86_64) || defined(__x86_64__) || \ | ||
94 | defined(_M_AMD64) || defined(_M_X64) || \ | ||
95 | defined(OPENSSL_IA32_SSE2)) && !defined(OPENSSL_NO_ASM) && !defined(__i386__) | ||
96 | #define COMPILE_HW_AESNI | ||
97 | #endif | ||
98 | static ENGINE *ENGINE_aesni (void); | ||
99 | |||
100 | void ENGINE_load_aesni (void) | ||
101 | { | ||
102 | /* On non-x86 CPUs it just returns. */ | ||
103 | #ifdef COMPILE_HW_AESNI | ||
104 | ENGINE *toadd = ENGINE_aesni(); | ||
105 | if (!toadd) return; | ||
106 | ENGINE_add (toadd); | ||
107 | ENGINE_register_complete (toadd); | ||
108 | ENGINE_free (toadd); | ||
109 | ERR_clear_error (); | ||
110 | #endif | ||
111 | } | ||
112 | |||
113 | #ifdef COMPILE_HW_AESNI | ||
114 | int aesni_set_encrypt_key(const unsigned char *userKey, int bits, | ||
115 | AES_KEY *key); | ||
116 | int aesni_set_decrypt_key(const unsigned char *userKey, int bits, | ||
117 | AES_KEY *key); | ||
118 | |||
119 | void aesni_encrypt(const unsigned char *in, unsigned char *out, | ||
120 | const AES_KEY *key); | ||
121 | void aesni_decrypt(const unsigned char *in, unsigned char *out, | ||
122 | const AES_KEY *key); | ||
123 | |||
124 | void aesni_ecb_encrypt(const unsigned char *in, | ||
125 | unsigned char *out, | ||
126 | size_t length, | ||
127 | const AES_KEY *key, | ||
128 | int enc); | ||
129 | void aesni_cbc_encrypt(const unsigned char *in, | ||
130 | unsigned char *out, | ||
131 | size_t length, | ||
132 | const AES_KEY *key, | ||
133 | unsigned char *ivec, int enc); | ||
134 | |||
135 | /* Function for ENGINE detection and control */ | ||
136 | static int aesni_init(ENGINE *e); | ||
137 | |||
138 | /* Cipher Stuff */ | ||
139 | static int aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | ||
140 | const int **nids, int nid); | ||
141 | |||
142 | #define AESNI_MIN_ALIGN 16 | ||
143 | #define AESNI_ALIGN(x) \ | ||
144 | ((void *)(((unsigned long)(x)+AESNI_MIN_ALIGN-1)&~(AESNI_MIN_ALIGN-1))) | ||
145 | |||
146 | /* Engine names */ | ||
147 | static const char aesni_id[] = "aesni", | ||
148 | aesni_name[] = "Intel AES-NI engine", | ||
149 | no_aesni_name[] = "Intel AES-NI engine (no-aesni)"; | ||
150 | |||
151 | |||
152 | /* The input and output encrypted as though 128bit cfb mode is being | ||
153 | * used. The extra state information to record how much of the | ||
154 | * 128bit block we have used is contained in *num; | ||
155 | */ | ||
156 | static void aesni_cfb128_encrypt(const unsigned char *in, unsigned char *out, | ||
157 | unsigned int len, const void *key, | ||
158 | unsigned char ivec[16], int *num, | ||
159 | int enc) | ||
160 | { | ||
161 | unsigned int n; | ||
162 | size_t l = 0; | ||
163 | |||
164 | assert(in && out && key && ivec && num); | ||
165 | |||
166 | n = *num; | ||
167 | |||
168 | if (enc) { | ||
169 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
170 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
171 | while (n && len) { | ||
172 | *(out++) = ivec[n] ^= *(in++); | ||
173 | --len; | ||
174 | n = (n+1) % 16; | ||
175 | } | ||
176 | while (len>=16) { | ||
177 | aesni_encrypt(ivec, ivec, key); | ||
178 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
179 | *(size_t*)(out+n) = | ||
180 | *(size_t*)(ivec+n) ^= *(size_t*)(in+n); | ||
181 | } | ||
182 | len -= 16; | ||
183 | out += 16; | ||
184 | in += 16; | ||
185 | } | ||
186 | n = 0; | ||
187 | if (len) { | ||
188 | aesni_encrypt(ivec, ivec, key); | ||
189 | while (len--) { | ||
190 | out[n] = ivec[n] ^= in[n]; | ||
191 | ++n; | ||
192 | } | ||
193 | } | ||
194 | *num = n; | ||
195 | return; | ||
196 | } while (0); | ||
197 | /* the rest would be commonly eliminated by x86* compiler */ | ||
198 | #endif | ||
199 | while (l<len) { | ||
200 | if (n == 0) { | ||
201 | aesni_encrypt(ivec, ivec, key); | ||
202 | } | ||
203 | out[l] = ivec[n] ^= in[l]; | ||
204 | ++l; | ||
205 | n = (n+1) % 16; | ||
206 | } | ||
207 | *num = n; | ||
208 | } else { | ||
209 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
210 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
211 | while (n && len) { | ||
212 | unsigned char c; | ||
213 | *(out++) = ivec[n] ^ (c = *(in++)); ivec[n] = c; | ||
214 | --len; | ||
215 | n = (n+1) % 16; | ||
216 | } | ||
217 | while (len>=16) { | ||
218 | aesni_encrypt(ivec, ivec, key); | ||
219 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
220 | size_t t = *(size_t*)(in+n); | ||
221 | *(size_t*)(out+n) = *(size_t*)(ivec+n) ^ t; | ||
222 | *(size_t*)(ivec+n) = t; | ||
223 | } | ||
224 | len -= 16; | ||
225 | out += 16; | ||
226 | in += 16; | ||
227 | } | ||
228 | n = 0; | ||
229 | if (len) { | ||
230 | aesni_encrypt(ivec, ivec, key); | ||
231 | while (len--) { | ||
232 | unsigned char c; | ||
233 | out[n] = ivec[n] ^ (c = in[n]); ivec[n] = c; | ||
234 | ++n; | ||
235 | } | ||
236 | } | ||
237 | *num = n; | ||
238 | return; | ||
239 | } while (0); | ||
240 | /* the rest would be commonly eliminated by x86* compiler */ | ||
241 | #endif | ||
242 | while (l<len) { | ||
243 | unsigned char c; | ||
244 | if (n == 0) { | ||
245 | aesni_encrypt(ivec, ivec, key); | ||
246 | } | ||
247 | out[l] = ivec[n] ^ (c = in[l]); ivec[n] = c; | ||
248 | ++l; | ||
249 | n = (n+1) % 16; | ||
250 | } | ||
251 | *num=n; | ||
252 | } | ||
253 | } | ||
254 | |||
255 | /* The input and output encrypted as though 128bit ofb mode is being | ||
256 | * used. The extra state information to record how much of the | ||
257 | * 128bit block we have used is contained in *num; | ||
258 | */ | ||
259 | static void aesni_ofb128_encrypt(const unsigned char *in, unsigned char *out, | ||
260 | unsigned int len, const void *key, | ||
261 | unsigned char ivec[16], int *num) | ||
262 | { | ||
263 | unsigned int n; | ||
264 | size_t l=0; | ||
265 | |||
266 | assert(in && out && key && ivec && num); | ||
267 | |||
268 | n = *num; | ||
269 | |||
270 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
271 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
272 | while (n && len) { | ||
273 | *(out++) = *(in++) ^ ivec[n]; | ||
274 | --len; | ||
275 | n = (n+1) % 16; | ||
276 | } | ||
277 | while (len>=16) { | ||
278 | aesni_encrypt(ivec, ivec, key); | ||
279 | for (n=0; n<16; n+=sizeof(size_t)) | ||
280 | *(size_t*)(out+n) = | ||
281 | *(size_t*)(in+n) ^ *(size_t*)(ivec+n); | ||
282 | len -= 16; | ||
283 | out += 16; | ||
284 | in += 16; | ||
285 | } | ||
286 | n = 0; | ||
287 | if (len) { | ||
288 | aesni_encrypt(ivec, ivec, key); | ||
289 | while (len--) { | ||
290 | out[n] = in[n] ^ ivec[n]; | ||
291 | ++n; | ||
292 | } | ||
293 | } | ||
294 | *num = n; | ||
295 | return; | ||
296 | } while(0); | ||
297 | /* the rest would be commonly eliminated by x86* compiler */ | ||
298 | #endif | ||
299 | while (l<len) { | ||
300 | if (n==0) { | ||
301 | aesni_encrypt(ivec, ivec, key); | ||
302 | } | ||
303 | out[l] = in[l] ^ ivec[n]; | ||
304 | ++l; | ||
305 | n = (n+1) % 16; | ||
306 | } | ||
307 | |||
308 | *num=n; | ||
309 | } | ||
310 | /* ===== Engine "management" functions ===== */ | ||
311 | |||
312 | #if defined(_WIN32) | ||
313 | typedef unsigned __int64 IA32CAP; | ||
314 | #else | ||
315 | typedef unsigned long long IA32CAP; | ||
316 | #endif | ||
317 | |||
318 | /* Prepare the ENGINE structure for registration */ | ||
319 | static int | ||
320 | aesni_bind_helper(ENGINE *e) | ||
321 | { | ||
322 | int engage; | ||
323 | if (sizeof(OPENSSL_ia32cap_P) > 4) { | ||
324 | engage = ((IA32CAP)OPENSSL_ia32cap_P >> 57) & 1; | ||
325 | } else { | ||
326 | IA32CAP OPENSSL_ia32_cpuid(void); | ||
327 | engage = (OPENSSL_ia32_cpuid() >> 57) & 1; | ||
328 | } | ||
329 | |||
330 | /* Register everything or return with an error */ | ||
331 | if (!ENGINE_set_id(e, aesni_id) || | ||
332 | !ENGINE_set_name(e, engage ? aesni_name : no_aesni_name) || | ||
333 | |||
334 | !ENGINE_set_init_function(e, aesni_init) || | ||
335 | (engage && !ENGINE_set_ciphers (e, aesni_ciphers)) | ||
336 | ) | ||
337 | return 0; | ||
338 | |||
339 | /* Everything looks good */ | ||
340 | return 1; | ||
341 | } | ||
342 | |||
343 | /* Constructor */ | ||
344 | static ENGINE * | ||
345 | ENGINE_aesni(void) | ||
346 | { | ||
347 | ENGINE *eng = ENGINE_new(); | ||
348 | |||
349 | if (!eng) { | ||
350 | return NULL; | ||
351 | } | ||
352 | |||
353 | if (!aesni_bind_helper(eng)) { | ||
354 | ENGINE_free(eng); | ||
355 | return NULL; | ||
356 | } | ||
357 | |||
358 | return eng; | ||
359 | } | ||
360 | |||
361 | /* Check availability of the engine */ | ||
362 | static int | ||
363 | aesni_init(ENGINE *e) | ||
364 | { | ||
365 | return 1; | ||
366 | } | ||
367 | |||
368 | #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb) | ||
369 | #define NID_aes_128_cfb NID_aes_128_cfb128 | ||
370 | #endif | ||
371 | |||
372 | #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb) | ||
373 | #define NID_aes_128_ofb NID_aes_128_ofb128 | ||
374 | #endif | ||
375 | |||
376 | #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb) | ||
377 | #define NID_aes_192_cfb NID_aes_192_cfb128 | ||
378 | #endif | ||
379 | |||
380 | #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb) | ||
381 | #define NID_aes_192_ofb NID_aes_192_ofb128 | ||
382 | #endif | ||
383 | |||
384 | #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb) | ||
385 | #define NID_aes_256_cfb NID_aes_256_cfb128 | ||
386 | #endif | ||
387 | |||
388 | #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb) | ||
389 | #define NID_aes_256_ofb NID_aes_256_ofb128 | ||
390 | #endif | ||
391 | |||
392 | /* List of supported ciphers. */ | ||
393 | static int aesni_cipher_nids[] = { | ||
394 | NID_aes_128_ecb, | ||
395 | NID_aes_128_cbc, | ||
396 | NID_aes_128_cfb, | ||
397 | NID_aes_128_ofb, | ||
398 | |||
399 | NID_aes_192_ecb, | ||
400 | NID_aes_192_cbc, | ||
401 | NID_aes_192_cfb, | ||
402 | NID_aes_192_ofb, | ||
403 | |||
404 | NID_aes_256_ecb, | ||
405 | NID_aes_256_cbc, | ||
406 | NID_aes_256_cfb, | ||
407 | NID_aes_256_ofb, | ||
408 | }; | ||
409 | static int aesni_cipher_nids_num = | ||
410 | (sizeof(aesni_cipher_nids)/sizeof(aesni_cipher_nids[0])); | ||
411 | |||
412 | typedef struct | ||
413 | { | ||
414 | AES_KEY ks; | ||
415 | unsigned int _pad1[3]; | ||
416 | } AESNI_KEY; | ||
417 | |||
418 | static int | ||
419 | aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *user_key, | ||
420 | const unsigned char *iv, int enc) | ||
421 | { | ||
422 | int ret; | ||
423 | AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
424 | |||
425 | if ((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CFB_MODE | ||
426 | || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_OFB_MODE | ||
427 | || enc) | ||
428 | ret=aesni_set_encrypt_key(user_key, ctx->key_len * 8, key); | ||
429 | else | ||
430 | ret=aesni_set_decrypt_key(user_key, ctx->key_len * 8, key); | ||
431 | |||
432 | if(ret < 0) { | ||
433 | EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED); | ||
434 | return 0; | ||
435 | } | ||
436 | |||
437 | return 1; | ||
438 | } | ||
439 | |||
440 | static int aesni_cipher_ecb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
441 | const unsigned char *in, unsigned int inl) | ||
442 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
443 | aesni_ecb_encrypt(in, out, inl, key, ctx->encrypt); | ||
444 | return 1; | ||
445 | } | ||
446 | static int aesni_cipher_cbc(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
447 | const unsigned char *in, unsigned int inl) | ||
448 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
449 | aesni_cbc_encrypt(in, out, inl, key, | ||
450 | ctx->iv, ctx->encrypt); | ||
451 | return 1; | ||
452 | } | ||
453 | static int aesni_cipher_cfb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
454 | const unsigned char *in, unsigned int inl) | ||
455 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
456 | |||
457 | aesni_cfb128_encrypt(in, out, inl, key, ctx->iv, | ||
458 | &ctx->num, ctx->encrypt); | ||
459 | return 1; | ||
460 | } | ||
461 | static int aesni_cipher_ofb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
462 | const unsigned char *in, unsigned int inl) | ||
463 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
464 | aesni_ofb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num); | ||
465 | return 1; | ||
466 | } | ||
467 | |||
468 | #define AES_BLOCK_SIZE 16 | ||
469 | |||
470 | #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE | ||
471 | #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE | ||
472 | #define EVP_CIPHER_block_size_OFB 1 | ||
473 | #define EVP_CIPHER_block_size_CFB 1 | ||
474 | |||
475 | /* Declaring so many ciphers by hand would be a pain. | ||
476 | Instead introduce a bit of preprocessor magic :-) */ | ||
477 | #define DECLARE_AES_EVP(ksize,lmode,umode) \ | ||
478 | static const EVP_CIPHER aesni_##ksize##_##lmode = { \ | ||
479 | NID_aes_##ksize##_##lmode, \ | ||
480 | EVP_CIPHER_block_size_##umode, \ | ||
481 | ksize / 8, \ | ||
482 | AES_BLOCK_SIZE, \ | ||
483 | 0 | EVP_CIPH_##umode##_MODE, \ | ||
484 | aesni_init_key, \ | ||
485 | aesni_cipher_##lmode, \ | ||
486 | NULL, \ | ||
487 | sizeof(AESNI_KEY), \ | ||
488 | EVP_CIPHER_set_asn1_iv, \ | ||
489 | EVP_CIPHER_get_asn1_iv, \ | ||
490 | NULL, \ | ||
491 | NULL \ | ||
492 | } | ||
493 | |||
494 | DECLARE_AES_EVP(128,ecb,ECB); | ||
495 | DECLARE_AES_EVP(128,cbc,CBC); | ||
496 | DECLARE_AES_EVP(128,cfb,CFB); | ||
497 | DECLARE_AES_EVP(128,ofb,OFB); | ||
498 | |||
499 | DECLARE_AES_EVP(192,ecb,ECB); | ||
500 | DECLARE_AES_EVP(192,cbc,CBC); | ||
501 | DECLARE_AES_EVP(192,cfb,CFB); | ||
502 | DECLARE_AES_EVP(192,ofb,OFB); | ||
503 | |||
504 | DECLARE_AES_EVP(256,ecb,ECB); | ||
505 | DECLARE_AES_EVP(256,cbc,CBC); | ||
506 | DECLARE_AES_EVP(256,cfb,CFB); | ||
507 | DECLARE_AES_EVP(256,ofb,OFB); | ||
508 | |||
509 | static int | ||
510 | aesni_ciphers (ENGINE *e, const EVP_CIPHER **cipher, | ||
511 | const int **nids, int nid) | ||
512 | { | ||
513 | /* No specific cipher => return a list of supported nids ... */ | ||
514 | if (!cipher) { | ||
515 | *nids = aesni_cipher_nids; | ||
516 | return aesni_cipher_nids_num; | ||
517 | } | ||
518 | |||
519 | /* ... or the requested "cipher" otherwise */ | ||
520 | switch (nid) { | ||
521 | case NID_aes_128_ecb: | ||
522 | *cipher = &aesni_128_ecb; | ||
523 | break; | ||
524 | case NID_aes_128_cbc: | ||
525 | *cipher = &aesni_128_cbc; | ||
526 | break; | ||
527 | case NID_aes_128_cfb: | ||
528 | *cipher = &aesni_128_cfb; | ||
529 | break; | ||
530 | case NID_aes_128_ofb: | ||
531 | *cipher = &aesni_128_ofb; | ||
532 | break; | ||
533 | |||
534 | case NID_aes_192_ecb: | ||
535 | *cipher = &aesni_192_ecb; | ||
536 | break; | ||
537 | case NID_aes_192_cbc: | ||
538 | *cipher = &aesni_192_cbc; | ||
539 | break; | ||
540 | case NID_aes_192_cfb: | ||
541 | *cipher = &aesni_192_cfb; | ||
542 | break; | ||
543 | case NID_aes_192_ofb: | ||
544 | *cipher = &aesni_192_ofb; | ||
545 | break; | ||
546 | |||
547 | case NID_aes_256_ecb: | ||
548 | *cipher = &aesni_256_ecb; | ||
549 | break; | ||
550 | case NID_aes_256_cbc: | ||
551 | *cipher = &aesni_256_cbc; | ||
552 | break; | ||
553 | case NID_aes_256_cfb: | ||
554 | *cipher = &aesni_256_cfb; | ||
555 | break; | ||
556 | case NID_aes_256_ofb: | ||
557 | *cipher = &aesni_256_ofb; | ||
558 | break; | ||
559 | |||
560 | default: | ||
561 | /* Sorry, we don't support this NID */ | ||
562 | *cipher = NULL; | ||
563 | return 0; | ||
564 | } | ||
565 | return 1; | ||
566 | } | ||
567 | |||
568 | #endif /* COMPILE_HW_AESNI */ | ||
569 | #endif /* !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) && !defined(OPENSSL_NO_AES) */ | ||
570 | |||
diff --git a/src/lib/libssl/src/crypto/engine/eng_all.c b/src/lib/libssl/src/crypto/engine/eng_all.c index d29cd57dc2..69e54f461e 100644 --- a/src/lib/libssl/src/crypto/engine/eng_all.c +++ b/src/lib/libssl/src/crypto/engine/eng_all.c | |||
@@ -71,6 +71,11 @@ void ENGINE_load_builtin_engines(void) | |||
71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) | 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) |
72 | ENGINE_load_padlock(); | 72 | ENGINE_load_padlock(); |
73 | #endif | 73 | #endif |
74 | |||
75 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) | ||
76 | ENGINE_load_aesni(); | ||
77 | #endif | ||
78 | |||
74 | ENGINE_load_dynamic(); | 79 | ENGINE_load_dynamic(); |
75 | #ifndef OPENSSL_NO_STATIC_ENGINE | 80 | #ifndef OPENSSL_NO_STATIC_ENGINE |
76 | #ifndef OPENSSL_NO_HW | 81 | #ifndef OPENSSL_NO_HW |
diff --git a/src/lib/libssl/src/crypto/engine/engine.h b/src/lib/libssl/src/crypto/engine/engine.h index f503595ece..ecab9453ba 100644 --- a/src/lib/libssl/src/crypto/engine/engine.h +++ b/src/lib/libssl/src/crypto/engine/engine.h | |||
@@ -337,6 +337,7 @@ void ENGINE_load_sureware(void); | |||
337 | void ENGINE_load_ubsec(void); | 337 | void ENGINE_load_ubsec(void); |
338 | #endif | 338 | #endif |
339 | void ENGINE_load_cryptodev(void); | 339 | void ENGINE_load_cryptodev(void); |
340 | void ENGINE_load_aesni(void); | ||
340 | void ENGINE_load_padlock(void); | 341 | void ENGINE_load_padlock(void); |
341 | void ENGINE_load_builtin_engines(void); | 342 | void ENGINE_load_builtin_engines(void); |
342 | #ifndef OPENSSL_NO_CAPIENG | 343 | #ifndef OPENSSL_NO_CAPIENG |
diff --git a/src/lib/libssl/src/crypto/evp/evp.h b/src/lib/libssl/src/crypto/evp/evp.h index 51011f2b14..82b5862ce7 100644 --- a/src/lib/libssl/src/crypto/evp/evp.h +++ b/src/lib/libssl/src/crypto/evp/evp.h | |||
@@ -959,6 +959,7 @@ void ERR_load_EVP_strings(void); | |||
959 | /* Error codes for the EVP functions. */ | 959 | /* Error codes for the EVP functions. */ |
960 | 960 | ||
961 | /* Function codes. */ | 961 | /* Function codes. */ |
962 | #define EVP_F_AESNI_INIT_KEY 165 | ||
962 | #define EVP_F_AES_INIT_KEY 133 | 963 | #define EVP_F_AES_INIT_KEY 133 |
963 | #define EVP_F_ALG_MODULE_INIT 138 | 964 | #define EVP_F_ALG_MODULE_INIT 138 |
964 | #define EVP_F_CAMELLIA_INIT_KEY 159 | 965 | #define EVP_F_CAMELLIA_INIT_KEY 159 |
diff --git a/src/lib/libssl/src/crypto/evp/evp_err.c b/src/lib/libssl/src/crypto/evp/evp_err.c index b5b900d4fe..ec2d127cd8 100644 --- a/src/lib/libssl/src/crypto/evp/evp_err.c +++ b/src/lib/libssl/src/crypto/evp/evp_err.c | |||
@@ -70,6 +70,7 @@ | |||
70 | 70 | ||
71 | static ERR_STRING_DATA EVP_str_functs[]= | 71 | static ERR_STRING_DATA EVP_str_functs[]= |
72 | { | 72 | { |
73 | {ERR_FUNC(EVP_F_AESNI_INIT_KEY), "AESNI_INIT_KEY"}, | ||
73 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, | 74 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, |
74 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, | 75 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, |
75 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, | 76 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, |
diff --git a/src/lib/libssl/src/test/test_aesni b/src/lib/libssl/src/test/test_aesni new file mode 100644 index 0000000000..e8fb63ee2b --- /dev/null +++ b/src/lib/libssl/src/test/test_aesni | |||
@@ -0,0 +1,69 @@ | |||
1 | #!/bin/sh | ||
2 | |||
3 | PROG=$1 | ||
4 | |||
5 | if [ -x $PROG ]; then | ||
6 | if expr "x`$PROG version`" : "xOpenSSL" > /dev/null; then | ||
7 | : | ||
8 | else | ||
9 | echo "$PROG is not OpenSSL executable" | ||
10 | exit 1 | ||
11 | fi | ||
12 | else | ||
13 | echo "$PROG is not executable" | ||
14 | exit 1; | ||
15 | fi | ||
16 | |||
17 | if $PROG engine aesni | grep -v no-aesni; then | ||
18 | |||
19 | HASH=`cat $PROG | $PROG dgst -hex` | ||
20 | |||
21 | AES_ALGS=" aes-128-ecb aes-192-ecb aes-256-ecb \ | ||
22 | aes-128-cbc aes-192-cbc aes-256-cbc \ | ||
23 | aes-128-cfb aes-192-cfb aes-256-cfb \ | ||
24 | aes-128-ofb aes-192-ofb aes-256-ofb" | ||
25 | BUFSIZE="16 32 48 64 80 96 128 144 999" | ||
26 | |||
27 | nerr=0 | ||
28 | |||
29 | for alg in $AES_ALGS; do | ||
30 | echo $alg | ||
31 | for bufsize in $BUFSIZE; do | ||
32 | TEST=`( cat $PROG | \ | ||
33 | $PROG enc -e -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
34 | $PROG enc -d -k "$HASH" -$alg | \ | ||
35 | $PROG dgst -hex ) 2>/dev/null` | ||
36 | if [ "$TEST" != "$HASH" ]; then | ||
37 | echo "-$alg/$bufsize encrypt test failed" | ||
38 | nerr=`expr $nerr + 1` | ||
39 | fi | ||
40 | done | ||
41 | for bufsize in $BUFSIZE; do | ||
42 | TEST=`( cat $PROG | \ | ||
43 | $PROG enc -e -k "$HASH" -$alg | \ | ||
44 | $PROG enc -d -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
45 | $PROG dgst -hex ) 2>/dev/null` | ||
46 | if [ "$TEST" != "$HASH" ]; then | ||
47 | echo "-$alg/$bufsize decrypt test failed" | ||
48 | nerr=`expr $nerr + 1` | ||
49 | fi | ||
50 | done | ||
51 | TEST=`( cat $PROG | \ | ||
52 | $PROG enc -e -k "$HASH" -$alg -engine aesni | \ | ||
53 | $PROG enc -d -k "$HASH" -$alg -engine aesni | \ | ||
54 | $PROG dgst -hex ) 2>/dev/null` | ||
55 | if [ "$TEST" != "$HASH" ]; then | ||
56 | echo "-$alg en/decrypt test failed" | ||
57 | nerr=`expr $nerr + 1` | ||
58 | fi | ||
59 | done | ||
60 | |||
61 | if [ $nerr -gt 0 ]; then | ||
62 | echo "AESNI engine test failed." | ||
63 | exit 1; | ||
64 | fi | ||
65 | else | ||
66 | echo "AESNI engine is not available" | ||
67 | fi | ||
68 | |||
69 | exit 0 | ||
diff --git a/src/lib/libssl/src/util/libeay.num b/src/lib/libssl/src/util/libeay.num index 74eb337227..0bad595233 100644 --- a/src/lib/libssl/src/util/libeay.num +++ b/src/lib/libssl/src/util/libeay.num | |||
@@ -3725,3 +3725,4 @@ JPAKE_STEP2_release 4110 EXIST::FUNCTION:JPAKE | |||
3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE | 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE |
3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE | 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE |
3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE | 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE |
3728 | ENGINE_load_aesni 4114 EXIST::FUNCTION:ENGINE | ||
diff --git a/src/lib/libssl/test/test_aesni b/src/lib/libssl/test/test_aesni new file mode 100644 index 0000000000..e8fb63ee2b --- /dev/null +++ b/src/lib/libssl/test/test_aesni | |||
@@ -0,0 +1,69 @@ | |||
1 | #!/bin/sh | ||
2 | |||
3 | PROG=$1 | ||
4 | |||
5 | if [ -x $PROG ]; then | ||
6 | if expr "x`$PROG version`" : "xOpenSSL" > /dev/null; then | ||
7 | : | ||
8 | else | ||
9 | echo "$PROG is not OpenSSL executable" | ||
10 | exit 1 | ||
11 | fi | ||
12 | else | ||
13 | echo "$PROG is not executable" | ||
14 | exit 1; | ||
15 | fi | ||
16 | |||
17 | if $PROG engine aesni | grep -v no-aesni; then | ||
18 | |||
19 | HASH=`cat $PROG | $PROG dgst -hex` | ||
20 | |||
21 | AES_ALGS=" aes-128-ecb aes-192-ecb aes-256-ecb \ | ||
22 | aes-128-cbc aes-192-cbc aes-256-cbc \ | ||
23 | aes-128-cfb aes-192-cfb aes-256-cfb \ | ||
24 | aes-128-ofb aes-192-ofb aes-256-ofb" | ||
25 | BUFSIZE="16 32 48 64 80 96 128 144 999" | ||
26 | |||
27 | nerr=0 | ||
28 | |||
29 | for alg in $AES_ALGS; do | ||
30 | echo $alg | ||
31 | for bufsize in $BUFSIZE; do | ||
32 | TEST=`( cat $PROG | \ | ||
33 | $PROG enc -e -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
34 | $PROG enc -d -k "$HASH" -$alg | \ | ||
35 | $PROG dgst -hex ) 2>/dev/null` | ||
36 | if [ "$TEST" != "$HASH" ]; then | ||
37 | echo "-$alg/$bufsize encrypt test failed" | ||
38 | nerr=`expr $nerr + 1` | ||
39 | fi | ||
40 | done | ||
41 | for bufsize in $BUFSIZE; do | ||
42 | TEST=`( cat $PROG | \ | ||
43 | $PROG enc -e -k "$HASH" -$alg | \ | ||
44 | $PROG enc -d -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
45 | $PROG dgst -hex ) 2>/dev/null` | ||
46 | if [ "$TEST" != "$HASH" ]; then | ||
47 | echo "-$alg/$bufsize decrypt test failed" | ||
48 | nerr=`expr $nerr + 1` | ||
49 | fi | ||
50 | done | ||
51 | TEST=`( cat $PROG | \ | ||
52 | $PROG enc -e -k "$HASH" -$alg -engine aesni | \ | ||
53 | $PROG enc -d -k "$HASH" -$alg -engine aesni | \ | ||
54 | $PROG dgst -hex ) 2>/dev/null` | ||
55 | if [ "$TEST" != "$HASH" ]; then | ||
56 | echo "-$alg en/decrypt test failed" | ||
57 | nerr=`expr $nerr + 1` | ||
58 | fi | ||
59 | done | ||
60 | |||
61 | if [ $nerr -gt 0 ]; then | ||
62 | echo "AESNI engine test failed." | ||
63 | exit 1; | ||
64 | fi | ||
65 | else | ||
66 | echo "AESNI engine is not available" | ||
67 | fi | ||
68 | |||
69 | exit 0 | ||