diff options
| author | thib <> | 2010-07-01 17:44:20 +0000 |
|---|---|---|
| committer | thib <> | 2010-07-01 17:44:20 +0000 |
| commit | aaf95101ecdb076e7121125b5b59b79338f68e63 (patch) | |
| tree | e732da07cd98066d0bfc1903e3ee73114f5b295c /src | |
| parent | 1b4deace3d997af54c492a67c6f9468ec87b9b5d (diff) | |
| download | openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.tar.gz openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.tar.bz2 openbsd-aaf95101ecdb076e7121125b5b59b79338f68e63.zip | |
AES-NI engine support for OpenSSL.
This is code mostly picked up from upstream OpenSSL, or to be more exact
a diff from David Woodhouse <dwmw2 at infradead dot org>.
Remember to make includes before doing a build!
no objections from djm@
OK deraadt@, reyk@ (AES is about 4.25x faster on his x201 now)
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/libcrypto/aes/asm/aesni-x86_64.pl | 992 | ||||
| -rw-r--r-- | src/lib/libcrypto/crypto-lib.com | 3 | ||||
| -rw-r--r-- | src/lib/libcrypto/engine/Makefile | 21 | ||||
| -rw-r--r-- | src/lib/libcrypto/engine/eng_aesni.c | 570 | ||||
| -rw-r--r-- | src/lib/libcrypto/engine/eng_all.c | 5 | ||||
| -rw-r--r-- | src/lib/libcrypto/engine/engine.h | 1 | ||||
| -rw-r--r-- | src/lib/libcrypto/evp/evp.h | 1 | ||||
| -rw-r--r-- | src/lib/libcrypto/evp/evp_err.c | 1 | ||||
| -rw-r--r-- | src/lib/libcrypto/util/libeay.num | 1 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl | 992 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/crypto-lib.com | 3 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/engine/Makefile | 21 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/engine/eng_aesni.c | 570 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/engine/eng_all.c | 5 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/engine/engine.h | 1 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/evp/evp.h | 1 | ||||
| -rw-r--r-- | src/lib/libssl/src/crypto/evp/evp_err.c | 1 | ||||
| -rw-r--r-- | src/lib/libssl/src/test/test_aesni | 69 | ||||
| -rw-r--r-- | src/lib/libssl/src/util/libeay.num | 1 | ||||
| -rw-r--r-- | src/lib/libssl/test/test_aesni | 69 |
20 files changed, 3322 insertions, 6 deletions
diff --git a/src/lib/libcrypto/aes/asm/aesni-x86_64.pl b/src/lib/libcrypto/aes/asm/aesni-x86_64.pl new file mode 100644 index 0000000000..49e0f4b351 --- /dev/null +++ b/src/lib/libcrypto/aes/asm/aesni-x86_64.pl | |||
| @@ -0,0 +1,992 @@ | |||
| 1 | #!/usr/bin/env perl | ||
| 2 | # | ||
| 3 | # ==================================================================== | ||
| 4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
| 5 | # project. The module is, however, dual licensed under OpenSSL and | ||
| 6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
| 7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
| 8 | # ==================================================================== | ||
| 9 | # | ||
| 10 | # This module implements support for Intel AES-NI extension. In | ||
| 11 | # OpenSSL context it's used with Intel engine, but can also be used as | ||
| 12 | # drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for | ||
| 13 | # details]. | ||
| 14 | |||
| 15 | $PREFIX="aesni"; # if $PREFIX is set to "AES", the script | ||
| 16 | # generates drop-in replacement for | ||
| 17 | # crypto/aes/asm/aes-x86_64.pl:-) | ||
| 18 | |||
| 19 | $flavour = shift; | ||
| 20 | $output = shift; | ||
| 21 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | ||
| 22 | |||
| 23 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | ||
| 24 | |||
| 25 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
| 26 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | ||
| 27 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | ||
| 28 | die "can't locate x86_64-xlate.pl"; | ||
| 29 | |||
| 30 | open STDOUT,"| $^X $xlate $flavour $output"; | ||
| 31 | |||
| 32 | $movkey = $PREFIX eq "aesni" ? "movaps" : "movups"; | ||
| 33 | @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order | ||
| 34 | ("%rdi","%rsi","%rdx","%rcx"); # Unix order | ||
| 35 | |||
| 36 | $code=".text\n"; | ||
| 37 | |||
| 38 | $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! | ||
| 39 | # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... | ||
| 40 | $inp="%rdi"; | ||
| 41 | $out="%rsi"; | ||
| 42 | $len="%rdx"; | ||
| 43 | $key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!! | ||
| 44 | $ivp="%r8"; # cbc | ||
| 45 | |||
| 46 | $rnds_="%r10d"; # backup copy for $rounds | ||
| 47 | $key_="%r11"; # backup copy for $key | ||
| 48 | |||
| 49 | # %xmm register layout | ||
| 50 | $inout0="%xmm0"; $inout1="%xmm1"; | ||
| 51 | $inout2="%xmm2"; $inout3="%xmm3"; | ||
| 52 | $rndkey0="%xmm4"; $rndkey1="%xmm5"; | ||
| 53 | |||
| 54 | $iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt | ||
| 55 | $in1="%xmm8"; $in2="%xmm9"; | ||
| 56 | |||
| 57 | # Inline version of internal aesni_[en|de]crypt1. | ||
| 58 | # | ||
| 59 | # Why folded loop? Because aes[enc|dec] is slow enough to accommodate | ||
| 60 | # cycles which take care of loop variables... | ||
| 61 | { my $sn; | ||
| 62 | sub aesni_generate1 { | ||
| 63 | my ($p,$key,$rounds)=@_; | ||
| 64 | ++$sn; | ||
| 65 | $code.=<<___; | ||
| 66 | $movkey ($key),$rndkey0 | ||
| 67 | $movkey 16($key),$rndkey1 | ||
| 68 | lea 32($key),$key | ||
| 69 | pxor $rndkey0,$inout0 | ||
| 70 | .Loop_${p}1_$sn: | ||
| 71 | aes${p} $rndkey1,$inout0 | ||
| 72 | dec $rounds | ||
| 73 | $movkey ($key),$rndkey1 | ||
| 74 | lea 16($key),$key | ||
| 75 | jnz .Loop_${p}1_$sn # loop body is 16 bytes | ||
| 76 | aes${p}last $rndkey1,$inout0 | ||
| 77 | ___ | ||
| 78 | }} | ||
| 79 | # void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key); | ||
| 80 | # | ||
| 81 | { my ($inp,$out,$key) = @_4args; | ||
| 82 | |||
| 83 | $code.=<<___; | ||
| 84 | .globl ${PREFIX}_encrypt | ||
| 85 | .type ${PREFIX}_encrypt,\@abi-omnipotent | ||
| 86 | .align 16 | ||
| 87 | ${PREFIX}_encrypt: | ||
| 88 | movups ($inp),$inout0 # load input | ||
| 89 | mov 240($key),$rounds # pull $rounds | ||
| 90 | ___ | ||
| 91 | &aesni_generate1("enc",$key,$rounds); | ||
| 92 | $code.=<<___; | ||
| 93 | movups $inout0,($out) # output | ||
| 94 | ret | ||
| 95 | .size ${PREFIX}_encrypt,.-${PREFIX}_encrypt | ||
| 96 | |||
| 97 | .globl ${PREFIX}_decrypt | ||
| 98 | .type ${PREFIX}_decrypt,\@abi-omnipotent | ||
| 99 | .align 16 | ||
| 100 | ${PREFIX}_decrypt: | ||
| 101 | movups ($inp),$inout0 # load input | ||
| 102 | mov 240($key),$rounds # pull $rounds | ||
| 103 | ___ | ||
| 104 | &aesni_generate1("dec",$key,$rounds); | ||
| 105 | $code.=<<___; | ||
| 106 | movups $inout0,($out) # output | ||
| 107 | ret | ||
| 108 | .size ${PREFIX}_decrypt, .-${PREFIX}_decrypt | ||
| 109 | ___ | ||
| 110 | } | ||
| 111 | |||
| 112 | # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave | ||
| 113 | # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec] | ||
| 114 | # latency is 6, it turned out that it can be scheduled only every | ||
| 115 | # *second* cycle. Thus 3x interleave is the one providing optimal | ||
| 116 | # utilization, i.e. when subroutine's throughput is virtually same as | ||
| 117 | # of non-interleaved subroutine [for number of input blocks up to 3]. | ||
| 118 | # This is why it makes no sense to implement 2x subroutine. As soon | ||
| 119 | # as/if Intel improves throughput by making it possible to schedule | ||
| 120 | # the instructions in question *every* cycles I would have to | ||
| 121 | # implement 6x interleave and use it in loop... | ||
| 122 | sub aesni_generate3 { | ||
| 123 | my $dir=shift; | ||
| 124 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
| 125 | # preserved. $inout[0-2] is cipher/clear text... | ||
| 126 | $code.=<<___; | ||
| 127 | .type _aesni_${dir}rypt3,\@abi-omnipotent | ||
| 128 | .align 16 | ||
| 129 | _aesni_${dir}rypt3: | ||
| 130 | $movkey ($key),$rndkey0 | ||
| 131 | shr \$1,$rounds | ||
| 132 | $movkey 16($key),$rndkey1 | ||
| 133 | lea 32($key),$key | ||
| 134 | pxor $rndkey0,$inout0 | ||
| 135 | pxor $rndkey0,$inout1 | ||
| 136 | pxor $rndkey0,$inout2 | ||
| 137 | |||
| 138 | .L${dir}_loop3: | ||
| 139 | aes${dir} $rndkey1,$inout0 | ||
| 140 | $movkey ($key),$rndkey0 | ||
| 141 | aes${dir} $rndkey1,$inout1 | ||
| 142 | dec $rounds | ||
| 143 | aes${dir} $rndkey1,$inout2 | ||
| 144 | aes${dir} $rndkey0,$inout0 | ||
| 145 | $movkey 16($key),$rndkey1 | ||
| 146 | aes${dir} $rndkey0,$inout1 | ||
| 147 | lea 32($key),$key | ||
| 148 | aes${dir} $rndkey0,$inout2 | ||
| 149 | jnz .L${dir}_loop3 | ||
| 150 | |||
| 151 | aes${dir} $rndkey1,$inout0 | ||
| 152 | $movkey ($key),$rndkey0 | ||
| 153 | aes${dir} $rndkey1,$inout1 | ||
| 154 | aes${dir} $rndkey1,$inout2 | ||
| 155 | aes${dir}last $rndkey0,$inout0 | ||
| 156 | aes${dir}last $rndkey0,$inout1 | ||
| 157 | aes${dir}last $rndkey0,$inout2 | ||
| 158 | ret | ||
| 159 | .size _aesni_${dir}rypt3,.-_aesni_${dir}rypt3 | ||
| 160 | ___ | ||
| 161 | } | ||
| 162 | # 4x interleave is implemented to improve small block performance, | ||
| 163 | # most notably [and naturally] 4 block by ~30%. One can argue that one | ||
| 164 | # should have implemented 5x as well, but improvement would be <20%, | ||
| 165 | # so it's not worth it... | ||
| 166 | sub aesni_generate4 { | ||
| 167 | my $dir=shift; | ||
| 168 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
| 169 | # preserved. $inout[0-3] is cipher/clear text... | ||
| 170 | $code.=<<___; | ||
| 171 | .type _aesni_${dir}rypt4,\@abi-omnipotent | ||
| 172 | .align 16 | ||
| 173 | _aesni_${dir}rypt4: | ||
| 174 | $movkey ($key),$rndkey0 | ||
| 175 | shr \$1,$rounds | ||
| 176 | $movkey 16($key),$rndkey1 | ||
| 177 | lea 32($key),$key | ||
| 178 | pxor $rndkey0,$inout0 | ||
| 179 | pxor $rndkey0,$inout1 | ||
| 180 | pxor $rndkey0,$inout2 | ||
| 181 | pxor $rndkey0,$inout3 | ||
| 182 | |||
| 183 | .L${dir}_loop4: | ||
| 184 | aes${dir} $rndkey1,$inout0 | ||
| 185 | $movkey ($key),$rndkey0 | ||
| 186 | aes${dir} $rndkey1,$inout1 | ||
| 187 | dec $rounds | ||
| 188 | aes${dir} $rndkey1,$inout2 | ||
| 189 | aes${dir} $rndkey1,$inout3 | ||
| 190 | aes${dir} $rndkey0,$inout0 | ||
| 191 | $movkey 16($key),$rndkey1 | ||
| 192 | aes${dir} $rndkey0,$inout1 | ||
| 193 | lea 32($key),$key | ||
| 194 | aes${dir} $rndkey0,$inout2 | ||
| 195 | aes${dir} $rndkey0,$inout3 | ||
| 196 | jnz .L${dir}_loop4 | ||
| 197 | |||
| 198 | aes${dir} $rndkey1,$inout0 | ||
| 199 | $movkey ($key),$rndkey0 | ||
| 200 | aes${dir} $rndkey1,$inout1 | ||
| 201 | aes${dir} $rndkey1,$inout2 | ||
| 202 | aes${dir} $rndkey1,$inout3 | ||
| 203 | aes${dir}last $rndkey0,$inout0 | ||
| 204 | aes${dir}last $rndkey0,$inout1 | ||
| 205 | aes${dir}last $rndkey0,$inout2 | ||
| 206 | aes${dir}last $rndkey0,$inout3 | ||
| 207 | ret | ||
| 208 | .size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4 | ||
| 209 | ___ | ||
| 210 | } | ||
| 211 | &aesni_generate3("enc") if ($PREFIX eq "aesni"); | ||
| 212 | &aesni_generate3("dec"); | ||
| 213 | &aesni_generate4("enc") if ($PREFIX eq "aesni"); | ||
| 214 | &aesni_generate4("dec"); | ||
| 215 | |||
| 216 | if ($PREFIX eq "aesni") { | ||
| 217 | # void aesni_ecb_encrypt (const void *in, void *out, | ||
| 218 | # size_t length, const AES_KEY *key, | ||
| 219 | # int enc); | ||
| 220 | $code.=<<___; | ||
| 221 | .globl aesni_ecb_encrypt | ||
| 222 | .type aesni_ecb_encrypt,\@function,5 | ||
| 223 | .align 16 | ||
| 224 | aesni_ecb_encrypt: | ||
| 225 | cmp \$16,$len # check length | ||
| 226 | jb .Lecb_ret | ||
| 227 | |||
| 228 | mov 240($key),$rounds # pull $rounds | ||
| 229 | and \$-16,$len | ||
| 230 | mov $key,$key_ # backup $key | ||
| 231 | test %r8d,%r8d # 5th argument | ||
| 232 | mov $rounds,$rnds_ # backup $rounds | ||
| 233 | jz .Lecb_decrypt | ||
| 234 | #--------------------------- ECB ENCRYPT ------------------------------# | ||
| 235 | sub \$0x40,$len | ||
| 236 | jbe .Lecb_enc_tail | ||
| 237 | jmp .Lecb_enc_loop3 | ||
| 238 | .align 16 | ||
| 239 | .Lecb_enc_loop3: | ||
| 240 | movups ($inp),$inout0 | ||
| 241 | movups 0x10($inp),$inout1 | ||
| 242 | movups 0x20($inp),$inout2 | ||
| 243 | call _aesni_encrypt3 | ||
| 244 | sub \$0x30,$len | ||
| 245 | lea 0x30($inp),$inp | ||
| 246 | lea 0x30($out),$out | ||
| 247 | movups $inout0,-0x30($out) | ||
| 248 | mov $rnds_,$rounds # restore $rounds | ||
| 249 | movups $inout1,-0x20($out) | ||
| 250 | mov $key_,$key # restore $key | ||
| 251 | movups $inout2,-0x10($out) | ||
| 252 | ja .Lecb_enc_loop3 | ||
| 253 | |||
| 254 | .Lecb_enc_tail: | ||
| 255 | add \$0x40,$len | ||
| 256 | jz .Lecb_ret | ||
| 257 | |||
| 258 | cmp \$0x10,$len | ||
| 259 | movups ($inp),$inout0 | ||
| 260 | je .Lecb_enc_one | ||
| 261 | cmp \$0x20,$len | ||
| 262 | movups 0x10($inp),$inout1 | ||
| 263 | je .Lecb_enc_two | ||
| 264 | cmp \$0x30,$len | ||
| 265 | movups 0x20($inp),$inout2 | ||
| 266 | je .Lecb_enc_three | ||
| 267 | movups 0x30($inp),$inout3 | ||
| 268 | call _aesni_encrypt4 | ||
| 269 | movups $inout0,($out) | ||
| 270 | movups $inout1,0x10($out) | ||
| 271 | movups $inout2,0x20($out) | ||
| 272 | movups $inout3,0x30($out) | ||
| 273 | jmp .Lecb_ret | ||
| 274 | .align 16 | ||
| 275 | .Lecb_enc_one: | ||
| 276 | ___ | ||
| 277 | &aesni_generate1("enc",$key,$rounds); | ||
| 278 | $code.=<<___; | ||
| 279 | movups $inout0,($out) | ||
| 280 | jmp .Lecb_ret | ||
| 281 | .align 16 | ||
| 282 | .Lecb_enc_two: | ||
| 283 | call _aesni_encrypt3 | ||
| 284 | movups $inout0,($out) | ||
| 285 | movups $inout1,0x10($out) | ||
| 286 | jmp .Lecb_ret | ||
| 287 | .align 16 | ||
| 288 | .Lecb_enc_three: | ||
| 289 | call _aesni_encrypt3 | ||
| 290 | movups $inout0,($out) | ||
| 291 | movups $inout1,0x10($out) | ||
| 292 | movups $inout2,0x20($out) | ||
| 293 | jmp .Lecb_ret | ||
| 294 | #--------------------------- ECB DECRYPT ------------------------------# | ||
| 295 | .align 16 | ||
| 296 | .Lecb_decrypt: | ||
| 297 | sub \$0x40,$len | ||
| 298 | jbe .Lecb_dec_tail | ||
| 299 | jmp .Lecb_dec_loop3 | ||
| 300 | .align 16 | ||
| 301 | .Lecb_dec_loop3: | ||
| 302 | movups ($inp),$inout0 | ||
| 303 | movups 0x10($inp),$inout1 | ||
| 304 | movups 0x20($inp),$inout2 | ||
| 305 | call _aesni_decrypt3 | ||
| 306 | sub \$0x30,$len | ||
| 307 | lea 0x30($inp),$inp | ||
| 308 | lea 0x30($out),$out | ||
| 309 | movups $inout0,-0x30($out) | ||
| 310 | mov $rnds_,$rounds # restore $rounds | ||
| 311 | movups $inout1,-0x20($out) | ||
| 312 | mov $key_,$key # restore $key | ||
| 313 | movups $inout2,-0x10($out) | ||
| 314 | ja .Lecb_dec_loop3 | ||
| 315 | |||
| 316 | .Lecb_dec_tail: | ||
| 317 | add \$0x40,$len | ||
| 318 | jz .Lecb_ret | ||
| 319 | |||
| 320 | cmp \$0x10,$len | ||
| 321 | movups ($inp),$inout0 | ||
| 322 | je .Lecb_dec_one | ||
| 323 | cmp \$0x20,$len | ||
| 324 | movups 0x10($inp),$inout1 | ||
| 325 | je .Lecb_dec_two | ||
| 326 | cmp \$0x30,$len | ||
| 327 | movups 0x20($inp),$inout2 | ||
| 328 | je .Lecb_dec_three | ||
| 329 | movups 0x30($inp),$inout3 | ||
| 330 | call _aesni_decrypt4 | ||
| 331 | movups $inout0,($out) | ||
| 332 | movups $inout1,0x10($out) | ||
| 333 | movups $inout2,0x20($out) | ||
| 334 | movups $inout3,0x30($out) | ||
| 335 | jmp .Lecb_ret | ||
| 336 | .align 16 | ||
| 337 | .Lecb_dec_one: | ||
| 338 | ___ | ||
| 339 | &aesni_generate1("dec",$key,$rounds); | ||
| 340 | $code.=<<___; | ||
| 341 | movups $inout0,($out) | ||
| 342 | jmp .Lecb_ret | ||
| 343 | .align 16 | ||
| 344 | .Lecb_dec_two: | ||
| 345 | call _aesni_decrypt3 | ||
| 346 | movups $inout0,($out) | ||
| 347 | movups $inout1,0x10($out) | ||
| 348 | jmp .Lecb_ret | ||
| 349 | .align 16 | ||
| 350 | .Lecb_dec_three: | ||
| 351 | call _aesni_decrypt3 | ||
| 352 | movups $inout0,($out) | ||
| 353 | movups $inout1,0x10($out) | ||
| 354 | movups $inout2,0x20($out) | ||
| 355 | |||
| 356 | .Lecb_ret: | ||
| 357 | ret | ||
| 358 | .size aesni_ecb_encrypt,.-aesni_ecb_encrypt | ||
| 359 | ___ | ||
| 360 | } | ||
| 361 | |||
| 362 | # void $PREFIX_cbc_encrypt (const void *inp, void *out, | ||
| 363 | # size_t length, const AES_KEY *key, | ||
| 364 | # unsigned char *ivp,const int enc); | ||
| 365 | $reserved = $win64?0x40:-0x18; # used in decrypt | ||
| 366 | $code.=<<___; | ||
| 367 | .globl ${PREFIX}_cbc_encrypt | ||
| 368 | .type ${PREFIX}_cbc_encrypt,\@function,6 | ||
| 369 | .align 16 | ||
| 370 | ${PREFIX}_cbc_encrypt: | ||
| 371 | test $len,$len # check length | ||
| 372 | jz .Lcbc_ret | ||
| 373 | |||
| 374 | mov 240($key),$rnds_ # pull $rounds | ||
| 375 | mov $key,$key_ # backup $key | ||
| 376 | test %r9d,%r9d # 6th argument | ||
| 377 | jz .Lcbc_decrypt | ||
| 378 | #--------------------------- CBC ENCRYPT ------------------------------# | ||
| 379 | movups ($ivp),$inout0 # load iv as initial state | ||
| 380 | cmp \$16,$len | ||
| 381 | mov $rnds_,$rounds | ||
| 382 | jb .Lcbc_enc_tail | ||
| 383 | sub \$16,$len | ||
| 384 | jmp .Lcbc_enc_loop | ||
| 385 | .align 16 | ||
| 386 | .Lcbc_enc_loop: | ||
| 387 | movups ($inp),$inout1 # load input | ||
| 388 | lea 16($inp),$inp | ||
| 389 | pxor $inout1,$inout0 | ||
| 390 | ___ | ||
| 391 | &aesni_generate1("enc",$key,$rounds); | ||
| 392 | $code.=<<___; | ||
| 393 | sub \$16,$len | ||
| 394 | lea 16($out),$out | ||
| 395 | mov $rnds_,$rounds # restore $rounds | ||
| 396 | mov $key_,$key # restore $key | ||
| 397 | movups $inout0,-16($out) # store output | ||
| 398 | jnc .Lcbc_enc_loop | ||
| 399 | add \$16,$len | ||
| 400 | jnz .Lcbc_enc_tail | ||
| 401 | movups $inout0,($ivp) | ||
| 402 | jmp .Lcbc_ret | ||
| 403 | |||
| 404 | .Lcbc_enc_tail: | ||
| 405 | mov $len,%rcx # zaps $key | ||
| 406 | xchg $inp,$out # $inp is %rsi and $out is %rdi now | ||
| 407 | .long 0x9066A4F3 # rep movsb | ||
| 408 | mov \$16,%ecx # zero tail | ||
| 409 | sub $len,%rcx | ||
| 410 | xor %eax,%eax | ||
| 411 | .long 0x9066AAF3 # rep stosb | ||
| 412 | lea -16(%rdi),%rdi # rewind $out by 1 block | ||
| 413 | mov $rnds_,$rounds # restore $rounds | ||
| 414 | mov %rdi,%rsi # $inp and $out are the same | ||
| 415 | mov $key_,$key # restore $key | ||
| 416 | xor $len,$len # len=16 | ||
| 417 | jmp .Lcbc_enc_loop # one more spin | ||
| 418 | #--------------------------- CBC DECRYPT ------------------------------# | ||
| 419 | .align 16 | ||
| 420 | .Lcbc_decrypt: | ||
| 421 | ___ | ||
| 422 | $code.=<<___ if ($win64); | ||
| 423 | lea -0x58(%rsp),%rsp | ||
| 424 | movaps %xmm6,(%rsp) | ||
| 425 | movaps %xmm7,0x10(%rsp) | ||
| 426 | movaps %xmm8,0x20(%rsp) | ||
| 427 | movaps %xmm9,0x30(%rsp) | ||
| 428 | .Lcbc_decrypt_body: | ||
| 429 | ___ | ||
| 430 | $code.=<<___; | ||
| 431 | movups ($ivp),$iv | ||
| 432 | sub \$0x40,$len | ||
| 433 | mov $rnds_,$rounds | ||
| 434 | jbe .Lcbc_dec_tail | ||
| 435 | jmp .Lcbc_dec_loop3 | ||
| 436 | .align 16 | ||
| 437 | .Lcbc_dec_loop3: | ||
| 438 | movups ($inp),$inout0 | ||
| 439 | movups 0x10($inp),$inout1 | ||
| 440 | movups 0x20($inp),$inout2 | ||
| 441 | movaps $inout0,$in0 | ||
| 442 | movaps $inout1,$in1 | ||
| 443 | movaps $inout2,$in2 | ||
| 444 | call _aesni_decrypt3 | ||
| 445 | sub \$0x30,$len | ||
| 446 | lea 0x30($inp),$inp | ||
| 447 | lea 0x30($out),$out | ||
| 448 | pxor $iv,$inout0 | ||
| 449 | pxor $in0,$inout1 | ||
| 450 | movaps $in2,$iv | ||
| 451 | pxor $in1,$inout2 | ||
| 452 | movups $inout0,-0x30($out) | ||
| 453 | mov $rnds_,$rounds # restore $rounds | ||
| 454 | movups $inout1,-0x20($out) | ||
| 455 | mov $key_,$key # restore $key | ||
| 456 | movups $inout2,-0x10($out) | ||
| 457 | ja .Lcbc_dec_loop3 | ||
| 458 | |||
| 459 | .Lcbc_dec_tail: | ||
| 460 | add \$0x40,$len | ||
| 461 | movups $iv,($ivp) | ||
| 462 | jz .Lcbc_dec_ret | ||
| 463 | |||
| 464 | movups ($inp),$inout0 | ||
| 465 | cmp \$0x10,$len | ||
| 466 | movaps $inout0,$in0 | ||
| 467 | jbe .Lcbc_dec_one | ||
| 468 | movups 0x10($inp),$inout1 | ||
| 469 | cmp \$0x20,$len | ||
| 470 | movaps $inout1,$in1 | ||
| 471 | jbe .Lcbc_dec_two | ||
| 472 | movups 0x20($inp),$inout2 | ||
| 473 | cmp \$0x30,$len | ||
| 474 | movaps $inout2,$in2 | ||
| 475 | jbe .Lcbc_dec_three | ||
| 476 | movups 0x30($inp),$inout3 | ||
| 477 | call _aesni_decrypt4 | ||
| 478 | pxor $iv,$inout0 | ||
| 479 | movups 0x30($inp),$iv | ||
| 480 | pxor $in0,$inout1 | ||
| 481 | movups $inout0,($out) | ||
| 482 | pxor $in1,$inout2 | ||
| 483 | movups $inout1,0x10($out) | ||
| 484 | pxor $in2,$inout3 | ||
| 485 | movups $inout2,0x20($out) | ||
| 486 | movaps $inout3,$inout0 | ||
| 487 | lea 0x30($out),$out | ||
| 488 | jmp .Lcbc_dec_tail_collected | ||
| 489 | .align 16 | ||
| 490 | .Lcbc_dec_one: | ||
| 491 | ___ | ||
| 492 | &aesni_generate1("dec",$key,$rounds); | ||
| 493 | $code.=<<___; | ||
| 494 | pxor $iv,$inout0 | ||
| 495 | movaps $in0,$iv | ||
| 496 | jmp .Lcbc_dec_tail_collected | ||
| 497 | .align 16 | ||
| 498 | .Lcbc_dec_two: | ||
| 499 | call _aesni_decrypt3 | ||
| 500 | pxor $iv,$inout0 | ||
| 501 | pxor $in0,$inout1 | ||
| 502 | movups $inout0,($out) | ||
| 503 | movaps $in1,$iv | ||
| 504 | movaps $inout1,$inout0 | ||
| 505 | lea 0x10($out),$out | ||
| 506 | jmp .Lcbc_dec_tail_collected | ||
| 507 | .align 16 | ||
| 508 | .Lcbc_dec_three: | ||
| 509 | call _aesni_decrypt3 | ||
| 510 | pxor $iv,$inout0 | ||
| 511 | pxor $in0,$inout1 | ||
| 512 | movups $inout0,($out) | ||
| 513 | pxor $in1,$inout2 | ||
| 514 | movups $inout1,0x10($out) | ||
| 515 | movaps $in2,$iv | ||
| 516 | movaps $inout2,$inout0 | ||
| 517 | lea 0x20($out),$out | ||
| 518 | jmp .Lcbc_dec_tail_collected | ||
| 519 | .align 16 | ||
| 520 | .Lcbc_dec_tail_collected: | ||
| 521 | and \$15,$len | ||
| 522 | movups $iv,($ivp) | ||
| 523 | jnz .Lcbc_dec_tail_partial | ||
| 524 | movups $inout0,($out) | ||
| 525 | jmp .Lcbc_dec_ret | ||
| 526 | .Lcbc_dec_tail_partial: | ||
| 527 | movaps $inout0,$reserved(%rsp) | ||
| 528 | mov $out,%rdi | ||
| 529 | mov $len,%rcx | ||
| 530 | lea $reserved(%rsp),%rsi | ||
| 531 | .long 0x9066A4F3 # rep movsb | ||
| 532 | |||
| 533 | .Lcbc_dec_ret: | ||
| 534 | ___ | ||
| 535 | $code.=<<___ if ($win64); | ||
| 536 | movaps (%rsp),%xmm6 | ||
| 537 | movaps 0x10(%rsp),%xmm7 | ||
| 538 | movaps 0x20(%rsp),%xmm8 | ||
| 539 | movaps 0x30(%rsp),%xmm9 | ||
| 540 | lea 0x58(%rsp),%rsp | ||
| 541 | ___ | ||
| 542 | $code.=<<___; | ||
| 543 | .Lcbc_ret: | ||
| 544 | ret | ||
| 545 | .size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt | ||
| 546 | ___ | ||
| 547 | |||
| 548 | # int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey, | ||
| 549 | # int bits, AES_KEY *key) | ||
| 550 | { my ($inp,$bits,$key) = @_4args; | ||
| 551 | $bits =~ s/%r/%e/; | ||
| 552 | |||
| 553 | $code.=<<___; | ||
| 554 | .globl ${PREFIX}_set_decrypt_key | ||
| 555 | .type ${PREFIX}_set_decrypt_key,\@abi-omnipotent | ||
| 556 | .align 16 | ||
| 557 | ${PREFIX}_set_decrypt_key: | ||
| 558 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
| 559 | call _aesni_set_encrypt_key | ||
| 560 | shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key | ||
| 561 | test %eax,%eax | ||
| 562 | jnz .Ldec_key_ret | ||
| 563 | lea 16($key,$bits),$inp # points at the end of key schedule | ||
| 564 | |||
| 565 | $movkey ($key),%xmm0 # just swap | ||
| 566 | $movkey ($inp),%xmm1 | ||
| 567 | $movkey %xmm0,($inp) | ||
| 568 | $movkey %xmm1,($key) | ||
| 569 | lea 16($key),$key | ||
| 570 | lea -16($inp),$inp | ||
| 571 | |||
| 572 | .Ldec_key_inverse: | ||
| 573 | $movkey ($key),%xmm0 # swap and inverse | ||
| 574 | $movkey ($inp),%xmm1 | ||
| 575 | aesimc %xmm0,%xmm0 | ||
| 576 | aesimc %xmm1,%xmm1 | ||
| 577 | lea 16($key),$key | ||
| 578 | lea -16($inp),$inp | ||
| 579 | cmp $key,$inp | ||
| 580 | $movkey %xmm0,16($inp) | ||
| 581 | $movkey %xmm1,-16($key) | ||
| 582 | ja .Ldec_key_inverse | ||
| 583 | |||
| 584 | $movkey ($key),%xmm0 # inverse middle | ||
| 585 | aesimc %xmm0,%xmm0 | ||
| 586 | $movkey %xmm0,($inp) | ||
| 587 | .Ldec_key_ret: | ||
| 588 | add \$8,%rsp | ||
| 589 | ret | ||
| 590 | .LSEH_end_set_decrypt_key: | ||
| 591 | .size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key | ||
| 592 | ___ | ||
| 593 | |||
| 594 | # This is based on submission by | ||
| 595 | # | ||
| 596 | # Huang Ying <ying.huang@intel.com> | ||
| 597 | # Vinodh Gopal <vinodh.gopal@intel.com> | ||
| 598 | # Kahraman Akdemir | ||
| 599 | # | ||
| 600 | # Agressively optimized in respect to aeskeygenassist's critical path | ||
| 601 | # and is contained in %xmm0-5 to meet Win64 ABI requirement. | ||
| 602 | # | ||
| 603 | $code.=<<___; | ||
| 604 | .globl ${PREFIX}_set_encrypt_key | ||
| 605 | .type ${PREFIX}_set_encrypt_key,\@abi-omnipotent | ||
| 606 | .align 16 | ||
| 607 | ${PREFIX}_set_encrypt_key: | ||
| 608 | _aesni_set_encrypt_key: | ||
| 609 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
| 610 | test $inp,$inp | ||
| 611 | mov \$-1,%rax | ||
| 612 | jz .Lenc_key_ret | ||
| 613 | test $key,$key | ||
| 614 | jz .Lenc_key_ret | ||
| 615 | |||
| 616 | movups ($inp),%xmm0 # pull first 128 bits of *userKey | ||
| 617 | pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0 | ||
| 618 | lea 16($key),%rax | ||
| 619 | cmp \$256,$bits | ||
| 620 | je .L14rounds | ||
| 621 | cmp \$192,$bits | ||
| 622 | je .L12rounds | ||
| 623 | cmp \$128,$bits | ||
| 624 | jne .Lbad_keybits | ||
| 625 | |||
| 626 | .L10rounds: | ||
| 627 | mov \$9,$bits # 10 rounds for 128-bit key | ||
| 628 | $movkey %xmm0,($key) # round 0 | ||
| 629 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 | ||
| 630 | call .Lkey_expansion_128_cold | ||
| 631 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 2 | ||
| 632 | call .Lkey_expansion_128 | ||
| 633 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 3 | ||
| 634 | call .Lkey_expansion_128 | ||
| 635 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 4 | ||
| 636 | call .Lkey_expansion_128 | ||
| 637 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 5 | ||
| 638 | call .Lkey_expansion_128 | ||
| 639 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 6 | ||
| 640 | call .Lkey_expansion_128 | ||
| 641 | aeskeygenassist \$0x40,%xmm0,%xmm1 # round 7 | ||
| 642 | call .Lkey_expansion_128 | ||
| 643 | aeskeygenassist \$0x80,%xmm0,%xmm1 # round 8 | ||
| 644 | call .Lkey_expansion_128 | ||
| 645 | aeskeygenassist \$0x1b,%xmm0,%xmm1 # round 9 | ||
| 646 | call .Lkey_expansion_128 | ||
| 647 | aeskeygenassist \$0x36,%xmm0,%xmm1 # round 10 | ||
| 648 | call .Lkey_expansion_128 | ||
| 649 | $movkey %xmm0,(%rax) | ||
| 650 | mov $bits,80(%rax) # 240(%rdx) | ||
| 651 | xor %eax,%eax | ||
| 652 | jmp .Lenc_key_ret | ||
| 653 | |||
| 654 | .align 16 | ||
| 655 | .L12rounds: | ||
| 656 | movq 16($inp),%xmm2 # remaining 1/3 of *userKey | ||
| 657 | mov \$11,$bits # 12 rounds for 192 | ||
| 658 | $movkey %xmm0,($key) # round 0 | ||
| 659 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 | ||
| 660 | call .Lkey_expansion_192a_cold | ||
| 661 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 2,3 | ||
| 662 | call .Lkey_expansion_192b | ||
| 663 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 4,5 | ||
| 664 | call .Lkey_expansion_192a | ||
| 665 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 5,6 | ||
| 666 | call .Lkey_expansion_192b | ||
| 667 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 7,8 | ||
| 668 | call .Lkey_expansion_192a | ||
| 669 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 8,9 | ||
| 670 | call .Lkey_expansion_192b | ||
| 671 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 10,11 | ||
| 672 | call .Lkey_expansion_192a | ||
| 673 | aeskeygenassist \$0x80,%xmm2,%xmm1 # round 11,12 | ||
| 674 | call .Lkey_expansion_192b | ||
| 675 | $movkey %xmm0,(%rax) | ||
| 676 | mov $bits,48(%rax) # 240(%rdx) | ||
| 677 | xor %rax, %rax | ||
| 678 | jmp .Lenc_key_ret | ||
| 679 | |||
| 680 | .align 16 | ||
| 681 | .L14rounds: | ||
| 682 | movups 16($inp),%xmm2 # remaning half of *userKey | ||
| 683 | mov \$13,$bits # 14 rounds for 256 | ||
| 684 | lea 16(%rax),%rax | ||
| 685 | $movkey %xmm0,($key) # round 0 | ||
| 686 | $movkey %xmm2,16($key) # round 1 | ||
| 687 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2 | ||
| 688 | call .Lkey_expansion_256a_cold | ||
| 689 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 3 | ||
| 690 | call .Lkey_expansion_256b | ||
| 691 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 4 | ||
| 692 | call .Lkey_expansion_256a | ||
| 693 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 5 | ||
| 694 | call .Lkey_expansion_256b | ||
| 695 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 6 | ||
| 696 | call .Lkey_expansion_256a | ||
| 697 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 7 | ||
| 698 | call .Lkey_expansion_256b | ||
| 699 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 8 | ||
| 700 | call .Lkey_expansion_256a | ||
| 701 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 9 | ||
| 702 | call .Lkey_expansion_256b | ||
| 703 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 10 | ||
| 704 | call .Lkey_expansion_256a | ||
| 705 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 11 | ||
| 706 | call .Lkey_expansion_256b | ||
| 707 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 12 | ||
| 708 | call .Lkey_expansion_256a | ||
| 709 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 13 | ||
| 710 | call .Lkey_expansion_256b | ||
| 711 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 14 | ||
| 712 | call .Lkey_expansion_256a | ||
| 713 | $movkey %xmm0,(%rax) | ||
| 714 | mov $bits,16(%rax) # 240(%rdx) | ||
| 715 | xor %rax,%rax | ||
| 716 | jmp .Lenc_key_ret | ||
| 717 | |||
| 718 | .align 16 | ||
| 719 | .Lbad_keybits: | ||
| 720 | mov \$-2,%rax | ||
| 721 | .Lenc_key_ret: | ||
| 722 | add \$8,%rsp | ||
| 723 | ret | ||
| 724 | .LSEH_end_set_encrypt_key: | ||
| 725 | |||
| 726 | .align 16 | ||
| 727 | .Lkey_expansion_128: | ||
| 728 | $movkey %xmm0,(%rax) | ||
| 729 | lea 16(%rax),%rax | ||
| 730 | .Lkey_expansion_128_cold: | ||
| 731 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 732 | pxor %xmm4, %xmm0 | ||
| 733 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 734 | pxor %xmm4, %xmm0 | ||
| 735 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
| 736 | pxor %xmm1,%xmm0 | ||
| 737 | ret | ||
| 738 | |||
| 739 | .align 16 | ||
| 740 | .Lkey_expansion_192a: | ||
| 741 | $movkey %xmm0,(%rax) | ||
| 742 | lea 16(%rax),%rax | ||
| 743 | .Lkey_expansion_192a_cold: | ||
| 744 | movaps %xmm2, %xmm5 | ||
| 745 | .Lkey_expansion_192b_warm: | ||
| 746 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 747 | movaps %xmm2,%xmm3 | ||
| 748 | pxor %xmm4,%xmm0 | ||
| 749 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 750 | pslldq \$4,%xmm3 | ||
| 751 | pxor %xmm4,%xmm0 | ||
| 752 | pshufd \$0b01010101,%xmm1,%xmm1 # critical path | ||
| 753 | pxor %xmm3,%xmm2 | ||
| 754 | pxor %xmm1,%xmm0 | ||
| 755 | pshufd \$0b11111111,%xmm0,%xmm3 | ||
| 756 | pxor %xmm3,%xmm2 | ||
| 757 | ret | ||
| 758 | |||
| 759 | .align 16 | ||
| 760 | .Lkey_expansion_192b: | ||
| 761 | movaps %xmm0,%xmm3 | ||
| 762 | shufps \$0b01000100,%xmm0,%xmm5 | ||
| 763 | $movkey %xmm5,(%rax) | ||
| 764 | shufps \$0b01001110,%xmm2,%xmm3 | ||
| 765 | $movkey %xmm3,16(%rax) | ||
| 766 | lea 32(%rax),%rax | ||
| 767 | jmp .Lkey_expansion_192b_warm | ||
| 768 | |||
| 769 | .align 16 | ||
| 770 | .Lkey_expansion_256a: | ||
| 771 | $movkey %xmm2,(%rax) | ||
| 772 | lea 16(%rax),%rax | ||
| 773 | .Lkey_expansion_256a_cold: | ||
| 774 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 775 | pxor %xmm4,%xmm0 | ||
| 776 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 777 | pxor %xmm4,%xmm0 | ||
| 778 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
| 779 | pxor %xmm1,%xmm0 | ||
| 780 | ret | ||
| 781 | |||
| 782 | .align 16 | ||
| 783 | .Lkey_expansion_256b: | ||
| 784 | $movkey %xmm0,(%rax) | ||
| 785 | lea 16(%rax),%rax | ||
| 786 | |||
| 787 | shufps \$0b00010000,%xmm2,%xmm4 | ||
| 788 | pxor %xmm4,%xmm2 | ||
| 789 | shufps \$0b10001100,%xmm2,%xmm4 | ||
| 790 | pxor %xmm4,%xmm2 | ||
| 791 | pshufd \$0b10101010,%xmm1,%xmm1 # critical path | ||
| 792 | pxor %xmm1,%xmm2 | ||
| 793 | ret | ||
| 794 | .size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key | ||
| 795 | ___ | ||
| 796 | } | ||
| 797 | |||
| 798 | $code.=<<___; | ||
| 799 | .asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>" | ||
| 800 | .align 64 | ||
| 801 | ___ | ||
| 802 | |||
| 803 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | ||
| 804 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | ||
| 805 | if ($win64) { | ||
| 806 | $rec="%rcx"; | ||
| 807 | $frame="%rdx"; | ||
| 808 | $context="%r8"; | ||
| 809 | $disp="%r9"; | ||
| 810 | |||
| 811 | $code.=<<___; | ||
| 812 | .extern __imp_RtlVirtualUnwind | ||
| 813 | .type cbc_se_handler,\@abi-omnipotent | ||
| 814 | .align 16 | ||
| 815 | cbc_se_handler: | ||
| 816 | push %rsi | ||
| 817 | push %rdi | ||
| 818 | push %rbx | ||
| 819 | push %rbp | ||
| 820 | push %r12 | ||
| 821 | push %r13 | ||
| 822 | push %r14 | ||
| 823 | push %r15 | ||
| 824 | pushfq | ||
| 825 | sub \$64,%rsp | ||
| 826 | |||
| 827 | mov 152($context),%rax # pull context->Rsp | ||
| 828 | mov 248($context),%rbx # pull context->Rip | ||
| 829 | |||
| 830 | lea .Lcbc_decrypt(%rip),%r10 | ||
| 831 | cmp %r10,%rbx # context->Rip<"prologue" label | ||
| 832 | jb .Lin_prologue | ||
| 833 | |||
| 834 | lea .Lcbc_decrypt_body(%rip),%r10 | ||
| 835 | cmp %r10,%rbx # context->Rip<cbc_decrypt_body | ||
| 836 | jb .Lrestore_rax | ||
| 837 | |||
| 838 | lea .Lcbc_ret(%rip),%r10 | ||
| 839 | cmp %r10,%rbx # context->Rip>="epilogue" label | ||
| 840 | jae .Lin_prologue | ||
| 841 | |||
| 842 | lea 0(%rax),%rsi # top of stack | ||
| 843 | lea 512($context),%rdi # &context.Xmm6 | ||
| 844 | mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax) | ||
| 845 | .long 0xa548f3fc # cld; rep movsq | ||
| 846 | lea 0x58(%rax),%rax # adjust stack pointer | ||
| 847 | jmp .Lin_prologue | ||
| 848 | |||
| 849 | .Lrestore_rax: | ||
| 850 | mov 120($context),%rax | ||
| 851 | .Lin_prologue: | ||
| 852 | mov 8(%rax),%rdi | ||
| 853 | mov 16(%rax),%rsi | ||
| 854 | mov %rax,152($context) # restore context->Rsp | ||
| 855 | mov %rsi,168($context) # restore context->Rsi | ||
| 856 | mov %rdi,176($context) # restore context->Rdi | ||
| 857 | |||
| 858 | jmp .Lcommon_seh_exit | ||
| 859 | .size cbc_se_handler,.-cbc_se_handler | ||
| 860 | |||
| 861 | .type ecb_se_handler,\@abi-omnipotent | ||
| 862 | .align 16 | ||
| 863 | ecb_se_handler: | ||
| 864 | push %rsi | ||
| 865 | push %rdi | ||
| 866 | push %rbx | ||
| 867 | push %rbp | ||
| 868 | push %r12 | ||
| 869 | push %r13 | ||
| 870 | push %r14 | ||
| 871 | push %r15 | ||
| 872 | pushfq | ||
| 873 | sub \$64,%rsp | ||
| 874 | |||
| 875 | mov 152($context),%rax # pull context->Rsp | ||
| 876 | mov 8(%rax),%rdi | ||
| 877 | mov 16(%rax),%rsi | ||
| 878 | mov %rsi,168($context) # restore context->Rsi | ||
| 879 | mov %rdi,176($context) # restore context->Rdi | ||
| 880 | |||
| 881 | .Lcommon_seh_exit: | ||
| 882 | |||
| 883 | mov 40($disp),%rdi # disp->ContextRecord | ||
| 884 | mov $context,%rsi # context | ||
| 885 | mov \$154,%ecx # sizeof(CONTEXT) | ||
| 886 | .long 0xa548f3fc # cld; rep movsq | ||
| 887 | |||
| 888 | mov $disp,%rsi | ||
| 889 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | ||
| 890 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | ||
| 891 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | ||
| 892 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | ||
| 893 | mov 40(%rsi),%r10 # disp->ContextRecord | ||
| 894 | lea 56(%rsi),%r11 # &disp->HandlerData | ||
| 895 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | ||
| 896 | mov %r10,32(%rsp) # arg5 | ||
| 897 | mov %r11,40(%rsp) # arg6 | ||
| 898 | mov %r12,48(%rsp) # arg7 | ||
| 899 | mov %rcx,56(%rsp) # arg8, (NULL) | ||
| 900 | call *__imp_RtlVirtualUnwind(%rip) | ||
| 901 | |||
| 902 | mov \$1,%eax # ExceptionContinueSearch | ||
| 903 | add \$64,%rsp | ||
| 904 | popfq | ||
| 905 | pop %r15 | ||
| 906 | pop %r14 | ||
| 907 | pop %r13 | ||
| 908 | pop %r12 | ||
| 909 | pop %rbp | ||
| 910 | pop %rbx | ||
| 911 | pop %rdi | ||
| 912 | pop %rsi | ||
| 913 | ret | ||
| 914 | .size cbc_se_handler,.-cbc_se_handler | ||
| 915 | |||
| 916 | .section .pdata | ||
| 917 | .align 4 | ||
| 918 | .rva .LSEH_begin_${PREFIX}_ecb_encrypt | ||
| 919 | .rva .LSEH_end_${PREFIX}_ecb_encrypt | ||
| 920 | .rva .LSEH_info_ecb | ||
| 921 | |||
| 922 | .rva .LSEH_begin_${PREFIX}_cbc_encrypt | ||
| 923 | .rva .LSEH_end_${PREFIX}_cbc_encrypt | ||
| 924 | .rva .LSEH_info_cbc | ||
| 925 | |||
| 926 | .rva ${PREFIX}_set_decrypt_key | ||
| 927 | .rva .LSEH_end_set_decrypt_key | ||
| 928 | .rva .LSEH_info_key | ||
| 929 | |||
| 930 | .rva ${PREFIX}_set_encrypt_key | ||
| 931 | .rva .LSEH_end_set_encrypt_key | ||
| 932 | .rva .LSEH_info_key | ||
| 933 | .section .xdata | ||
| 934 | .align 8 | ||
| 935 | .LSEH_info_ecb: | ||
| 936 | .byte 9,0,0,0 | ||
| 937 | .rva ecb_se_handler | ||
| 938 | .LSEH_info_cbc: | ||
| 939 | .byte 9,0,0,0 | ||
| 940 | .rva cbc_se_handler | ||
| 941 | .LSEH_info_key: | ||
| 942 | .byte 0x01,0x04,0x01,0x00 | ||
| 943 | .byte 0x04,0x02,0x00,0x00 | ||
| 944 | ___ | ||
| 945 | } | ||
| 946 | |||
| 947 | sub rex { | ||
| 948 | local *opcode=shift; | ||
| 949 | my ($dst,$src)=@_; | ||
| 950 | |||
| 951 | if ($dst>=8 || $src>=8) { | ||
| 952 | $rex=0x40; | ||
| 953 | $rex|=0x04 if($dst>=8); | ||
| 954 | $rex|=0x01 if($src>=8); | ||
| 955 | push @opcode,$rex; | ||
| 956 | } | ||
| 957 | } | ||
| 958 | |||
| 959 | sub aesni { | ||
| 960 | my $line=shift; | ||
| 961 | my @opcode=(0x66); | ||
| 962 | |||
| 963 | if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
| 964 | rex(\@opcode,$4,$3); | ||
| 965 | push @opcode,0x0f,0x3a,0xdf; | ||
| 966 | push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M | ||
| 967 | my $c=$2; | ||
| 968 | push @opcode,$c=~/^0/?oct($c):$c; | ||
| 969 | return ".byte\t".join(',',@opcode); | ||
| 970 | } | ||
| 971 | elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
| 972 | my %opcodelet = ( | ||
| 973 | "aesimc" => 0xdb, | ||
| 974 | "aesenc" => 0xdc, "aesenclast" => 0xdd, | ||
| 975 | "aesdec" => 0xde, "aesdeclast" => 0xdf | ||
| 976 | ); | ||
| 977 | return undef if (!defined($opcodelet{$1})); | ||
| 978 | rex(\@opcode,$3,$2); | ||
| 979 | push @opcode,0x0f,0x38,$opcodelet{$1}; | ||
| 980 | push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M | ||
| 981 | return ".byte\t".join(',',@opcode); | ||
| 982 | } | ||
| 983 | return $line; | ||
| 984 | } | ||
| 985 | |||
| 986 | $code =~ s/\`([^\`]*)\`/eval($1)/gem; | ||
| 987 | $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; | ||
| 988 | |||
| 989 | print $code; | ||
| 990 | |||
| 991 | close STDOUT; | ||
| 992 | |||
diff --git a/src/lib/libcrypto/crypto-lib.com b/src/lib/libcrypto/crypto-lib.com index e72af90822..8f1c1d5a80 100644 --- a/src/lib/libcrypto/crypto-lib.com +++ b/src/lib/libcrypto/crypto-lib.com | |||
| @@ -209,7 +209,8 @@ $ LIB_ENGINE = "eng_err,eng_lib,eng_list,eng_init,eng_ctrl,"+ - | |||
| 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - | 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - |
| 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - | 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - |
| 211 | "tb_cipher,tb_digest,"+ - | 211 | "tb_cipher,tb_digest,"+ - |
| 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock" | 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock"+ - |
| 213 | "eng_aesni" | ||
| 213 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - | 214 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - |
| 214 | "aes_ctr,aes_ige,aes_wrap" | 215 | "aes_ctr,aes_ige,aes_wrap" |
| 215 | $ LIB_BUFFER = "buffer,buf_str,buf_err" | 216 | $ LIB_BUFFER = "buffer,buf_str,buf_err" |
diff --git a/src/lib/libcrypto/engine/Makefile b/src/lib/libcrypto/engine/Makefile index 0cc3722089..0736153075 100644 --- a/src/lib/libcrypto/engine/Makefile +++ b/src/lib/libcrypto/engine/Makefile | |||
| @@ -21,12 +21,14 @@ LIBSRC= eng_err.c eng_lib.c eng_list.c eng_init.c eng_ctrl.c \ | |||
| 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ | 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ |
| 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ | 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ |
| 23 | tb_cipher.c tb_digest.c \ | 23 | tb_cipher.c tb_digest.c \ |
| 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c | 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c \ |
| 25 | eng_aesni.c | ||
| 25 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ | 26 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ |
| 26 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ | 27 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ |
| 27 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ | 28 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ |
| 28 | tb_cipher.o tb_digest.o \ | 29 | tb_cipher.o tb_digest.o \ |
| 29 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o | 30 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o \ |
| 31 | eng_aesni.o | ||
| 30 | 32 | ||
| 31 | SRC= $(LIBSRC) | 33 | SRC= $(LIBSRC) |
| 32 | 34 | ||
| @@ -82,6 +84,21 @@ clean: | |||
| 82 | 84 | ||
| 83 | # DO NOT DELETE THIS LINE -- make depend depends on it. | 85 | # DO NOT DELETE THIS LINE -- make depend depends on it. |
| 84 | 86 | ||
| 87 | eng_aesni.o: ../../e_os.h ../../include/openssl/aes.h | ||
| 88 | eng_aesni.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h | ||
| 89 | eng_aesni.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h | ||
| 90 | eng_aesni.o: ../../include/openssl/dso.h ../../include/openssl/e_os2.h | ||
| 91 | eng_aesni.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h | ||
| 92 | eng_aesni.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h | ||
| 93 | eng_aesni.o: ../../include/openssl/err.h ../../include/openssl/evp.h | ||
| 94 | eng_aesni.o: ../../include/openssl/lhash.h ../../include/openssl/modes.h | ||
| 95 | eng_aesni.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h | ||
| 96 | eng_aesni.o: ../../include/openssl/opensslconf.h | ||
| 97 | eng_aesni.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h | ||
| 98 | eng_aesni.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h | ||
| 99 | eng_aesni.o: ../../include/openssl/sha.h ../../include/openssl/stack.h | ||
| 100 | eng_aesni.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h | ||
| 101 | eng_aesni.o: ../../include/openssl/x509_vfy.h ../cryptlib.h eng_aesni.c | ||
| 85 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h | 102 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h |
| 86 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h | 103 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h |
| 87 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h | 104 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h |
diff --git a/src/lib/libcrypto/engine/eng_aesni.c b/src/lib/libcrypto/engine/eng_aesni.c new file mode 100644 index 0000000000..eeded1eefa --- /dev/null +++ b/src/lib/libcrypto/engine/eng_aesni.c | |||
| @@ -0,0 +1,570 @@ | |||
| 1 | /* | ||
| 2 | * Support for Intel AES-NI intruction set | ||
| 3 | * Author: Huang Ying <ying.huang@intel.com> | ||
| 4 | * | ||
| 5 | * Intel AES-NI is a new set of Single Instruction Multiple Data | ||
| 6 | * (SIMD) instructions that are going to be introduced in the next | ||
| 7 | * generation of Intel processor, as of 2009. These instructions | ||
| 8 | * enable fast and secure data encryption and decryption, using the | ||
| 9 | * Advanced Encryption Standard (AES), defined by FIPS Publication | ||
| 10 | * number 197. The architecture introduces six instructions that | ||
| 11 | * offer full hardware support for AES. Four of them support high | ||
| 12 | * performance data encryption and decryption, and the other two | ||
| 13 | * instructions support the AES key expansion procedure. | ||
| 14 | * | ||
| 15 | * The white paper can be downloaded from: | ||
| 16 | * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf | ||
| 17 | * | ||
| 18 | * This file is based on engines/e_padlock.c | ||
| 19 | */ | ||
| 20 | |||
| 21 | /* ==================================================================== | ||
| 22 | * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved. | ||
| 23 | * | ||
| 24 | * Redistribution and use in source and binary forms, with or without | ||
| 25 | * modification, are permitted provided that the following conditions | ||
| 26 | * are met: | ||
| 27 | * | ||
| 28 | * 1. Redistributions of source code must retain the above copyright | ||
| 29 | * notice, this list of conditions and the following disclaimer. | ||
| 30 | * | ||
| 31 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 32 | * notice, this list of conditions and the following disclaimer in | ||
| 33 | * the documentation and/or other materials provided with the | ||
| 34 | * distribution. | ||
| 35 | * | ||
| 36 | * 3. All advertising materials mentioning features or use of this | ||
| 37 | * software must display the following acknowledgment: | ||
| 38 | * "This product includes software developed by the OpenSSL Project | ||
| 39 | * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" | ||
| 40 | * | ||
| 41 | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to | ||
| 42 | * endorse or promote products derived from this software without | ||
| 43 | * prior written permission. For written permission, please contact | ||
| 44 | * licensing@OpenSSL.org. | ||
| 45 | * | ||
| 46 | * 5. Products derived from this software may not be called "OpenSSL" | ||
| 47 | * nor may "OpenSSL" appear in their names without prior written | ||
| 48 | * permission of the OpenSSL Project. | ||
| 49 | * | ||
| 50 | * 6. Redistributions of any form whatsoever must retain the following | ||
| 51 | * acknowledgment: | ||
| 52 | * "This product includes software developed by the OpenSSL Project | ||
| 53 | * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" | ||
| 54 | * | ||
| 55 | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY | ||
| 56 | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| 57 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 58 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR | ||
| 59 | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
| 60 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | ||
| 61 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | ||
| 62 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||
| 63 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | ||
| 64 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| 65 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED | ||
| 66 | * OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 67 | * ==================================================================== | ||
| 68 | * | ||
| 69 | * This product includes cryptographic software written by Eric Young | ||
| 70 | * (eay@cryptsoft.com). This product includes software written by Tim | ||
| 71 | * Hudson (tjh@cryptsoft.com). | ||
| 72 | * | ||
| 73 | */ | ||
| 74 | |||
| 75 | |||
| 76 | #include <openssl/opensslconf.h> | ||
| 77 | |||
| 78 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AES_NI) && !defined(OPENSSL_NO_AES) | ||
| 79 | |||
| 80 | #include <stdio.h> | ||
| 81 | #include <assert.h> | ||
| 82 | #include "cryptlib.h" | ||
| 83 | #include <openssl/dso.h> | ||
| 84 | #include <openssl/engine.h> | ||
| 85 | #include <openssl/evp.h> | ||
| 86 | #include <openssl/aes.h> | ||
| 87 | #include <openssl/err.h> | ||
| 88 | |||
| 89 | /* AES-NI is available *ONLY* on some x86 CPUs. Not only that it | ||
| 90 | doesn't exist elsewhere, but it even can't be compiled on other | ||
| 91 | platforms! */ | ||
| 92 | #undef COMPILE_HW_AESNI | ||
| 93 | #if (defined(__x86_64) || defined(__x86_64__) || \ | ||
| 94 | defined(_M_AMD64) || defined(_M_X64) || \ | ||
| 95 | defined(OPENSSL_IA32_SSE2)) && !defined(OPENSSL_NO_ASM) && !defined(__i386__) | ||
| 96 | #define COMPILE_HW_AESNI | ||
| 97 | #endif | ||
| 98 | static ENGINE *ENGINE_aesni (void); | ||
| 99 | |||
| 100 | void ENGINE_load_aesni (void) | ||
| 101 | { | ||
| 102 | /* On non-x86 CPUs it just returns. */ | ||
| 103 | #ifdef COMPILE_HW_AESNI | ||
| 104 | ENGINE *toadd = ENGINE_aesni(); | ||
| 105 | if (!toadd) return; | ||
| 106 | ENGINE_add (toadd); | ||
| 107 | ENGINE_register_complete (toadd); | ||
| 108 | ENGINE_free (toadd); | ||
| 109 | ERR_clear_error (); | ||
| 110 | #endif | ||
| 111 | } | ||
| 112 | |||
| 113 | #ifdef COMPILE_HW_AESNI | ||
| 114 | int aesni_set_encrypt_key(const unsigned char *userKey, int bits, | ||
| 115 | AES_KEY *key); | ||
| 116 | int aesni_set_decrypt_key(const unsigned char *userKey, int bits, | ||
| 117 | AES_KEY *key); | ||
| 118 | |||
| 119 | void aesni_encrypt(const unsigned char *in, unsigned char *out, | ||
| 120 | const AES_KEY *key); | ||
| 121 | void aesni_decrypt(const unsigned char *in, unsigned char *out, | ||
| 122 | const AES_KEY *key); | ||
| 123 | |||
| 124 | void aesni_ecb_encrypt(const unsigned char *in, | ||
| 125 | unsigned char *out, | ||
| 126 | size_t length, | ||
| 127 | const AES_KEY *key, | ||
| 128 | int enc); | ||
| 129 | void aesni_cbc_encrypt(const unsigned char *in, | ||
| 130 | unsigned char *out, | ||
| 131 | size_t length, | ||
| 132 | const AES_KEY *key, | ||
| 133 | unsigned char *ivec, int enc); | ||
| 134 | |||
| 135 | /* Function for ENGINE detection and control */ | ||
| 136 | static int aesni_init(ENGINE *e); | ||
| 137 | |||
| 138 | /* Cipher Stuff */ | ||
| 139 | static int aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | ||
| 140 | const int **nids, int nid); | ||
| 141 | |||
| 142 | #define AESNI_MIN_ALIGN 16 | ||
| 143 | #define AESNI_ALIGN(x) \ | ||
| 144 | ((void *)(((unsigned long)(x)+AESNI_MIN_ALIGN-1)&~(AESNI_MIN_ALIGN-1))) | ||
| 145 | |||
| 146 | /* Engine names */ | ||
| 147 | static const char aesni_id[] = "aesni", | ||
| 148 | aesni_name[] = "Intel AES-NI engine", | ||
| 149 | no_aesni_name[] = "Intel AES-NI engine (no-aesni)"; | ||
| 150 | |||
| 151 | |||
| 152 | /* The input and output encrypted as though 128bit cfb mode is being | ||
| 153 | * used. The extra state information to record how much of the | ||
| 154 | * 128bit block we have used is contained in *num; | ||
| 155 | */ | ||
| 156 | static void aesni_cfb128_encrypt(const unsigned char *in, unsigned char *out, | ||
| 157 | unsigned int len, const void *key, | ||
| 158 | unsigned char ivec[16], int *num, | ||
| 159 | int enc) | ||
| 160 | { | ||
| 161 | unsigned int n; | ||
| 162 | size_t l = 0; | ||
| 163 | |||
| 164 | assert(in && out && key && ivec && num); | ||
| 165 | |||
| 166 | n = *num; | ||
| 167 | |||
| 168 | if (enc) { | ||
| 169 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 170 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 171 | while (n && len) { | ||
| 172 | *(out++) = ivec[n] ^= *(in++); | ||
| 173 | --len; | ||
| 174 | n = (n+1) % 16; | ||
| 175 | } | ||
| 176 | while (len>=16) { | ||
| 177 | aesni_encrypt(ivec, ivec, key); | ||
| 178 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
| 179 | *(size_t*)(out+n) = | ||
| 180 | *(size_t*)(ivec+n) ^= *(size_t*)(in+n); | ||
| 181 | } | ||
| 182 | len -= 16; | ||
| 183 | out += 16; | ||
| 184 | in += 16; | ||
| 185 | } | ||
| 186 | n = 0; | ||
| 187 | if (len) { | ||
| 188 | aesni_encrypt(ivec, ivec, key); | ||
| 189 | while (len--) { | ||
| 190 | out[n] = ivec[n] ^= in[n]; | ||
| 191 | ++n; | ||
| 192 | } | ||
| 193 | } | ||
| 194 | *num = n; | ||
| 195 | return; | ||
| 196 | } while (0); | ||
| 197 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 198 | #endif | ||
| 199 | while (l<len) { | ||
| 200 | if (n == 0) { | ||
| 201 | aesni_encrypt(ivec, ivec, key); | ||
| 202 | } | ||
| 203 | out[l] = ivec[n] ^= in[l]; | ||
| 204 | ++l; | ||
| 205 | n = (n+1) % 16; | ||
| 206 | } | ||
| 207 | *num = n; | ||
| 208 | } else { | ||
| 209 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 210 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 211 | while (n && len) { | ||
| 212 | unsigned char c; | ||
| 213 | *(out++) = ivec[n] ^ (c = *(in++)); ivec[n] = c; | ||
| 214 | --len; | ||
| 215 | n = (n+1) % 16; | ||
| 216 | } | ||
| 217 | while (len>=16) { | ||
| 218 | aesni_encrypt(ivec, ivec, key); | ||
| 219 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
| 220 | size_t t = *(size_t*)(in+n); | ||
| 221 | *(size_t*)(out+n) = *(size_t*)(ivec+n) ^ t; | ||
| 222 | *(size_t*)(ivec+n) = t; | ||
| 223 | } | ||
| 224 | len -= 16; | ||
| 225 | out += 16; | ||
| 226 | in += 16; | ||
| 227 | } | ||
| 228 | n = 0; | ||
| 229 | if (len) { | ||
| 230 | aesni_encrypt(ivec, ivec, key); | ||
| 231 | while (len--) { | ||
| 232 | unsigned char c; | ||
| 233 | out[n] = ivec[n] ^ (c = in[n]); ivec[n] = c; | ||
| 234 | ++n; | ||
| 235 | } | ||
| 236 | } | ||
| 237 | *num = n; | ||
| 238 | return; | ||
| 239 | } while (0); | ||
| 240 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 241 | #endif | ||
| 242 | while (l<len) { | ||
| 243 | unsigned char c; | ||
| 244 | if (n == 0) { | ||
| 245 | aesni_encrypt(ivec, ivec, key); | ||
| 246 | } | ||
| 247 | out[l] = ivec[n] ^ (c = in[l]); ivec[n] = c; | ||
| 248 | ++l; | ||
| 249 | n = (n+1) % 16; | ||
| 250 | } | ||
| 251 | *num=n; | ||
| 252 | } | ||
| 253 | } | ||
| 254 | |||
| 255 | /* The input and output encrypted as though 128bit ofb mode is being | ||
| 256 | * used. The extra state information to record how much of the | ||
| 257 | * 128bit block we have used is contained in *num; | ||
| 258 | */ | ||
| 259 | static void aesni_ofb128_encrypt(const unsigned char *in, unsigned char *out, | ||
| 260 | unsigned int len, const void *key, | ||
| 261 | unsigned char ivec[16], int *num) | ||
| 262 | { | ||
| 263 | unsigned int n; | ||
| 264 | size_t l=0; | ||
| 265 | |||
| 266 | assert(in && out && key && ivec && num); | ||
| 267 | |||
| 268 | n = *num; | ||
| 269 | |||
| 270 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 271 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 272 | while (n && len) { | ||
| 273 | *(out++) = *(in++) ^ ivec[n]; | ||
| 274 | --len; | ||
| 275 | n = (n+1) % 16; | ||
| 276 | } | ||
| 277 | while (len>=16) { | ||
| 278 | aesni_encrypt(ivec, ivec, key); | ||
| 279 | for (n=0; n<16; n+=sizeof(size_t)) | ||
| 280 | *(size_t*)(out+n) = | ||
| 281 | *(size_t*)(in+n) ^ *(size_t*)(ivec+n); | ||
| 282 | len -= 16; | ||
| 283 | out += 16; | ||
| 284 | in += 16; | ||
| 285 | } | ||
| 286 | n = 0; | ||
| 287 | if (len) { | ||
| 288 | aesni_encrypt(ivec, ivec, key); | ||
| 289 | while (len--) { | ||
| 290 | out[n] = in[n] ^ ivec[n]; | ||
| 291 | ++n; | ||
| 292 | } | ||
| 293 | } | ||
| 294 | *num = n; | ||
| 295 | return; | ||
| 296 | } while(0); | ||
| 297 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 298 | #endif | ||
| 299 | while (l<len) { | ||
| 300 | if (n==0) { | ||
| 301 | aesni_encrypt(ivec, ivec, key); | ||
| 302 | } | ||
| 303 | out[l] = in[l] ^ ivec[n]; | ||
| 304 | ++l; | ||
| 305 | n = (n+1) % 16; | ||
| 306 | } | ||
| 307 | |||
| 308 | *num=n; | ||
| 309 | } | ||
| 310 | /* ===== Engine "management" functions ===== */ | ||
| 311 | |||
| 312 | #if defined(_WIN32) | ||
| 313 | typedef unsigned __int64 IA32CAP; | ||
| 314 | #else | ||
| 315 | typedef unsigned long long IA32CAP; | ||
| 316 | #endif | ||
| 317 | |||
| 318 | /* Prepare the ENGINE structure for registration */ | ||
| 319 | static int | ||
| 320 | aesni_bind_helper(ENGINE *e) | ||
| 321 | { | ||
| 322 | int engage; | ||
| 323 | if (sizeof(OPENSSL_ia32cap_P) > 4) { | ||
| 324 | engage = ((IA32CAP)OPENSSL_ia32cap_P >> 57) & 1; | ||
| 325 | } else { | ||
| 326 | IA32CAP OPENSSL_ia32_cpuid(void); | ||
| 327 | engage = (OPENSSL_ia32_cpuid() >> 57) & 1; | ||
| 328 | } | ||
| 329 | |||
| 330 | /* Register everything or return with an error */ | ||
| 331 | if (!ENGINE_set_id(e, aesni_id) || | ||
| 332 | !ENGINE_set_name(e, engage ? aesni_name : no_aesni_name) || | ||
| 333 | |||
| 334 | !ENGINE_set_init_function(e, aesni_init) || | ||
| 335 | (engage && !ENGINE_set_ciphers (e, aesni_ciphers)) | ||
| 336 | ) | ||
| 337 | return 0; | ||
| 338 | |||
| 339 | /* Everything looks good */ | ||
| 340 | return 1; | ||
| 341 | } | ||
| 342 | |||
| 343 | /* Constructor */ | ||
| 344 | static ENGINE * | ||
| 345 | ENGINE_aesni(void) | ||
| 346 | { | ||
| 347 | ENGINE *eng = ENGINE_new(); | ||
| 348 | |||
| 349 | if (!eng) { | ||
| 350 | return NULL; | ||
| 351 | } | ||
| 352 | |||
| 353 | if (!aesni_bind_helper(eng)) { | ||
| 354 | ENGINE_free(eng); | ||
| 355 | return NULL; | ||
| 356 | } | ||
| 357 | |||
| 358 | return eng; | ||
| 359 | } | ||
| 360 | |||
| 361 | /* Check availability of the engine */ | ||
| 362 | static int | ||
| 363 | aesni_init(ENGINE *e) | ||
| 364 | { | ||
| 365 | return 1; | ||
| 366 | } | ||
| 367 | |||
| 368 | #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb) | ||
| 369 | #define NID_aes_128_cfb NID_aes_128_cfb128 | ||
| 370 | #endif | ||
| 371 | |||
| 372 | #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb) | ||
| 373 | #define NID_aes_128_ofb NID_aes_128_ofb128 | ||
| 374 | #endif | ||
| 375 | |||
| 376 | #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb) | ||
| 377 | #define NID_aes_192_cfb NID_aes_192_cfb128 | ||
| 378 | #endif | ||
| 379 | |||
| 380 | #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb) | ||
| 381 | #define NID_aes_192_ofb NID_aes_192_ofb128 | ||
| 382 | #endif | ||
| 383 | |||
| 384 | #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb) | ||
| 385 | #define NID_aes_256_cfb NID_aes_256_cfb128 | ||
| 386 | #endif | ||
| 387 | |||
| 388 | #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb) | ||
| 389 | #define NID_aes_256_ofb NID_aes_256_ofb128 | ||
| 390 | #endif | ||
| 391 | |||
| 392 | /* List of supported ciphers. */ | ||
| 393 | static int aesni_cipher_nids[] = { | ||
| 394 | NID_aes_128_ecb, | ||
| 395 | NID_aes_128_cbc, | ||
| 396 | NID_aes_128_cfb, | ||
| 397 | NID_aes_128_ofb, | ||
| 398 | |||
| 399 | NID_aes_192_ecb, | ||
| 400 | NID_aes_192_cbc, | ||
| 401 | NID_aes_192_cfb, | ||
| 402 | NID_aes_192_ofb, | ||
| 403 | |||
| 404 | NID_aes_256_ecb, | ||
| 405 | NID_aes_256_cbc, | ||
| 406 | NID_aes_256_cfb, | ||
| 407 | NID_aes_256_ofb, | ||
| 408 | }; | ||
| 409 | static int aesni_cipher_nids_num = | ||
| 410 | (sizeof(aesni_cipher_nids)/sizeof(aesni_cipher_nids[0])); | ||
| 411 | |||
| 412 | typedef struct | ||
| 413 | { | ||
| 414 | AES_KEY ks; | ||
| 415 | unsigned int _pad1[3]; | ||
| 416 | } AESNI_KEY; | ||
| 417 | |||
| 418 | static int | ||
| 419 | aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *user_key, | ||
| 420 | const unsigned char *iv, int enc) | ||
| 421 | { | ||
| 422 | int ret; | ||
| 423 | AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 424 | |||
| 425 | if ((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CFB_MODE | ||
| 426 | || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_OFB_MODE | ||
| 427 | || enc) | ||
| 428 | ret=aesni_set_encrypt_key(user_key, ctx->key_len * 8, key); | ||
| 429 | else | ||
| 430 | ret=aesni_set_decrypt_key(user_key, ctx->key_len * 8, key); | ||
| 431 | |||
| 432 | if(ret < 0) { | ||
| 433 | EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED); | ||
| 434 | return 0; | ||
| 435 | } | ||
| 436 | |||
| 437 | return 1; | ||
| 438 | } | ||
| 439 | |||
| 440 | static int aesni_cipher_ecb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 441 | const unsigned char *in, unsigned int inl) | ||
| 442 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 443 | aesni_ecb_encrypt(in, out, inl, key, ctx->encrypt); | ||
| 444 | return 1; | ||
| 445 | } | ||
| 446 | static int aesni_cipher_cbc(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 447 | const unsigned char *in, unsigned int inl) | ||
| 448 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 449 | aesni_cbc_encrypt(in, out, inl, key, | ||
| 450 | ctx->iv, ctx->encrypt); | ||
| 451 | return 1; | ||
| 452 | } | ||
| 453 | static int aesni_cipher_cfb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 454 | const unsigned char *in, unsigned int inl) | ||
| 455 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 456 | |||
| 457 | aesni_cfb128_encrypt(in, out, inl, key, ctx->iv, | ||
| 458 | &ctx->num, ctx->encrypt); | ||
| 459 | return 1; | ||
| 460 | } | ||
| 461 | static int aesni_cipher_ofb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 462 | const unsigned char *in, unsigned int inl) | ||
| 463 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 464 | aesni_ofb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num); | ||
| 465 | return 1; | ||
| 466 | } | ||
| 467 | |||
| 468 | #define AES_BLOCK_SIZE 16 | ||
| 469 | |||
| 470 | #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE | ||
| 471 | #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE | ||
| 472 | #define EVP_CIPHER_block_size_OFB 1 | ||
| 473 | #define EVP_CIPHER_block_size_CFB 1 | ||
| 474 | |||
| 475 | /* Declaring so many ciphers by hand would be a pain. | ||
| 476 | Instead introduce a bit of preprocessor magic :-) */ | ||
| 477 | #define DECLARE_AES_EVP(ksize,lmode,umode) \ | ||
| 478 | static const EVP_CIPHER aesni_##ksize##_##lmode = { \ | ||
| 479 | NID_aes_##ksize##_##lmode, \ | ||
| 480 | EVP_CIPHER_block_size_##umode, \ | ||
| 481 | ksize / 8, \ | ||
| 482 | AES_BLOCK_SIZE, \ | ||
| 483 | 0 | EVP_CIPH_##umode##_MODE, \ | ||
| 484 | aesni_init_key, \ | ||
| 485 | aesni_cipher_##lmode, \ | ||
| 486 | NULL, \ | ||
| 487 | sizeof(AESNI_KEY), \ | ||
| 488 | EVP_CIPHER_set_asn1_iv, \ | ||
| 489 | EVP_CIPHER_get_asn1_iv, \ | ||
| 490 | NULL, \ | ||
| 491 | NULL \ | ||
| 492 | } | ||
| 493 | |||
| 494 | DECLARE_AES_EVP(128,ecb,ECB); | ||
| 495 | DECLARE_AES_EVP(128,cbc,CBC); | ||
| 496 | DECLARE_AES_EVP(128,cfb,CFB); | ||
| 497 | DECLARE_AES_EVP(128,ofb,OFB); | ||
| 498 | |||
| 499 | DECLARE_AES_EVP(192,ecb,ECB); | ||
| 500 | DECLARE_AES_EVP(192,cbc,CBC); | ||
| 501 | DECLARE_AES_EVP(192,cfb,CFB); | ||
| 502 | DECLARE_AES_EVP(192,ofb,OFB); | ||
| 503 | |||
| 504 | DECLARE_AES_EVP(256,ecb,ECB); | ||
| 505 | DECLARE_AES_EVP(256,cbc,CBC); | ||
| 506 | DECLARE_AES_EVP(256,cfb,CFB); | ||
| 507 | DECLARE_AES_EVP(256,ofb,OFB); | ||
| 508 | |||
| 509 | static int | ||
| 510 | aesni_ciphers (ENGINE *e, const EVP_CIPHER **cipher, | ||
| 511 | const int **nids, int nid) | ||
| 512 | { | ||
| 513 | /* No specific cipher => return a list of supported nids ... */ | ||
| 514 | if (!cipher) { | ||
| 515 | *nids = aesni_cipher_nids; | ||
| 516 | return aesni_cipher_nids_num; | ||
| 517 | } | ||
| 518 | |||
| 519 | /* ... or the requested "cipher" otherwise */ | ||
| 520 | switch (nid) { | ||
| 521 | case NID_aes_128_ecb: | ||
| 522 | *cipher = &aesni_128_ecb; | ||
| 523 | break; | ||
| 524 | case NID_aes_128_cbc: | ||
| 525 | *cipher = &aesni_128_cbc; | ||
| 526 | break; | ||
| 527 | case NID_aes_128_cfb: | ||
| 528 | *cipher = &aesni_128_cfb; | ||
| 529 | break; | ||
| 530 | case NID_aes_128_ofb: | ||
| 531 | *cipher = &aesni_128_ofb; | ||
| 532 | break; | ||
| 533 | |||
| 534 | case NID_aes_192_ecb: | ||
| 535 | *cipher = &aesni_192_ecb; | ||
| 536 | break; | ||
| 537 | case NID_aes_192_cbc: | ||
| 538 | *cipher = &aesni_192_cbc; | ||
| 539 | break; | ||
| 540 | case NID_aes_192_cfb: | ||
| 541 | *cipher = &aesni_192_cfb; | ||
| 542 | break; | ||
| 543 | case NID_aes_192_ofb: | ||
| 544 | *cipher = &aesni_192_ofb; | ||
| 545 | break; | ||
| 546 | |||
| 547 | case NID_aes_256_ecb: | ||
| 548 | *cipher = &aesni_256_ecb; | ||
| 549 | break; | ||
| 550 | case NID_aes_256_cbc: | ||
| 551 | *cipher = &aesni_256_cbc; | ||
| 552 | break; | ||
| 553 | case NID_aes_256_cfb: | ||
| 554 | *cipher = &aesni_256_cfb; | ||
| 555 | break; | ||
| 556 | case NID_aes_256_ofb: | ||
| 557 | *cipher = &aesni_256_ofb; | ||
| 558 | break; | ||
| 559 | |||
| 560 | default: | ||
| 561 | /* Sorry, we don't support this NID */ | ||
| 562 | *cipher = NULL; | ||
| 563 | return 0; | ||
| 564 | } | ||
| 565 | return 1; | ||
| 566 | } | ||
| 567 | |||
| 568 | #endif /* COMPILE_HW_AESNI */ | ||
| 569 | #endif /* !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) && !defined(OPENSSL_NO_AES) */ | ||
| 570 | |||
diff --git a/src/lib/libcrypto/engine/eng_all.c b/src/lib/libcrypto/engine/eng_all.c index d29cd57dc2..69e54f461e 100644 --- a/src/lib/libcrypto/engine/eng_all.c +++ b/src/lib/libcrypto/engine/eng_all.c | |||
| @@ -71,6 +71,11 @@ void ENGINE_load_builtin_engines(void) | |||
| 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) | 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) |
| 72 | ENGINE_load_padlock(); | 72 | ENGINE_load_padlock(); |
| 73 | #endif | 73 | #endif |
| 74 | |||
| 75 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) | ||
| 76 | ENGINE_load_aesni(); | ||
| 77 | #endif | ||
| 78 | |||
| 74 | ENGINE_load_dynamic(); | 79 | ENGINE_load_dynamic(); |
| 75 | #ifndef OPENSSL_NO_STATIC_ENGINE | 80 | #ifndef OPENSSL_NO_STATIC_ENGINE |
| 76 | #ifndef OPENSSL_NO_HW | 81 | #ifndef OPENSSL_NO_HW |
diff --git a/src/lib/libcrypto/engine/engine.h b/src/lib/libcrypto/engine/engine.h index f503595ece..ecab9453ba 100644 --- a/src/lib/libcrypto/engine/engine.h +++ b/src/lib/libcrypto/engine/engine.h | |||
| @@ -337,6 +337,7 @@ void ENGINE_load_sureware(void); | |||
| 337 | void ENGINE_load_ubsec(void); | 337 | void ENGINE_load_ubsec(void); |
| 338 | #endif | 338 | #endif |
| 339 | void ENGINE_load_cryptodev(void); | 339 | void ENGINE_load_cryptodev(void); |
| 340 | void ENGINE_load_aesni(void); | ||
| 340 | void ENGINE_load_padlock(void); | 341 | void ENGINE_load_padlock(void); |
| 341 | void ENGINE_load_builtin_engines(void); | 342 | void ENGINE_load_builtin_engines(void); |
| 342 | #ifndef OPENSSL_NO_CAPIENG | 343 | #ifndef OPENSSL_NO_CAPIENG |
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h index 51011f2b14..82b5862ce7 100644 --- a/src/lib/libcrypto/evp/evp.h +++ b/src/lib/libcrypto/evp/evp.h | |||
| @@ -959,6 +959,7 @@ void ERR_load_EVP_strings(void); | |||
| 959 | /* Error codes for the EVP functions. */ | 959 | /* Error codes for the EVP functions. */ |
| 960 | 960 | ||
| 961 | /* Function codes. */ | 961 | /* Function codes. */ |
| 962 | #define EVP_F_AESNI_INIT_KEY 165 | ||
| 962 | #define EVP_F_AES_INIT_KEY 133 | 963 | #define EVP_F_AES_INIT_KEY 133 |
| 963 | #define EVP_F_ALG_MODULE_INIT 138 | 964 | #define EVP_F_ALG_MODULE_INIT 138 |
| 964 | #define EVP_F_CAMELLIA_INIT_KEY 159 | 965 | #define EVP_F_CAMELLIA_INIT_KEY 159 |
diff --git a/src/lib/libcrypto/evp/evp_err.c b/src/lib/libcrypto/evp/evp_err.c index b5b900d4fe..ec2d127cd8 100644 --- a/src/lib/libcrypto/evp/evp_err.c +++ b/src/lib/libcrypto/evp/evp_err.c | |||
| @@ -70,6 +70,7 @@ | |||
| 70 | 70 | ||
| 71 | static ERR_STRING_DATA EVP_str_functs[]= | 71 | static ERR_STRING_DATA EVP_str_functs[]= |
| 72 | { | 72 | { |
| 73 | {ERR_FUNC(EVP_F_AESNI_INIT_KEY), "AESNI_INIT_KEY"}, | ||
| 73 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, | 74 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, |
| 74 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, | 75 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, |
| 75 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, | 76 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, |
diff --git a/src/lib/libcrypto/util/libeay.num b/src/lib/libcrypto/util/libeay.num index 74eb337227..0bad595233 100644 --- a/src/lib/libcrypto/util/libeay.num +++ b/src/lib/libcrypto/util/libeay.num | |||
| @@ -3725,3 +3725,4 @@ JPAKE_STEP2_release 4110 EXIST::FUNCTION:JPAKE | |||
| 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE | 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE |
| 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE | 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE |
| 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE | 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE |
| 3728 | ENGINE_load_aesni 4114 EXIST::FUNCTION:ENGINE | ||
diff --git a/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl b/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl new file mode 100644 index 0000000000..49e0f4b351 --- /dev/null +++ b/src/lib/libssl/src/crypto/aes/asm/aesni-x86_64.pl | |||
| @@ -0,0 +1,992 @@ | |||
| 1 | #!/usr/bin/env perl | ||
| 2 | # | ||
| 3 | # ==================================================================== | ||
| 4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
| 5 | # project. The module is, however, dual licensed under OpenSSL and | ||
| 6 | # CRYPTOGAMS licenses depending on where you obtain it. For further | ||
| 7 | # details see http://www.openssl.org/~appro/cryptogams/. | ||
| 8 | # ==================================================================== | ||
| 9 | # | ||
| 10 | # This module implements support for Intel AES-NI extension. In | ||
| 11 | # OpenSSL context it's used with Intel engine, but can also be used as | ||
| 12 | # drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for | ||
| 13 | # details]. | ||
| 14 | |||
| 15 | $PREFIX="aesni"; # if $PREFIX is set to "AES", the script | ||
| 16 | # generates drop-in replacement for | ||
| 17 | # crypto/aes/asm/aes-x86_64.pl:-) | ||
| 18 | |||
| 19 | $flavour = shift; | ||
| 20 | $output = shift; | ||
| 21 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | ||
| 22 | |||
| 23 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | ||
| 24 | |||
| 25 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | ||
| 26 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | ||
| 27 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | ||
| 28 | die "can't locate x86_64-xlate.pl"; | ||
| 29 | |||
| 30 | open STDOUT,"| $^X $xlate $flavour $output"; | ||
| 31 | |||
| 32 | $movkey = $PREFIX eq "aesni" ? "movaps" : "movups"; | ||
| 33 | @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order | ||
| 34 | ("%rdi","%rsi","%rdx","%rcx"); # Unix order | ||
| 35 | |||
| 36 | $code=".text\n"; | ||
| 37 | |||
| 38 | $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! | ||
| 39 | # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... | ||
| 40 | $inp="%rdi"; | ||
| 41 | $out="%rsi"; | ||
| 42 | $len="%rdx"; | ||
| 43 | $key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!! | ||
| 44 | $ivp="%r8"; # cbc | ||
| 45 | |||
| 46 | $rnds_="%r10d"; # backup copy for $rounds | ||
| 47 | $key_="%r11"; # backup copy for $key | ||
| 48 | |||
| 49 | # %xmm register layout | ||
| 50 | $inout0="%xmm0"; $inout1="%xmm1"; | ||
| 51 | $inout2="%xmm2"; $inout3="%xmm3"; | ||
| 52 | $rndkey0="%xmm4"; $rndkey1="%xmm5"; | ||
| 53 | |||
| 54 | $iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt | ||
| 55 | $in1="%xmm8"; $in2="%xmm9"; | ||
| 56 | |||
| 57 | # Inline version of internal aesni_[en|de]crypt1. | ||
| 58 | # | ||
| 59 | # Why folded loop? Because aes[enc|dec] is slow enough to accommodate | ||
| 60 | # cycles which take care of loop variables... | ||
| 61 | { my $sn; | ||
| 62 | sub aesni_generate1 { | ||
| 63 | my ($p,$key,$rounds)=@_; | ||
| 64 | ++$sn; | ||
| 65 | $code.=<<___; | ||
| 66 | $movkey ($key),$rndkey0 | ||
| 67 | $movkey 16($key),$rndkey1 | ||
| 68 | lea 32($key),$key | ||
| 69 | pxor $rndkey0,$inout0 | ||
| 70 | .Loop_${p}1_$sn: | ||
| 71 | aes${p} $rndkey1,$inout0 | ||
| 72 | dec $rounds | ||
| 73 | $movkey ($key),$rndkey1 | ||
| 74 | lea 16($key),$key | ||
| 75 | jnz .Loop_${p}1_$sn # loop body is 16 bytes | ||
| 76 | aes${p}last $rndkey1,$inout0 | ||
| 77 | ___ | ||
| 78 | }} | ||
| 79 | # void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key); | ||
| 80 | # | ||
| 81 | { my ($inp,$out,$key) = @_4args; | ||
| 82 | |||
| 83 | $code.=<<___; | ||
| 84 | .globl ${PREFIX}_encrypt | ||
| 85 | .type ${PREFIX}_encrypt,\@abi-omnipotent | ||
| 86 | .align 16 | ||
| 87 | ${PREFIX}_encrypt: | ||
| 88 | movups ($inp),$inout0 # load input | ||
| 89 | mov 240($key),$rounds # pull $rounds | ||
| 90 | ___ | ||
| 91 | &aesni_generate1("enc",$key,$rounds); | ||
| 92 | $code.=<<___; | ||
| 93 | movups $inout0,($out) # output | ||
| 94 | ret | ||
| 95 | .size ${PREFIX}_encrypt,.-${PREFIX}_encrypt | ||
| 96 | |||
| 97 | .globl ${PREFIX}_decrypt | ||
| 98 | .type ${PREFIX}_decrypt,\@abi-omnipotent | ||
| 99 | .align 16 | ||
| 100 | ${PREFIX}_decrypt: | ||
| 101 | movups ($inp),$inout0 # load input | ||
| 102 | mov 240($key),$rounds # pull $rounds | ||
| 103 | ___ | ||
| 104 | &aesni_generate1("dec",$key,$rounds); | ||
| 105 | $code.=<<___; | ||
| 106 | movups $inout0,($out) # output | ||
| 107 | ret | ||
| 108 | .size ${PREFIX}_decrypt, .-${PREFIX}_decrypt | ||
| 109 | ___ | ||
| 110 | } | ||
| 111 | |||
| 112 | # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave | ||
| 113 | # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec] | ||
| 114 | # latency is 6, it turned out that it can be scheduled only every | ||
| 115 | # *second* cycle. Thus 3x interleave is the one providing optimal | ||
| 116 | # utilization, i.e. when subroutine's throughput is virtually same as | ||
| 117 | # of non-interleaved subroutine [for number of input blocks up to 3]. | ||
| 118 | # This is why it makes no sense to implement 2x subroutine. As soon | ||
| 119 | # as/if Intel improves throughput by making it possible to schedule | ||
| 120 | # the instructions in question *every* cycles I would have to | ||
| 121 | # implement 6x interleave and use it in loop... | ||
| 122 | sub aesni_generate3 { | ||
| 123 | my $dir=shift; | ||
| 124 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
| 125 | # preserved. $inout[0-2] is cipher/clear text... | ||
| 126 | $code.=<<___; | ||
| 127 | .type _aesni_${dir}rypt3,\@abi-omnipotent | ||
| 128 | .align 16 | ||
| 129 | _aesni_${dir}rypt3: | ||
| 130 | $movkey ($key),$rndkey0 | ||
| 131 | shr \$1,$rounds | ||
| 132 | $movkey 16($key),$rndkey1 | ||
| 133 | lea 32($key),$key | ||
| 134 | pxor $rndkey0,$inout0 | ||
| 135 | pxor $rndkey0,$inout1 | ||
| 136 | pxor $rndkey0,$inout2 | ||
| 137 | |||
| 138 | .L${dir}_loop3: | ||
| 139 | aes${dir} $rndkey1,$inout0 | ||
| 140 | $movkey ($key),$rndkey0 | ||
| 141 | aes${dir} $rndkey1,$inout1 | ||
| 142 | dec $rounds | ||
| 143 | aes${dir} $rndkey1,$inout2 | ||
| 144 | aes${dir} $rndkey0,$inout0 | ||
| 145 | $movkey 16($key),$rndkey1 | ||
| 146 | aes${dir} $rndkey0,$inout1 | ||
| 147 | lea 32($key),$key | ||
| 148 | aes${dir} $rndkey0,$inout2 | ||
| 149 | jnz .L${dir}_loop3 | ||
| 150 | |||
| 151 | aes${dir} $rndkey1,$inout0 | ||
| 152 | $movkey ($key),$rndkey0 | ||
| 153 | aes${dir} $rndkey1,$inout1 | ||
| 154 | aes${dir} $rndkey1,$inout2 | ||
| 155 | aes${dir}last $rndkey0,$inout0 | ||
| 156 | aes${dir}last $rndkey0,$inout1 | ||
| 157 | aes${dir}last $rndkey0,$inout2 | ||
| 158 | ret | ||
| 159 | .size _aesni_${dir}rypt3,.-_aesni_${dir}rypt3 | ||
| 160 | ___ | ||
| 161 | } | ||
| 162 | # 4x interleave is implemented to improve small block performance, | ||
| 163 | # most notably [and naturally] 4 block by ~30%. One can argue that one | ||
| 164 | # should have implemented 5x as well, but improvement would be <20%, | ||
| 165 | # so it's not worth it... | ||
| 166 | sub aesni_generate4 { | ||
| 167 | my $dir=shift; | ||
| 168 | # As already mentioned it takes in $key and $rounds, which are *not* | ||
| 169 | # preserved. $inout[0-3] is cipher/clear text... | ||
| 170 | $code.=<<___; | ||
| 171 | .type _aesni_${dir}rypt4,\@abi-omnipotent | ||
| 172 | .align 16 | ||
| 173 | _aesni_${dir}rypt4: | ||
| 174 | $movkey ($key),$rndkey0 | ||
| 175 | shr \$1,$rounds | ||
| 176 | $movkey 16($key),$rndkey1 | ||
| 177 | lea 32($key),$key | ||
| 178 | pxor $rndkey0,$inout0 | ||
| 179 | pxor $rndkey0,$inout1 | ||
| 180 | pxor $rndkey0,$inout2 | ||
| 181 | pxor $rndkey0,$inout3 | ||
| 182 | |||
| 183 | .L${dir}_loop4: | ||
| 184 | aes${dir} $rndkey1,$inout0 | ||
| 185 | $movkey ($key),$rndkey0 | ||
| 186 | aes${dir} $rndkey1,$inout1 | ||
| 187 | dec $rounds | ||
| 188 | aes${dir} $rndkey1,$inout2 | ||
| 189 | aes${dir} $rndkey1,$inout3 | ||
| 190 | aes${dir} $rndkey0,$inout0 | ||
| 191 | $movkey 16($key),$rndkey1 | ||
| 192 | aes${dir} $rndkey0,$inout1 | ||
| 193 | lea 32($key),$key | ||
| 194 | aes${dir} $rndkey0,$inout2 | ||
| 195 | aes${dir} $rndkey0,$inout3 | ||
| 196 | jnz .L${dir}_loop4 | ||
| 197 | |||
| 198 | aes${dir} $rndkey1,$inout0 | ||
| 199 | $movkey ($key),$rndkey0 | ||
| 200 | aes${dir} $rndkey1,$inout1 | ||
| 201 | aes${dir} $rndkey1,$inout2 | ||
| 202 | aes${dir} $rndkey1,$inout3 | ||
| 203 | aes${dir}last $rndkey0,$inout0 | ||
| 204 | aes${dir}last $rndkey0,$inout1 | ||
| 205 | aes${dir}last $rndkey0,$inout2 | ||
| 206 | aes${dir}last $rndkey0,$inout3 | ||
| 207 | ret | ||
| 208 | .size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4 | ||
| 209 | ___ | ||
| 210 | } | ||
| 211 | &aesni_generate3("enc") if ($PREFIX eq "aesni"); | ||
| 212 | &aesni_generate3("dec"); | ||
| 213 | &aesni_generate4("enc") if ($PREFIX eq "aesni"); | ||
| 214 | &aesni_generate4("dec"); | ||
| 215 | |||
| 216 | if ($PREFIX eq "aesni") { | ||
| 217 | # void aesni_ecb_encrypt (const void *in, void *out, | ||
| 218 | # size_t length, const AES_KEY *key, | ||
| 219 | # int enc); | ||
| 220 | $code.=<<___; | ||
| 221 | .globl aesni_ecb_encrypt | ||
| 222 | .type aesni_ecb_encrypt,\@function,5 | ||
| 223 | .align 16 | ||
| 224 | aesni_ecb_encrypt: | ||
| 225 | cmp \$16,$len # check length | ||
| 226 | jb .Lecb_ret | ||
| 227 | |||
| 228 | mov 240($key),$rounds # pull $rounds | ||
| 229 | and \$-16,$len | ||
| 230 | mov $key,$key_ # backup $key | ||
| 231 | test %r8d,%r8d # 5th argument | ||
| 232 | mov $rounds,$rnds_ # backup $rounds | ||
| 233 | jz .Lecb_decrypt | ||
| 234 | #--------------------------- ECB ENCRYPT ------------------------------# | ||
| 235 | sub \$0x40,$len | ||
| 236 | jbe .Lecb_enc_tail | ||
| 237 | jmp .Lecb_enc_loop3 | ||
| 238 | .align 16 | ||
| 239 | .Lecb_enc_loop3: | ||
| 240 | movups ($inp),$inout0 | ||
| 241 | movups 0x10($inp),$inout1 | ||
| 242 | movups 0x20($inp),$inout2 | ||
| 243 | call _aesni_encrypt3 | ||
| 244 | sub \$0x30,$len | ||
| 245 | lea 0x30($inp),$inp | ||
| 246 | lea 0x30($out),$out | ||
| 247 | movups $inout0,-0x30($out) | ||
| 248 | mov $rnds_,$rounds # restore $rounds | ||
| 249 | movups $inout1,-0x20($out) | ||
| 250 | mov $key_,$key # restore $key | ||
| 251 | movups $inout2,-0x10($out) | ||
| 252 | ja .Lecb_enc_loop3 | ||
| 253 | |||
| 254 | .Lecb_enc_tail: | ||
| 255 | add \$0x40,$len | ||
| 256 | jz .Lecb_ret | ||
| 257 | |||
| 258 | cmp \$0x10,$len | ||
| 259 | movups ($inp),$inout0 | ||
| 260 | je .Lecb_enc_one | ||
| 261 | cmp \$0x20,$len | ||
| 262 | movups 0x10($inp),$inout1 | ||
| 263 | je .Lecb_enc_two | ||
| 264 | cmp \$0x30,$len | ||
| 265 | movups 0x20($inp),$inout2 | ||
| 266 | je .Lecb_enc_three | ||
| 267 | movups 0x30($inp),$inout3 | ||
| 268 | call _aesni_encrypt4 | ||
| 269 | movups $inout0,($out) | ||
| 270 | movups $inout1,0x10($out) | ||
| 271 | movups $inout2,0x20($out) | ||
| 272 | movups $inout3,0x30($out) | ||
| 273 | jmp .Lecb_ret | ||
| 274 | .align 16 | ||
| 275 | .Lecb_enc_one: | ||
| 276 | ___ | ||
| 277 | &aesni_generate1("enc",$key,$rounds); | ||
| 278 | $code.=<<___; | ||
| 279 | movups $inout0,($out) | ||
| 280 | jmp .Lecb_ret | ||
| 281 | .align 16 | ||
| 282 | .Lecb_enc_two: | ||
| 283 | call _aesni_encrypt3 | ||
| 284 | movups $inout0,($out) | ||
| 285 | movups $inout1,0x10($out) | ||
| 286 | jmp .Lecb_ret | ||
| 287 | .align 16 | ||
| 288 | .Lecb_enc_three: | ||
| 289 | call _aesni_encrypt3 | ||
| 290 | movups $inout0,($out) | ||
| 291 | movups $inout1,0x10($out) | ||
| 292 | movups $inout2,0x20($out) | ||
| 293 | jmp .Lecb_ret | ||
| 294 | #--------------------------- ECB DECRYPT ------------------------------# | ||
| 295 | .align 16 | ||
| 296 | .Lecb_decrypt: | ||
| 297 | sub \$0x40,$len | ||
| 298 | jbe .Lecb_dec_tail | ||
| 299 | jmp .Lecb_dec_loop3 | ||
| 300 | .align 16 | ||
| 301 | .Lecb_dec_loop3: | ||
| 302 | movups ($inp),$inout0 | ||
| 303 | movups 0x10($inp),$inout1 | ||
| 304 | movups 0x20($inp),$inout2 | ||
| 305 | call _aesni_decrypt3 | ||
| 306 | sub \$0x30,$len | ||
| 307 | lea 0x30($inp),$inp | ||
| 308 | lea 0x30($out),$out | ||
| 309 | movups $inout0,-0x30($out) | ||
| 310 | mov $rnds_,$rounds # restore $rounds | ||
| 311 | movups $inout1,-0x20($out) | ||
| 312 | mov $key_,$key # restore $key | ||
| 313 | movups $inout2,-0x10($out) | ||
| 314 | ja .Lecb_dec_loop3 | ||
| 315 | |||
| 316 | .Lecb_dec_tail: | ||
| 317 | add \$0x40,$len | ||
| 318 | jz .Lecb_ret | ||
| 319 | |||
| 320 | cmp \$0x10,$len | ||
| 321 | movups ($inp),$inout0 | ||
| 322 | je .Lecb_dec_one | ||
| 323 | cmp \$0x20,$len | ||
| 324 | movups 0x10($inp),$inout1 | ||
| 325 | je .Lecb_dec_two | ||
| 326 | cmp \$0x30,$len | ||
| 327 | movups 0x20($inp),$inout2 | ||
| 328 | je .Lecb_dec_three | ||
| 329 | movups 0x30($inp),$inout3 | ||
| 330 | call _aesni_decrypt4 | ||
| 331 | movups $inout0,($out) | ||
| 332 | movups $inout1,0x10($out) | ||
| 333 | movups $inout2,0x20($out) | ||
| 334 | movups $inout3,0x30($out) | ||
| 335 | jmp .Lecb_ret | ||
| 336 | .align 16 | ||
| 337 | .Lecb_dec_one: | ||
| 338 | ___ | ||
| 339 | &aesni_generate1("dec",$key,$rounds); | ||
| 340 | $code.=<<___; | ||
| 341 | movups $inout0,($out) | ||
| 342 | jmp .Lecb_ret | ||
| 343 | .align 16 | ||
| 344 | .Lecb_dec_two: | ||
| 345 | call _aesni_decrypt3 | ||
| 346 | movups $inout0,($out) | ||
| 347 | movups $inout1,0x10($out) | ||
| 348 | jmp .Lecb_ret | ||
| 349 | .align 16 | ||
| 350 | .Lecb_dec_three: | ||
| 351 | call _aesni_decrypt3 | ||
| 352 | movups $inout0,($out) | ||
| 353 | movups $inout1,0x10($out) | ||
| 354 | movups $inout2,0x20($out) | ||
| 355 | |||
| 356 | .Lecb_ret: | ||
| 357 | ret | ||
| 358 | .size aesni_ecb_encrypt,.-aesni_ecb_encrypt | ||
| 359 | ___ | ||
| 360 | } | ||
| 361 | |||
| 362 | # void $PREFIX_cbc_encrypt (const void *inp, void *out, | ||
| 363 | # size_t length, const AES_KEY *key, | ||
| 364 | # unsigned char *ivp,const int enc); | ||
| 365 | $reserved = $win64?0x40:-0x18; # used in decrypt | ||
| 366 | $code.=<<___; | ||
| 367 | .globl ${PREFIX}_cbc_encrypt | ||
| 368 | .type ${PREFIX}_cbc_encrypt,\@function,6 | ||
| 369 | .align 16 | ||
| 370 | ${PREFIX}_cbc_encrypt: | ||
| 371 | test $len,$len # check length | ||
| 372 | jz .Lcbc_ret | ||
| 373 | |||
| 374 | mov 240($key),$rnds_ # pull $rounds | ||
| 375 | mov $key,$key_ # backup $key | ||
| 376 | test %r9d,%r9d # 6th argument | ||
| 377 | jz .Lcbc_decrypt | ||
| 378 | #--------------------------- CBC ENCRYPT ------------------------------# | ||
| 379 | movups ($ivp),$inout0 # load iv as initial state | ||
| 380 | cmp \$16,$len | ||
| 381 | mov $rnds_,$rounds | ||
| 382 | jb .Lcbc_enc_tail | ||
| 383 | sub \$16,$len | ||
| 384 | jmp .Lcbc_enc_loop | ||
| 385 | .align 16 | ||
| 386 | .Lcbc_enc_loop: | ||
| 387 | movups ($inp),$inout1 # load input | ||
| 388 | lea 16($inp),$inp | ||
| 389 | pxor $inout1,$inout0 | ||
| 390 | ___ | ||
| 391 | &aesni_generate1("enc",$key,$rounds); | ||
| 392 | $code.=<<___; | ||
| 393 | sub \$16,$len | ||
| 394 | lea 16($out),$out | ||
| 395 | mov $rnds_,$rounds # restore $rounds | ||
| 396 | mov $key_,$key # restore $key | ||
| 397 | movups $inout0,-16($out) # store output | ||
| 398 | jnc .Lcbc_enc_loop | ||
| 399 | add \$16,$len | ||
| 400 | jnz .Lcbc_enc_tail | ||
| 401 | movups $inout0,($ivp) | ||
| 402 | jmp .Lcbc_ret | ||
| 403 | |||
| 404 | .Lcbc_enc_tail: | ||
| 405 | mov $len,%rcx # zaps $key | ||
| 406 | xchg $inp,$out # $inp is %rsi and $out is %rdi now | ||
| 407 | .long 0x9066A4F3 # rep movsb | ||
| 408 | mov \$16,%ecx # zero tail | ||
| 409 | sub $len,%rcx | ||
| 410 | xor %eax,%eax | ||
| 411 | .long 0x9066AAF3 # rep stosb | ||
| 412 | lea -16(%rdi),%rdi # rewind $out by 1 block | ||
| 413 | mov $rnds_,$rounds # restore $rounds | ||
| 414 | mov %rdi,%rsi # $inp and $out are the same | ||
| 415 | mov $key_,$key # restore $key | ||
| 416 | xor $len,$len # len=16 | ||
| 417 | jmp .Lcbc_enc_loop # one more spin | ||
| 418 | #--------------------------- CBC DECRYPT ------------------------------# | ||
| 419 | .align 16 | ||
| 420 | .Lcbc_decrypt: | ||
| 421 | ___ | ||
| 422 | $code.=<<___ if ($win64); | ||
| 423 | lea -0x58(%rsp),%rsp | ||
| 424 | movaps %xmm6,(%rsp) | ||
| 425 | movaps %xmm7,0x10(%rsp) | ||
| 426 | movaps %xmm8,0x20(%rsp) | ||
| 427 | movaps %xmm9,0x30(%rsp) | ||
| 428 | .Lcbc_decrypt_body: | ||
| 429 | ___ | ||
| 430 | $code.=<<___; | ||
| 431 | movups ($ivp),$iv | ||
| 432 | sub \$0x40,$len | ||
| 433 | mov $rnds_,$rounds | ||
| 434 | jbe .Lcbc_dec_tail | ||
| 435 | jmp .Lcbc_dec_loop3 | ||
| 436 | .align 16 | ||
| 437 | .Lcbc_dec_loop3: | ||
| 438 | movups ($inp),$inout0 | ||
| 439 | movups 0x10($inp),$inout1 | ||
| 440 | movups 0x20($inp),$inout2 | ||
| 441 | movaps $inout0,$in0 | ||
| 442 | movaps $inout1,$in1 | ||
| 443 | movaps $inout2,$in2 | ||
| 444 | call _aesni_decrypt3 | ||
| 445 | sub \$0x30,$len | ||
| 446 | lea 0x30($inp),$inp | ||
| 447 | lea 0x30($out),$out | ||
| 448 | pxor $iv,$inout0 | ||
| 449 | pxor $in0,$inout1 | ||
| 450 | movaps $in2,$iv | ||
| 451 | pxor $in1,$inout2 | ||
| 452 | movups $inout0,-0x30($out) | ||
| 453 | mov $rnds_,$rounds # restore $rounds | ||
| 454 | movups $inout1,-0x20($out) | ||
| 455 | mov $key_,$key # restore $key | ||
| 456 | movups $inout2,-0x10($out) | ||
| 457 | ja .Lcbc_dec_loop3 | ||
| 458 | |||
| 459 | .Lcbc_dec_tail: | ||
| 460 | add \$0x40,$len | ||
| 461 | movups $iv,($ivp) | ||
| 462 | jz .Lcbc_dec_ret | ||
| 463 | |||
| 464 | movups ($inp),$inout0 | ||
| 465 | cmp \$0x10,$len | ||
| 466 | movaps $inout0,$in0 | ||
| 467 | jbe .Lcbc_dec_one | ||
| 468 | movups 0x10($inp),$inout1 | ||
| 469 | cmp \$0x20,$len | ||
| 470 | movaps $inout1,$in1 | ||
| 471 | jbe .Lcbc_dec_two | ||
| 472 | movups 0x20($inp),$inout2 | ||
| 473 | cmp \$0x30,$len | ||
| 474 | movaps $inout2,$in2 | ||
| 475 | jbe .Lcbc_dec_three | ||
| 476 | movups 0x30($inp),$inout3 | ||
| 477 | call _aesni_decrypt4 | ||
| 478 | pxor $iv,$inout0 | ||
| 479 | movups 0x30($inp),$iv | ||
| 480 | pxor $in0,$inout1 | ||
| 481 | movups $inout0,($out) | ||
| 482 | pxor $in1,$inout2 | ||
| 483 | movups $inout1,0x10($out) | ||
| 484 | pxor $in2,$inout3 | ||
| 485 | movups $inout2,0x20($out) | ||
| 486 | movaps $inout3,$inout0 | ||
| 487 | lea 0x30($out),$out | ||
| 488 | jmp .Lcbc_dec_tail_collected | ||
| 489 | .align 16 | ||
| 490 | .Lcbc_dec_one: | ||
| 491 | ___ | ||
| 492 | &aesni_generate1("dec",$key,$rounds); | ||
| 493 | $code.=<<___; | ||
| 494 | pxor $iv,$inout0 | ||
| 495 | movaps $in0,$iv | ||
| 496 | jmp .Lcbc_dec_tail_collected | ||
| 497 | .align 16 | ||
| 498 | .Lcbc_dec_two: | ||
| 499 | call _aesni_decrypt3 | ||
| 500 | pxor $iv,$inout0 | ||
| 501 | pxor $in0,$inout1 | ||
| 502 | movups $inout0,($out) | ||
| 503 | movaps $in1,$iv | ||
| 504 | movaps $inout1,$inout0 | ||
| 505 | lea 0x10($out),$out | ||
| 506 | jmp .Lcbc_dec_tail_collected | ||
| 507 | .align 16 | ||
| 508 | .Lcbc_dec_three: | ||
| 509 | call _aesni_decrypt3 | ||
| 510 | pxor $iv,$inout0 | ||
| 511 | pxor $in0,$inout1 | ||
| 512 | movups $inout0,($out) | ||
| 513 | pxor $in1,$inout2 | ||
| 514 | movups $inout1,0x10($out) | ||
| 515 | movaps $in2,$iv | ||
| 516 | movaps $inout2,$inout0 | ||
| 517 | lea 0x20($out),$out | ||
| 518 | jmp .Lcbc_dec_tail_collected | ||
| 519 | .align 16 | ||
| 520 | .Lcbc_dec_tail_collected: | ||
| 521 | and \$15,$len | ||
| 522 | movups $iv,($ivp) | ||
| 523 | jnz .Lcbc_dec_tail_partial | ||
| 524 | movups $inout0,($out) | ||
| 525 | jmp .Lcbc_dec_ret | ||
| 526 | .Lcbc_dec_tail_partial: | ||
| 527 | movaps $inout0,$reserved(%rsp) | ||
| 528 | mov $out,%rdi | ||
| 529 | mov $len,%rcx | ||
| 530 | lea $reserved(%rsp),%rsi | ||
| 531 | .long 0x9066A4F3 # rep movsb | ||
| 532 | |||
| 533 | .Lcbc_dec_ret: | ||
| 534 | ___ | ||
| 535 | $code.=<<___ if ($win64); | ||
| 536 | movaps (%rsp),%xmm6 | ||
| 537 | movaps 0x10(%rsp),%xmm7 | ||
| 538 | movaps 0x20(%rsp),%xmm8 | ||
| 539 | movaps 0x30(%rsp),%xmm9 | ||
| 540 | lea 0x58(%rsp),%rsp | ||
| 541 | ___ | ||
| 542 | $code.=<<___; | ||
| 543 | .Lcbc_ret: | ||
| 544 | ret | ||
| 545 | .size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt | ||
| 546 | ___ | ||
| 547 | |||
| 548 | # int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey, | ||
| 549 | # int bits, AES_KEY *key) | ||
| 550 | { my ($inp,$bits,$key) = @_4args; | ||
| 551 | $bits =~ s/%r/%e/; | ||
| 552 | |||
| 553 | $code.=<<___; | ||
| 554 | .globl ${PREFIX}_set_decrypt_key | ||
| 555 | .type ${PREFIX}_set_decrypt_key,\@abi-omnipotent | ||
| 556 | .align 16 | ||
| 557 | ${PREFIX}_set_decrypt_key: | ||
| 558 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
| 559 | call _aesni_set_encrypt_key | ||
| 560 | shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key | ||
| 561 | test %eax,%eax | ||
| 562 | jnz .Ldec_key_ret | ||
| 563 | lea 16($key,$bits),$inp # points at the end of key schedule | ||
| 564 | |||
| 565 | $movkey ($key),%xmm0 # just swap | ||
| 566 | $movkey ($inp),%xmm1 | ||
| 567 | $movkey %xmm0,($inp) | ||
| 568 | $movkey %xmm1,($key) | ||
| 569 | lea 16($key),$key | ||
| 570 | lea -16($inp),$inp | ||
| 571 | |||
| 572 | .Ldec_key_inverse: | ||
| 573 | $movkey ($key),%xmm0 # swap and inverse | ||
| 574 | $movkey ($inp),%xmm1 | ||
| 575 | aesimc %xmm0,%xmm0 | ||
| 576 | aesimc %xmm1,%xmm1 | ||
| 577 | lea 16($key),$key | ||
| 578 | lea -16($inp),$inp | ||
| 579 | cmp $key,$inp | ||
| 580 | $movkey %xmm0,16($inp) | ||
| 581 | $movkey %xmm1,-16($key) | ||
| 582 | ja .Ldec_key_inverse | ||
| 583 | |||
| 584 | $movkey ($key),%xmm0 # inverse middle | ||
| 585 | aesimc %xmm0,%xmm0 | ||
| 586 | $movkey %xmm0,($inp) | ||
| 587 | .Ldec_key_ret: | ||
| 588 | add \$8,%rsp | ||
| 589 | ret | ||
| 590 | .LSEH_end_set_decrypt_key: | ||
| 591 | .size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key | ||
| 592 | ___ | ||
| 593 | |||
| 594 | # This is based on submission by | ||
| 595 | # | ||
| 596 | # Huang Ying <ying.huang@intel.com> | ||
| 597 | # Vinodh Gopal <vinodh.gopal@intel.com> | ||
| 598 | # Kahraman Akdemir | ||
| 599 | # | ||
| 600 | # Agressively optimized in respect to aeskeygenassist's critical path | ||
| 601 | # and is contained in %xmm0-5 to meet Win64 ABI requirement. | ||
| 602 | # | ||
| 603 | $code.=<<___; | ||
| 604 | .globl ${PREFIX}_set_encrypt_key | ||
| 605 | .type ${PREFIX}_set_encrypt_key,\@abi-omnipotent | ||
| 606 | .align 16 | ||
| 607 | ${PREFIX}_set_encrypt_key: | ||
| 608 | _aesni_set_encrypt_key: | ||
| 609 | .byte 0x48,0x83,0xEC,0x08 # sub rsp,8 | ||
| 610 | test $inp,$inp | ||
| 611 | mov \$-1,%rax | ||
| 612 | jz .Lenc_key_ret | ||
| 613 | test $key,$key | ||
| 614 | jz .Lenc_key_ret | ||
| 615 | |||
| 616 | movups ($inp),%xmm0 # pull first 128 bits of *userKey | ||
| 617 | pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0 | ||
| 618 | lea 16($key),%rax | ||
| 619 | cmp \$256,$bits | ||
| 620 | je .L14rounds | ||
| 621 | cmp \$192,$bits | ||
| 622 | je .L12rounds | ||
| 623 | cmp \$128,$bits | ||
| 624 | jne .Lbad_keybits | ||
| 625 | |||
| 626 | .L10rounds: | ||
| 627 | mov \$9,$bits # 10 rounds for 128-bit key | ||
| 628 | $movkey %xmm0,($key) # round 0 | ||
| 629 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 | ||
| 630 | call .Lkey_expansion_128_cold | ||
| 631 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 2 | ||
| 632 | call .Lkey_expansion_128 | ||
| 633 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 3 | ||
| 634 | call .Lkey_expansion_128 | ||
| 635 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 4 | ||
| 636 | call .Lkey_expansion_128 | ||
| 637 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 5 | ||
| 638 | call .Lkey_expansion_128 | ||
| 639 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 6 | ||
| 640 | call .Lkey_expansion_128 | ||
| 641 | aeskeygenassist \$0x40,%xmm0,%xmm1 # round 7 | ||
| 642 | call .Lkey_expansion_128 | ||
| 643 | aeskeygenassist \$0x80,%xmm0,%xmm1 # round 8 | ||
| 644 | call .Lkey_expansion_128 | ||
| 645 | aeskeygenassist \$0x1b,%xmm0,%xmm1 # round 9 | ||
| 646 | call .Lkey_expansion_128 | ||
| 647 | aeskeygenassist \$0x36,%xmm0,%xmm1 # round 10 | ||
| 648 | call .Lkey_expansion_128 | ||
| 649 | $movkey %xmm0,(%rax) | ||
| 650 | mov $bits,80(%rax) # 240(%rdx) | ||
| 651 | xor %eax,%eax | ||
| 652 | jmp .Lenc_key_ret | ||
| 653 | |||
| 654 | .align 16 | ||
| 655 | .L12rounds: | ||
| 656 | movq 16($inp),%xmm2 # remaining 1/3 of *userKey | ||
| 657 | mov \$11,$bits # 12 rounds for 192 | ||
| 658 | $movkey %xmm0,($key) # round 0 | ||
| 659 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 | ||
| 660 | call .Lkey_expansion_192a_cold | ||
| 661 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 2,3 | ||
| 662 | call .Lkey_expansion_192b | ||
| 663 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 4,5 | ||
| 664 | call .Lkey_expansion_192a | ||
| 665 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 5,6 | ||
| 666 | call .Lkey_expansion_192b | ||
| 667 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 7,8 | ||
| 668 | call .Lkey_expansion_192a | ||
| 669 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 8,9 | ||
| 670 | call .Lkey_expansion_192b | ||
| 671 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 10,11 | ||
| 672 | call .Lkey_expansion_192a | ||
| 673 | aeskeygenassist \$0x80,%xmm2,%xmm1 # round 11,12 | ||
| 674 | call .Lkey_expansion_192b | ||
| 675 | $movkey %xmm0,(%rax) | ||
| 676 | mov $bits,48(%rax) # 240(%rdx) | ||
| 677 | xor %rax, %rax | ||
| 678 | jmp .Lenc_key_ret | ||
| 679 | |||
| 680 | .align 16 | ||
| 681 | .L14rounds: | ||
| 682 | movups 16($inp),%xmm2 # remaning half of *userKey | ||
| 683 | mov \$13,$bits # 14 rounds for 256 | ||
| 684 | lea 16(%rax),%rax | ||
| 685 | $movkey %xmm0,($key) # round 0 | ||
| 686 | $movkey %xmm2,16($key) # round 1 | ||
| 687 | aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2 | ||
| 688 | call .Lkey_expansion_256a_cold | ||
| 689 | aeskeygenassist \$0x1,%xmm0,%xmm1 # round 3 | ||
| 690 | call .Lkey_expansion_256b | ||
| 691 | aeskeygenassist \$0x2,%xmm2,%xmm1 # round 4 | ||
| 692 | call .Lkey_expansion_256a | ||
| 693 | aeskeygenassist \$0x2,%xmm0,%xmm1 # round 5 | ||
| 694 | call .Lkey_expansion_256b | ||
| 695 | aeskeygenassist \$0x4,%xmm2,%xmm1 # round 6 | ||
| 696 | call .Lkey_expansion_256a | ||
| 697 | aeskeygenassist \$0x4,%xmm0,%xmm1 # round 7 | ||
| 698 | call .Lkey_expansion_256b | ||
| 699 | aeskeygenassist \$0x8,%xmm2,%xmm1 # round 8 | ||
| 700 | call .Lkey_expansion_256a | ||
| 701 | aeskeygenassist \$0x8,%xmm0,%xmm1 # round 9 | ||
| 702 | call .Lkey_expansion_256b | ||
| 703 | aeskeygenassist \$0x10,%xmm2,%xmm1 # round 10 | ||
| 704 | call .Lkey_expansion_256a | ||
| 705 | aeskeygenassist \$0x10,%xmm0,%xmm1 # round 11 | ||
| 706 | call .Lkey_expansion_256b | ||
| 707 | aeskeygenassist \$0x20,%xmm2,%xmm1 # round 12 | ||
| 708 | call .Lkey_expansion_256a | ||
| 709 | aeskeygenassist \$0x20,%xmm0,%xmm1 # round 13 | ||
| 710 | call .Lkey_expansion_256b | ||
| 711 | aeskeygenassist \$0x40,%xmm2,%xmm1 # round 14 | ||
| 712 | call .Lkey_expansion_256a | ||
| 713 | $movkey %xmm0,(%rax) | ||
| 714 | mov $bits,16(%rax) # 240(%rdx) | ||
| 715 | xor %rax,%rax | ||
| 716 | jmp .Lenc_key_ret | ||
| 717 | |||
| 718 | .align 16 | ||
| 719 | .Lbad_keybits: | ||
| 720 | mov \$-2,%rax | ||
| 721 | .Lenc_key_ret: | ||
| 722 | add \$8,%rsp | ||
| 723 | ret | ||
| 724 | .LSEH_end_set_encrypt_key: | ||
| 725 | |||
| 726 | .align 16 | ||
| 727 | .Lkey_expansion_128: | ||
| 728 | $movkey %xmm0,(%rax) | ||
| 729 | lea 16(%rax),%rax | ||
| 730 | .Lkey_expansion_128_cold: | ||
| 731 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 732 | pxor %xmm4, %xmm0 | ||
| 733 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 734 | pxor %xmm4, %xmm0 | ||
| 735 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
| 736 | pxor %xmm1,%xmm0 | ||
| 737 | ret | ||
| 738 | |||
| 739 | .align 16 | ||
| 740 | .Lkey_expansion_192a: | ||
| 741 | $movkey %xmm0,(%rax) | ||
| 742 | lea 16(%rax),%rax | ||
| 743 | .Lkey_expansion_192a_cold: | ||
| 744 | movaps %xmm2, %xmm5 | ||
| 745 | .Lkey_expansion_192b_warm: | ||
| 746 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 747 | movaps %xmm2,%xmm3 | ||
| 748 | pxor %xmm4,%xmm0 | ||
| 749 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 750 | pslldq \$4,%xmm3 | ||
| 751 | pxor %xmm4,%xmm0 | ||
| 752 | pshufd \$0b01010101,%xmm1,%xmm1 # critical path | ||
| 753 | pxor %xmm3,%xmm2 | ||
| 754 | pxor %xmm1,%xmm0 | ||
| 755 | pshufd \$0b11111111,%xmm0,%xmm3 | ||
| 756 | pxor %xmm3,%xmm2 | ||
| 757 | ret | ||
| 758 | |||
| 759 | .align 16 | ||
| 760 | .Lkey_expansion_192b: | ||
| 761 | movaps %xmm0,%xmm3 | ||
| 762 | shufps \$0b01000100,%xmm0,%xmm5 | ||
| 763 | $movkey %xmm5,(%rax) | ||
| 764 | shufps \$0b01001110,%xmm2,%xmm3 | ||
| 765 | $movkey %xmm3,16(%rax) | ||
| 766 | lea 32(%rax),%rax | ||
| 767 | jmp .Lkey_expansion_192b_warm | ||
| 768 | |||
| 769 | .align 16 | ||
| 770 | .Lkey_expansion_256a: | ||
| 771 | $movkey %xmm2,(%rax) | ||
| 772 | lea 16(%rax),%rax | ||
| 773 | .Lkey_expansion_256a_cold: | ||
| 774 | shufps \$0b00010000,%xmm0,%xmm4 | ||
| 775 | pxor %xmm4,%xmm0 | ||
| 776 | shufps \$0b10001100,%xmm0,%xmm4 | ||
| 777 | pxor %xmm4,%xmm0 | ||
| 778 | pshufd \$0b11111111,%xmm1,%xmm1 # critical path | ||
| 779 | pxor %xmm1,%xmm0 | ||
| 780 | ret | ||
| 781 | |||
| 782 | .align 16 | ||
| 783 | .Lkey_expansion_256b: | ||
| 784 | $movkey %xmm0,(%rax) | ||
| 785 | lea 16(%rax),%rax | ||
| 786 | |||
| 787 | shufps \$0b00010000,%xmm2,%xmm4 | ||
| 788 | pxor %xmm4,%xmm2 | ||
| 789 | shufps \$0b10001100,%xmm2,%xmm4 | ||
| 790 | pxor %xmm4,%xmm2 | ||
| 791 | pshufd \$0b10101010,%xmm1,%xmm1 # critical path | ||
| 792 | pxor %xmm1,%xmm2 | ||
| 793 | ret | ||
| 794 | .size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key | ||
| 795 | ___ | ||
| 796 | } | ||
| 797 | |||
| 798 | $code.=<<___; | ||
| 799 | .asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>" | ||
| 800 | .align 64 | ||
| 801 | ___ | ||
| 802 | |||
| 803 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | ||
| 804 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | ||
| 805 | if ($win64) { | ||
| 806 | $rec="%rcx"; | ||
| 807 | $frame="%rdx"; | ||
| 808 | $context="%r8"; | ||
| 809 | $disp="%r9"; | ||
| 810 | |||
| 811 | $code.=<<___; | ||
| 812 | .extern __imp_RtlVirtualUnwind | ||
| 813 | .type cbc_se_handler,\@abi-omnipotent | ||
| 814 | .align 16 | ||
| 815 | cbc_se_handler: | ||
| 816 | push %rsi | ||
| 817 | push %rdi | ||
| 818 | push %rbx | ||
| 819 | push %rbp | ||
| 820 | push %r12 | ||
| 821 | push %r13 | ||
| 822 | push %r14 | ||
| 823 | push %r15 | ||
| 824 | pushfq | ||
| 825 | sub \$64,%rsp | ||
| 826 | |||
| 827 | mov 152($context),%rax # pull context->Rsp | ||
| 828 | mov 248($context),%rbx # pull context->Rip | ||
| 829 | |||
| 830 | lea .Lcbc_decrypt(%rip),%r10 | ||
| 831 | cmp %r10,%rbx # context->Rip<"prologue" label | ||
| 832 | jb .Lin_prologue | ||
| 833 | |||
| 834 | lea .Lcbc_decrypt_body(%rip),%r10 | ||
| 835 | cmp %r10,%rbx # context->Rip<cbc_decrypt_body | ||
| 836 | jb .Lrestore_rax | ||
| 837 | |||
| 838 | lea .Lcbc_ret(%rip),%r10 | ||
| 839 | cmp %r10,%rbx # context->Rip>="epilogue" label | ||
| 840 | jae .Lin_prologue | ||
| 841 | |||
| 842 | lea 0(%rax),%rsi # top of stack | ||
| 843 | lea 512($context),%rdi # &context.Xmm6 | ||
| 844 | mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax) | ||
| 845 | .long 0xa548f3fc # cld; rep movsq | ||
| 846 | lea 0x58(%rax),%rax # adjust stack pointer | ||
| 847 | jmp .Lin_prologue | ||
| 848 | |||
| 849 | .Lrestore_rax: | ||
| 850 | mov 120($context),%rax | ||
| 851 | .Lin_prologue: | ||
| 852 | mov 8(%rax),%rdi | ||
| 853 | mov 16(%rax),%rsi | ||
| 854 | mov %rax,152($context) # restore context->Rsp | ||
| 855 | mov %rsi,168($context) # restore context->Rsi | ||
| 856 | mov %rdi,176($context) # restore context->Rdi | ||
| 857 | |||
| 858 | jmp .Lcommon_seh_exit | ||
| 859 | .size cbc_se_handler,.-cbc_se_handler | ||
| 860 | |||
| 861 | .type ecb_se_handler,\@abi-omnipotent | ||
| 862 | .align 16 | ||
| 863 | ecb_se_handler: | ||
| 864 | push %rsi | ||
| 865 | push %rdi | ||
| 866 | push %rbx | ||
| 867 | push %rbp | ||
| 868 | push %r12 | ||
| 869 | push %r13 | ||
| 870 | push %r14 | ||
| 871 | push %r15 | ||
| 872 | pushfq | ||
| 873 | sub \$64,%rsp | ||
| 874 | |||
| 875 | mov 152($context),%rax # pull context->Rsp | ||
| 876 | mov 8(%rax),%rdi | ||
| 877 | mov 16(%rax),%rsi | ||
| 878 | mov %rsi,168($context) # restore context->Rsi | ||
| 879 | mov %rdi,176($context) # restore context->Rdi | ||
| 880 | |||
| 881 | .Lcommon_seh_exit: | ||
| 882 | |||
| 883 | mov 40($disp),%rdi # disp->ContextRecord | ||
| 884 | mov $context,%rsi # context | ||
| 885 | mov \$154,%ecx # sizeof(CONTEXT) | ||
| 886 | .long 0xa548f3fc # cld; rep movsq | ||
| 887 | |||
| 888 | mov $disp,%rsi | ||
| 889 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | ||
| 890 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | ||
| 891 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | ||
| 892 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | ||
| 893 | mov 40(%rsi),%r10 # disp->ContextRecord | ||
| 894 | lea 56(%rsi),%r11 # &disp->HandlerData | ||
| 895 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | ||
| 896 | mov %r10,32(%rsp) # arg5 | ||
| 897 | mov %r11,40(%rsp) # arg6 | ||
| 898 | mov %r12,48(%rsp) # arg7 | ||
| 899 | mov %rcx,56(%rsp) # arg8, (NULL) | ||
| 900 | call *__imp_RtlVirtualUnwind(%rip) | ||
| 901 | |||
| 902 | mov \$1,%eax # ExceptionContinueSearch | ||
| 903 | add \$64,%rsp | ||
| 904 | popfq | ||
| 905 | pop %r15 | ||
| 906 | pop %r14 | ||
| 907 | pop %r13 | ||
| 908 | pop %r12 | ||
| 909 | pop %rbp | ||
| 910 | pop %rbx | ||
| 911 | pop %rdi | ||
| 912 | pop %rsi | ||
| 913 | ret | ||
| 914 | .size cbc_se_handler,.-cbc_se_handler | ||
| 915 | |||
| 916 | .section .pdata | ||
| 917 | .align 4 | ||
| 918 | .rva .LSEH_begin_${PREFIX}_ecb_encrypt | ||
| 919 | .rva .LSEH_end_${PREFIX}_ecb_encrypt | ||
| 920 | .rva .LSEH_info_ecb | ||
| 921 | |||
| 922 | .rva .LSEH_begin_${PREFIX}_cbc_encrypt | ||
| 923 | .rva .LSEH_end_${PREFIX}_cbc_encrypt | ||
| 924 | .rva .LSEH_info_cbc | ||
| 925 | |||
| 926 | .rva ${PREFIX}_set_decrypt_key | ||
| 927 | .rva .LSEH_end_set_decrypt_key | ||
| 928 | .rva .LSEH_info_key | ||
| 929 | |||
| 930 | .rva ${PREFIX}_set_encrypt_key | ||
| 931 | .rva .LSEH_end_set_encrypt_key | ||
| 932 | .rva .LSEH_info_key | ||
| 933 | .section .xdata | ||
| 934 | .align 8 | ||
| 935 | .LSEH_info_ecb: | ||
| 936 | .byte 9,0,0,0 | ||
| 937 | .rva ecb_se_handler | ||
| 938 | .LSEH_info_cbc: | ||
| 939 | .byte 9,0,0,0 | ||
| 940 | .rva cbc_se_handler | ||
| 941 | .LSEH_info_key: | ||
| 942 | .byte 0x01,0x04,0x01,0x00 | ||
| 943 | .byte 0x04,0x02,0x00,0x00 | ||
| 944 | ___ | ||
| 945 | } | ||
| 946 | |||
| 947 | sub rex { | ||
| 948 | local *opcode=shift; | ||
| 949 | my ($dst,$src)=@_; | ||
| 950 | |||
| 951 | if ($dst>=8 || $src>=8) { | ||
| 952 | $rex=0x40; | ||
| 953 | $rex|=0x04 if($dst>=8); | ||
| 954 | $rex|=0x01 if($src>=8); | ||
| 955 | push @opcode,$rex; | ||
| 956 | } | ||
| 957 | } | ||
| 958 | |||
| 959 | sub aesni { | ||
| 960 | my $line=shift; | ||
| 961 | my @opcode=(0x66); | ||
| 962 | |||
| 963 | if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
| 964 | rex(\@opcode,$4,$3); | ||
| 965 | push @opcode,0x0f,0x3a,0xdf; | ||
| 966 | push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M | ||
| 967 | my $c=$2; | ||
| 968 | push @opcode,$c=~/^0/?oct($c):$c; | ||
| 969 | return ".byte\t".join(',',@opcode); | ||
| 970 | } | ||
| 971 | elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { | ||
| 972 | my %opcodelet = ( | ||
| 973 | "aesimc" => 0xdb, | ||
| 974 | "aesenc" => 0xdc, "aesenclast" => 0xdd, | ||
| 975 | "aesdec" => 0xde, "aesdeclast" => 0xdf | ||
| 976 | ); | ||
| 977 | return undef if (!defined($opcodelet{$1})); | ||
| 978 | rex(\@opcode,$3,$2); | ||
| 979 | push @opcode,0x0f,0x38,$opcodelet{$1}; | ||
| 980 | push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M | ||
| 981 | return ".byte\t".join(',',@opcode); | ||
| 982 | } | ||
| 983 | return $line; | ||
| 984 | } | ||
| 985 | |||
| 986 | $code =~ s/\`([^\`]*)\`/eval($1)/gem; | ||
| 987 | $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; | ||
| 988 | |||
| 989 | print $code; | ||
| 990 | |||
| 991 | close STDOUT; | ||
| 992 | |||
diff --git a/src/lib/libssl/src/crypto/crypto-lib.com b/src/lib/libssl/src/crypto/crypto-lib.com index e72af90822..8f1c1d5a80 100644 --- a/src/lib/libssl/src/crypto/crypto-lib.com +++ b/src/lib/libssl/src/crypto/crypto-lib.com | |||
| @@ -209,7 +209,8 @@ $ LIB_ENGINE = "eng_err,eng_lib,eng_list,eng_init,eng_ctrl,"+ - | |||
| 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - | 209 | "eng_table,eng_pkey,eng_fat,eng_all,"+ - |
| 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - | 210 | "tb_rsa,tb_dsa,tb_ecdsa,tb_dh,tb_ecdh,tb_rand,tb_store,"+ - |
| 211 | "tb_cipher,tb_digest,"+ - | 211 | "tb_cipher,tb_digest,"+ - |
| 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock" | 212 | "eng_openssl,eng_dyn,eng_cnf,eng_cryptodev,eng_padlock"+ - |
| 213 | "eng_aesni" | ||
| 213 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - | 214 | $ LIB_AES = "aes_core,aes_misc,aes_ecb,aes_cbc,aes_cfb,aes_ofb,"+ - |
| 214 | "aes_ctr,aes_ige,aes_wrap" | 215 | "aes_ctr,aes_ige,aes_wrap" |
| 215 | $ LIB_BUFFER = "buffer,buf_str,buf_err" | 216 | $ LIB_BUFFER = "buffer,buf_str,buf_err" |
diff --git a/src/lib/libssl/src/crypto/engine/Makefile b/src/lib/libssl/src/crypto/engine/Makefile index 0cc3722089..0736153075 100644 --- a/src/lib/libssl/src/crypto/engine/Makefile +++ b/src/lib/libssl/src/crypto/engine/Makefile | |||
| @@ -21,12 +21,14 @@ LIBSRC= eng_err.c eng_lib.c eng_list.c eng_init.c eng_ctrl.c \ | |||
| 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ | 21 | eng_table.c eng_pkey.c eng_fat.c eng_all.c \ |
| 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ | 22 | tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ |
| 23 | tb_cipher.c tb_digest.c \ | 23 | tb_cipher.c tb_digest.c \ |
| 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c | 24 | eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c eng_padlock.c \ |
| 25 | eng_aesni.c | ||
| 25 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ | 26 | LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ |
| 26 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ | 27 | eng_table.o eng_pkey.o eng_fat.o eng_all.o \ |
| 27 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ | 28 | tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ |
| 28 | tb_cipher.o tb_digest.o \ | 29 | tb_cipher.o tb_digest.o \ |
| 29 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o | 30 | eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o eng_padlock.o \ |
| 31 | eng_aesni.o | ||
| 30 | 32 | ||
| 31 | SRC= $(LIBSRC) | 33 | SRC= $(LIBSRC) |
| 32 | 34 | ||
| @@ -82,6 +84,21 @@ clean: | |||
| 82 | 84 | ||
| 83 | # DO NOT DELETE THIS LINE -- make depend depends on it. | 85 | # DO NOT DELETE THIS LINE -- make depend depends on it. |
| 84 | 86 | ||
| 87 | eng_aesni.o: ../../e_os.h ../../include/openssl/aes.h | ||
| 88 | eng_aesni.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h | ||
| 89 | eng_aesni.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h | ||
| 90 | eng_aesni.o: ../../include/openssl/dso.h ../../include/openssl/e_os2.h | ||
| 91 | eng_aesni.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h | ||
| 92 | eng_aesni.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h | ||
| 93 | eng_aesni.o: ../../include/openssl/err.h ../../include/openssl/evp.h | ||
| 94 | eng_aesni.o: ../../include/openssl/lhash.h ../../include/openssl/modes.h | ||
| 95 | eng_aesni.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h | ||
| 96 | eng_aesni.o: ../../include/openssl/opensslconf.h | ||
| 97 | eng_aesni.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h | ||
| 98 | eng_aesni.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h | ||
| 99 | eng_aesni.o: ../../include/openssl/sha.h ../../include/openssl/stack.h | ||
| 100 | eng_aesni.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h | ||
| 101 | eng_aesni.o: ../../include/openssl/x509_vfy.h ../cryptlib.h eng_aesni.c | ||
| 85 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h | 102 | eng_all.o: ../../e_os.h ../../include/openssl/asn1.h |
| 86 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h | 103 | eng_all.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h |
| 87 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h | 104 | eng_all.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h |
diff --git a/src/lib/libssl/src/crypto/engine/eng_aesni.c b/src/lib/libssl/src/crypto/engine/eng_aesni.c new file mode 100644 index 0000000000..eeded1eefa --- /dev/null +++ b/src/lib/libssl/src/crypto/engine/eng_aesni.c | |||
| @@ -0,0 +1,570 @@ | |||
| 1 | /* | ||
| 2 | * Support for Intel AES-NI intruction set | ||
| 3 | * Author: Huang Ying <ying.huang@intel.com> | ||
| 4 | * | ||
| 5 | * Intel AES-NI is a new set of Single Instruction Multiple Data | ||
| 6 | * (SIMD) instructions that are going to be introduced in the next | ||
| 7 | * generation of Intel processor, as of 2009. These instructions | ||
| 8 | * enable fast and secure data encryption and decryption, using the | ||
| 9 | * Advanced Encryption Standard (AES), defined by FIPS Publication | ||
| 10 | * number 197. The architecture introduces six instructions that | ||
| 11 | * offer full hardware support for AES. Four of them support high | ||
| 12 | * performance data encryption and decryption, and the other two | ||
| 13 | * instructions support the AES key expansion procedure. | ||
| 14 | * | ||
| 15 | * The white paper can be downloaded from: | ||
| 16 | * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf | ||
| 17 | * | ||
| 18 | * This file is based on engines/e_padlock.c | ||
| 19 | */ | ||
| 20 | |||
| 21 | /* ==================================================================== | ||
| 22 | * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved. | ||
| 23 | * | ||
| 24 | * Redistribution and use in source and binary forms, with or without | ||
| 25 | * modification, are permitted provided that the following conditions | ||
| 26 | * are met: | ||
| 27 | * | ||
| 28 | * 1. Redistributions of source code must retain the above copyright | ||
| 29 | * notice, this list of conditions and the following disclaimer. | ||
| 30 | * | ||
| 31 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 32 | * notice, this list of conditions and the following disclaimer in | ||
| 33 | * the documentation and/or other materials provided with the | ||
| 34 | * distribution. | ||
| 35 | * | ||
| 36 | * 3. All advertising materials mentioning features or use of this | ||
| 37 | * software must display the following acknowledgment: | ||
| 38 | * "This product includes software developed by the OpenSSL Project | ||
| 39 | * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" | ||
| 40 | * | ||
| 41 | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to | ||
| 42 | * endorse or promote products derived from this software without | ||
| 43 | * prior written permission. For written permission, please contact | ||
| 44 | * licensing@OpenSSL.org. | ||
| 45 | * | ||
| 46 | * 5. Products derived from this software may not be called "OpenSSL" | ||
| 47 | * nor may "OpenSSL" appear in their names without prior written | ||
| 48 | * permission of the OpenSSL Project. | ||
| 49 | * | ||
| 50 | * 6. Redistributions of any form whatsoever must retain the following | ||
| 51 | * acknowledgment: | ||
| 52 | * "This product includes software developed by the OpenSSL Project | ||
| 53 | * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" | ||
| 54 | * | ||
| 55 | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY | ||
| 56 | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| 57 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 58 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR | ||
| 59 | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
| 60 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | ||
| 61 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | ||
| 62 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||
| 63 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | ||
| 64 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| 65 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED | ||
| 66 | * OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 67 | * ==================================================================== | ||
| 68 | * | ||
| 69 | * This product includes cryptographic software written by Eric Young | ||
| 70 | * (eay@cryptsoft.com). This product includes software written by Tim | ||
| 71 | * Hudson (tjh@cryptsoft.com). | ||
| 72 | * | ||
| 73 | */ | ||
| 74 | |||
| 75 | |||
| 76 | #include <openssl/opensslconf.h> | ||
| 77 | |||
| 78 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AES_NI) && !defined(OPENSSL_NO_AES) | ||
| 79 | |||
| 80 | #include <stdio.h> | ||
| 81 | #include <assert.h> | ||
| 82 | #include "cryptlib.h" | ||
| 83 | #include <openssl/dso.h> | ||
| 84 | #include <openssl/engine.h> | ||
| 85 | #include <openssl/evp.h> | ||
| 86 | #include <openssl/aes.h> | ||
| 87 | #include <openssl/err.h> | ||
| 88 | |||
| 89 | /* AES-NI is available *ONLY* on some x86 CPUs. Not only that it | ||
| 90 | doesn't exist elsewhere, but it even can't be compiled on other | ||
| 91 | platforms! */ | ||
| 92 | #undef COMPILE_HW_AESNI | ||
| 93 | #if (defined(__x86_64) || defined(__x86_64__) || \ | ||
| 94 | defined(_M_AMD64) || defined(_M_X64) || \ | ||
| 95 | defined(OPENSSL_IA32_SSE2)) && !defined(OPENSSL_NO_ASM) && !defined(__i386__) | ||
| 96 | #define COMPILE_HW_AESNI | ||
| 97 | #endif | ||
| 98 | static ENGINE *ENGINE_aesni (void); | ||
| 99 | |||
| 100 | void ENGINE_load_aesni (void) | ||
| 101 | { | ||
| 102 | /* On non-x86 CPUs it just returns. */ | ||
| 103 | #ifdef COMPILE_HW_AESNI | ||
| 104 | ENGINE *toadd = ENGINE_aesni(); | ||
| 105 | if (!toadd) return; | ||
| 106 | ENGINE_add (toadd); | ||
| 107 | ENGINE_register_complete (toadd); | ||
| 108 | ENGINE_free (toadd); | ||
| 109 | ERR_clear_error (); | ||
| 110 | #endif | ||
| 111 | } | ||
| 112 | |||
| 113 | #ifdef COMPILE_HW_AESNI | ||
| 114 | int aesni_set_encrypt_key(const unsigned char *userKey, int bits, | ||
| 115 | AES_KEY *key); | ||
| 116 | int aesni_set_decrypt_key(const unsigned char *userKey, int bits, | ||
| 117 | AES_KEY *key); | ||
| 118 | |||
| 119 | void aesni_encrypt(const unsigned char *in, unsigned char *out, | ||
| 120 | const AES_KEY *key); | ||
| 121 | void aesni_decrypt(const unsigned char *in, unsigned char *out, | ||
| 122 | const AES_KEY *key); | ||
| 123 | |||
| 124 | void aesni_ecb_encrypt(const unsigned char *in, | ||
| 125 | unsigned char *out, | ||
| 126 | size_t length, | ||
| 127 | const AES_KEY *key, | ||
| 128 | int enc); | ||
| 129 | void aesni_cbc_encrypt(const unsigned char *in, | ||
| 130 | unsigned char *out, | ||
| 131 | size_t length, | ||
| 132 | const AES_KEY *key, | ||
| 133 | unsigned char *ivec, int enc); | ||
| 134 | |||
| 135 | /* Function for ENGINE detection and control */ | ||
| 136 | static int aesni_init(ENGINE *e); | ||
| 137 | |||
| 138 | /* Cipher Stuff */ | ||
| 139 | static int aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | ||
| 140 | const int **nids, int nid); | ||
| 141 | |||
| 142 | #define AESNI_MIN_ALIGN 16 | ||
| 143 | #define AESNI_ALIGN(x) \ | ||
| 144 | ((void *)(((unsigned long)(x)+AESNI_MIN_ALIGN-1)&~(AESNI_MIN_ALIGN-1))) | ||
| 145 | |||
| 146 | /* Engine names */ | ||
| 147 | static const char aesni_id[] = "aesni", | ||
| 148 | aesni_name[] = "Intel AES-NI engine", | ||
| 149 | no_aesni_name[] = "Intel AES-NI engine (no-aesni)"; | ||
| 150 | |||
| 151 | |||
| 152 | /* The input and output encrypted as though 128bit cfb mode is being | ||
| 153 | * used. The extra state information to record how much of the | ||
| 154 | * 128bit block we have used is contained in *num; | ||
| 155 | */ | ||
| 156 | static void aesni_cfb128_encrypt(const unsigned char *in, unsigned char *out, | ||
| 157 | unsigned int len, const void *key, | ||
| 158 | unsigned char ivec[16], int *num, | ||
| 159 | int enc) | ||
| 160 | { | ||
| 161 | unsigned int n; | ||
| 162 | size_t l = 0; | ||
| 163 | |||
| 164 | assert(in && out && key && ivec && num); | ||
| 165 | |||
| 166 | n = *num; | ||
| 167 | |||
| 168 | if (enc) { | ||
| 169 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 170 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 171 | while (n && len) { | ||
| 172 | *(out++) = ivec[n] ^= *(in++); | ||
| 173 | --len; | ||
| 174 | n = (n+1) % 16; | ||
| 175 | } | ||
| 176 | while (len>=16) { | ||
| 177 | aesni_encrypt(ivec, ivec, key); | ||
| 178 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
| 179 | *(size_t*)(out+n) = | ||
| 180 | *(size_t*)(ivec+n) ^= *(size_t*)(in+n); | ||
| 181 | } | ||
| 182 | len -= 16; | ||
| 183 | out += 16; | ||
| 184 | in += 16; | ||
| 185 | } | ||
| 186 | n = 0; | ||
| 187 | if (len) { | ||
| 188 | aesni_encrypt(ivec, ivec, key); | ||
| 189 | while (len--) { | ||
| 190 | out[n] = ivec[n] ^= in[n]; | ||
| 191 | ++n; | ||
| 192 | } | ||
| 193 | } | ||
| 194 | *num = n; | ||
| 195 | return; | ||
| 196 | } while (0); | ||
| 197 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 198 | #endif | ||
| 199 | while (l<len) { | ||
| 200 | if (n == 0) { | ||
| 201 | aesni_encrypt(ivec, ivec, key); | ||
| 202 | } | ||
| 203 | out[l] = ivec[n] ^= in[l]; | ||
| 204 | ++l; | ||
| 205 | n = (n+1) % 16; | ||
| 206 | } | ||
| 207 | *num = n; | ||
| 208 | } else { | ||
| 209 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 210 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 211 | while (n && len) { | ||
| 212 | unsigned char c; | ||
| 213 | *(out++) = ivec[n] ^ (c = *(in++)); ivec[n] = c; | ||
| 214 | --len; | ||
| 215 | n = (n+1) % 16; | ||
| 216 | } | ||
| 217 | while (len>=16) { | ||
| 218 | aesni_encrypt(ivec, ivec, key); | ||
| 219 | for (n=0; n<16; n+=sizeof(size_t)) { | ||
| 220 | size_t t = *(size_t*)(in+n); | ||
| 221 | *(size_t*)(out+n) = *(size_t*)(ivec+n) ^ t; | ||
| 222 | *(size_t*)(ivec+n) = t; | ||
| 223 | } | ||
| 224 | len -= 16; | ||
| 225 | out += 16; | ||
| 226 | in += 16; | ||
| 227 | } | ||
| 228 | n = 0; | ||
| 229 | if (len) { | ||
| 230 | aesni_encrypt(ivec, ivec, key); | ||
| 231 | while (len--) { | ||
| 232 | unsigned char c; | ||
| 233 | out[n] = ivec[n] ^ (c = in[n]); ivec[n] = c; | ||
| 234 | ++n; | ||
| 235 | } | ||
| 236 | } | ||
| 237 | *num = n; | ||
| 238 | return; | ||
| 239 | } while (0); | ||
| 240 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 241 | #endif | ||
| 242 | while (l<len) { | ||
| 243 | unsigned char c; | ||
| 244 | if (n == 0) { | ||
| 245 | aesni_encrypt(ivec, ivec, key); | ||
| 246 | } | ||
| 247 | out[l] = ivec[n] ^ (c = in[l]); ivec[n] = c; | ||
| 248 | ++l; | ||
| 249 | n = (n+1) % 16; | ||
| 250 | } | ||
| 251 | *num=n; | ||
| 252 | } | ||
| 253 | } | ||
| 254 | |||
| 255 | /* The input and output encrypted as though 128bit ofb mode is being | ||
| 256 | * used. The extra state information to record how much of the | ||
| 257 | * 128bit block we have used is contained in *num; | ||
| 258 | */ | ||
| 259 | static void aesni_ofb128_encrypt(const unsigned char *in, unsigned char *out, | ||
| 260 | unsigned int len, const void *key, | ||
| 261 | unsigned char ivec[16], int *num) | ||
| 262 | { | ||
| 263 | unsigned int n; | ||
| 264 | size_t l=0; | ||
| 265 | |||
| 266 | assert(in && out && key && ivec && num); | ||
| 267 | |||
| 268 | n = *num; | ||
| 269 | |||
| 270 | #if !defined(OPENSSL_SMALL_FOOTPRINT) | ||
| 271 | if (16%sizeof(size_t) == 0) do { /* always true actually */ | ||
| 272 | while (n && len) { | ||
| 273 | *(out++) = *(in++) ^ ivec[n]; | ||
| 274 | --len; | ||
| 275 | n = (n+1) % 16; | ||
| 276 | } | ||
| 277 | while (len>=16) { | ||
| 278 | aesni_encrypt(ivec, ivec, key); | ||
| 279 | for (n=0; n<16; n+=sizeof(size_t)) | ||
| 280 | *(size_t*)(out+n) = | ||
| 281 | *(size_t*)(in+n) ^ *(size_t*)(ivec+n); | ||
| 282 | len -= 16; | ||
| 283 | out += 16; | ||
| 284 | in += 16; | ||
| 285 | } | ||
| 286 | n = 0; | ||
| 287 | if (len) { | ||
| 288 | aesni_encrypt(ivec, ivec, key); | ||
| 289 | while (len--) { | ||
| 290 | out[n] = in[n] ^ ivec[n]; | ||
| 291 | ++n; | ||
| 292 | } | ||
| 293 | } | ||
| 294 | *num = n; | ||
| 295 | return; | ||
| 296 | } while(0); | ||
| 297 | /* the rest would be commonly eliminated by x86* compiler */ | ||
| 298 | #endif | ||
| 299 | while (l<len) { | ||
| 300 | if (n==0) { | ||
| 301 | aesni_encrypt(ivec, ivec, key); | ||
| 302 | } | ||
| 303 | out[l] = in[l] ^ ivec[n]; | ||
| 304 | ++l; | ||
| 305 | n = (n+1) % 16; | ||
| 306 | } | ||
| 307 | |||
| 308 | *num=n; | ||
| 309 | } | ||
| 310 | /* ===== Engine "management" functions ===== */ | ||
| 311 | |||
| 312 | #if defined(_WIN32) | ||
| 313 | typedef unsigned __int64 IA32CAP; | ||
| 314 | #else | ||
| 315 | typedef unsigned long long IA32CAP; | ||
| 316 | #endif | ||
| 317 | |||
| 318 | /* Prepare the ENGINE structure for registration */ | ||
| 319 | static int | ||
| 320 | aesni_bind_helper(ENGINE *e) | ||
| 321 | { | ||
| 322 | int engage; | ||
| 323 | if (sizeof(OPENSSL_ia32cap_P) > 4) { | ||
| 324 | engage = ((IA32CAP)OPENSSL_ia32cap_P >> 57) & 1; | ||
| 325 | } else { | ||
| 326 | IA32CAP OPENSSL_ia32_cpuid(void); | ||
| 327 | engage = (OPENSSL_ia32_cpuid() >> 57) & 1; | ||
| 328 | } | ||
| 329 | |||
| 330 | /* Register everything or return with an error */ | ||
| 331 | if (!ENGINE_set_id(e, aesni_id) || | ||
| 332 | !ENGINE_set_name(e, engage ? aesni_name : no_aesni_name) || | ||
| 333 | |||
| 334 | !ENGINE_set_init_function(e, aesni_init) || | ||
| 335 | (engage && !ENGINE_set_ciphers (e, aesni_ciphers)) | ||
| 336 | ) | ||
| 337 | return 0; | ||
| 338 | |||
| 339 | /* Everything looks good */ | ||
| 340 | return 1; | ||
| 341 | } | ||
| 342 | |||
| 343 | /* Constructor */ | ||
| 344 | static ENGINE * | ||
| 345 | ENGINE_aesni(void) | ||
| 346 | { | ||
| 347 | ENGINE *eng = ENGINE_new(); | ||
| 348 | |||
| 349 | if (!eng) { | ||
| 350 | return NULL; | ||
| 351 | } | ||
| 352 | |||
| 353 | if (!aesni_bind_helper(eng)) { | ||
| 354 | ENGINE_free(eng); | ||
| 355 | return NULL; | ||
| 356 | } | ||
| 357 | |||
| 358 | return eng; | ||
| 359 | } | ||
| 360 | |||
| 361 | /* Check availability of the engine */ | ||
| 362 | static int | ||
| 363 | aesni_init(ENGINE *e) | ||
| 364 | { | ||
| 365 | return 1; | ||
| 366 | } | ||
| 367 | |||
| 368 | #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb) | ||
| 369 | #define NID_aes_128_cfb NID_aes_128_cfb128 | ||
| 370 | #endif | ||
| 371 | |||
| 372 | #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb) | ||
| 373 | #define NID_aes_128_ofb NID_aes_128_ofb128 | ||
| 374 | #endif | ||
| 375 | |||
| 376 | #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb) | ||
| 377 | #define NID_aes_192_cfb NID_aes_192_cfb128 | ||
| 378 | #endif | ||
| 379 | |||
| 380 | #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb) | ||
| 381 | #define NID_aes_192_ofb NID_aes_192_ofb128 | ||
| 382 | #endif | ||
| 383 | |||
| 384 | #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb) | ||
| 385 | #define NID_aes_256_cfb NID_aes_256_cfb128 | ||
| 386 | #endif | ||
| 387 | |||
| 388 | #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb) | ||
| 389 | #define NID_aes_256_ofb NID_aes_256_ofb128 | ||
| 390 | #endif | ||
| 391 | |||
| 392 | /* List of supported ciphers. */ | ||
| 393 | static int aesni_cipher_nids[] = { | ||
| 394 | NID_aes_128_ecb, | ||
| 395 | NID_aes_128_cbc, | ||
| 396 | NID_aes_128_cfb, | ||
| 397 | NID_aes_128_ofb, | ||
| 398 | |||
| 399 | NID_aes_192_ecb, | ||
| 400 | NID_aes_192_cbc, | ||
| 401 | NID_aes_192_cfb, | ||
| 402 | NID_aes_192_ofb, | ||
| 403 | |||
| 404 | NID_aes_256_ecb, | ||
| 405 | NID_aes_256_cbc, | ||
| 406 | NID_aes_256_cfb, | ||
| 407 | NID_aes_256_ofb, | ||
| 408 | }; | ||
| 409 | static int aesni_cipher_nids_num = | ||
| 410 | (sizeof(aesni_cipher_nids)/sizeof(aesni_cipher_nids[0])); | ||
| 411 | |||
| 412 | typedef struct | ||
| 413 | { | ||
| 414 | AES_KEY ks; | ||
| 415 | unsigned int _pad1[3]; | ||
| 416 | } AESNI_KEY; | ||
| 417 | |||
| 418 | static int | ||
| 419 | aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *user_key, | ||
| 420 | const unsigned char *iv, int enc) | ||
| 421 | { | ||
| 422 | int ret; | ||
| 423 | AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 424 | |||
| 425 | if ((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CFB_MODE | ||
| 426 | || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_OFB_MODE | ||
| 427 | || enc) | ||
| 428 | ret=aesni_set_encrypt_key(user_key, ctx->key_len * 8, key); | ||
| 429 | else | ||
| 430 | ret=aesni_set_decrypt_key(user_key, ctx->key_len * 8, key); | ||
| 431 | |||
| 432 | if(ret < 0) { | ||
| 433 | EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED); | ||
| 434 | return 0; | ||
| 435 | } | ||
| 436 | |||
| 437 | return 1; | ||
| 438 | } | ||
| 439 | |||
| 440 | static int aesni_cipher_ecb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 441 | const unsigned char *in, unsigned int inl) | ||
| 442 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 443 | aesni_ecb_encrypt(in, out, inl, key, ctx->encrypt); | ||
| 444 | return 1; | ||
| 445 | } | ||
| 446 | static int aesni_cipher_cbc(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 447 | const unsigned char *in, unsigned int inl) | ||
| 448 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 449 | aesni_cbc_encrypt(in, out, inl, key, | ||
| 450 | ctx->iv, ctx->encrypt); | ||
| 451 | return 1; | ||
| 452 | } | ||
| 453 | static int aesni_cipher_cfb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 454 | const unsigned char *in, unsigned int inl) | ||
| 455 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 456 | |||
| 457 | aesni_cfb128_encrypt(in, out, inl, key, ctx->iv, | ||
| 458 | &ctx->num, ctx->encrypt); | ||
| 459 | return 1; | ||
| 460 | } | ||
| 461 | static int aesni_cipher_ofb(EVP_CIPHER_CTX *ctx, unsigned char *out, | ||
| 462 | const unsigned char *in, unsigned int inl) | ||
| 463 | { AES_KEY *key = AESNI_ALIGN(ctx->cipher_data); | ||
| 464 | aesni_ofb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num); | ||
| 465 | return 1; | ||
| 466 | } | ||
| 467 | |||
| 468 | #define AES_BLOCK_SIZE 16 | ||
| 469 | |||
| 470 | #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE | ||
| 471 | #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE | ||
| 472 | #define EVP_CIPHER_block_size_OFB 1 | ||
| 473 | #define EVP_CIPHER_block_size_CFB 1 | ||
| 474 | |||
| 475 | /* Declaring so many ciphers by hand would be a pain. | ||
| 476 | Instead introduce a bit of preprocessor magic :-) */ | ||
| 477 | #define DECLARE_AES_EVP(ksize,lmode,umode) \ | ||
| 478 | static const EVP_CIPHER aesni_##ksize##_##lmode = { \ | ||
| 479 | NID_aes_##ksize##_##lmode, \ | ||
| 480 | EVP_CIPHER_block_size_##umode, \ | ||
| 481 | ksize / 8, \ | ||
| 482 | AES_BLOCK_SIZE, \ | ||
| 483 | 0 | EVP_CIPH_##umode##_MODE, \ | ||
| 484 | aesni_init_key, \ | ||
| 485 | aesni_cipher_##lmode, \ | ||
| 486 | NULL, \ | ||
| 487 | sizeof(AESNI_KEY), \ | ||
| 488 | EVP_CIPHER_set_asn1_iv, \ | ||
| 489 | EVP_CIPHER_get_asn1_iv, \ | ||
| 490 | NULL, \ | ||
| 491 | NULL \ | ||
| 492 | } | ||
| 493 | |||
| 494 | DECLARE_AES_EVP(128,ecb,ECB); | ||
| 495 | DECLARE_AES_EVP(128,cbc,CBC); | ||
| 496 | DECLARE_AES_EVP(128,cfb,CFB); | ||
| 497 | DECLARE_AES_EVP(128,ofb,OFB); | ||
| 498 | |||
| 499 | DECLARE_AES_EVP(192,ecb,ECB); | ||
| 500 | DECLARE_AES_EVP(192,cbc,CBC); | ||
| 501 | DECLARE_AES_EVP(192,cfb,CFB); | ||
| 502 | DECLARE_AES_EVP(192,ofb,OFB); | ||
| 503 | |||
| 504 | DECLARE_AES_EVP(256,ecb,ECB); | ||
| 505 | DECLARE_AES_EVP(256,cbc,CBC); | ||
| 506 | DECLARE_AES_EVP(256,cfb,CFB); | ||
| 507 | DECLARE_AES_EVP(256,ofb,OFB); | ||
| 508 | |||
| 509 | static int | ||
| 510 | aesni_ciphers (ENGINE *e, const EVP_CIPHER **cipher, | ||
| 511 | const int **nids, int nid) | ||
| 512 | { | ||
| 513 | /* No specific cipher => return a list of supported nids ... */ | ||
| 514 | if (!cipher) { | ||
| 515 | *nids = aesni_cipher_nids; | ||
| 516 | return aesni_cipher_nids_num; | ||
| 517 | } | ||
| 518 | |||
| 519 | /* ... or the requested "cipher" otherwise */ | ||
| 520 | switch (nid) { | ||
| 521 | case NID_aes_128_ecb: | ||
| 522 | *cipher = &aesni_128_ecb; | ||
| 523 | break; | ||
| 524 | case NID_aes_128_cbc: | ||
| 525 | *cipher = &aesni_128_cbc; | ||
| 526 | break; | ||
| 527 | case NID_aes_128_cfb: | ||
| 528 | *cipher = &aesni_128_cfb; | ||
| 529 | break; | ||
| 530 | case NID_aes_128_ofb: | ||
| 531 | *cipher = &aesni_128_ofb; | ||
| 532 | break; | ||
| 533 | |||
| 534 | case NID_aes_192_ecb: | ||
| 535 | *cipher = &aesni_192_ecb; | ||
| 536 | break; | ||
| 537 | case NID_aes_192_cbc: | ||
| 538 | *cipher = &aesni_192_cbc; | ||
| 539 | break; | ||
| 540 | case NID_aes_192_cfb: | ||
| 541 | *cipher = &aesni_192_cfb; | ||
| 542 | break; | ||
| 543 | case NID_aes_192_ofb: | ||
| 544 | *cipher = &aesni_192_ofb; | ||
| 545 | break; | ||
| 546 | |||
| 547 | case NID_aes_256_ecb: | ||
| 548 | *cipher = &aesni_256_ecb; | ||
| 549 | break; | ||
| 550 | case NID_aes_256_cbc: | ||
| 551 | *cipher = &aesni_256_cbc; | ||
| 552 | break; | ||
| 553 | case NID_aes_256_cfb: | ||
| 554 | *cipher = &aesni_256_cfb; | ||
| 555 | break; | ||
| 556 | case NID_aes_256_ofb: | ||
| 557 | *cipher = &aesni_256_ofb; | ||
| 558 | break; | ||
| 559 | |||
| 560 | default: | ||
| 561 | /* Sorry, we don't support this NID */ | ||
| 562 | *cipher = NULL; | ||
| 563 | return 0; | ||
| 564 | } | ||
| 565 | return 1; | ||
| 566 | } | ||
| 567 | |||
| 568 | #endif /* COMPILE_HW_AESNI */ | ||
| 569 | #endif /* !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) && !defined(OPENSSL_NO_AES) */ | ||
| 570 | |||
diff --git a/src/lib/libssl/src/crypto/engine/eng_all.c b/src/lib/libssl/src/crypto/engine/eng_all.c index d29cd57dc2..69e54f461e 100644 --- a/src/lib/libssl/src/crypto/engine/eng_all.c +++ b/src/lib/libssl/src/crypto/engine/eng_all.c | |||
| @@ -71,6 +71,11 @@ void ENGINE_load_builtin_engines(void) | |||
| 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) | 71 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_PADLOCK) |
| 72 | ENGINE_load_padlock(); | 72 | ENGINE_load_padlock(); |
| 73 | #endif | 73 | #endif |
| 74 | |||
| 75 | #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) | ||
| 76 | ENGINE_load_aesni(); | ||
| 77 | #endif | ||
| 78 | |||
| 74 | ENGINE_load_dynamic(); | 79 | ENGINE_load_dynamic(); |
| 75 | #ifndef OPENSSL_NO_STATIC_ENGINE | 80 | #ifndef OPENSSL_NO_STATIC_ENGINE |
| 76 | #ifndef OPENSSL_NO_HW | 81 | #ifndef OPENSSL_NO_HW |
diff --git a/src/lib/libssl/src/crypto/engine/engine.h b/src/lib/libssl/src/crypto/engine/engine.h index f503595ece..ecab9453ba 100644 --- a/src/lib/libssl/src/crypto/engine/engine.h +++ b/src/lib/libssl/src/crypto/engine/engine.h | |||
| @@ -337,6 +337,7 @@ void ENGINE_load_sureware(void); | |||
| 337 | void ENGINE_load_ubsec(void); | 337 | void ENGINE_load_ubsec(void); |
| 338 | #endif | 338 | #endif |
| 339 | void ENGINE_load_cryptodev(void); | 339 | void ENGINE_load_cryptodev(void); |
| 340 | void ENGINE_load_aesni(void); | ||
| 340 | void ENGINE_load_padlock(void); | 341 | void ENGINE_load_padlock(void); |
| 341 | void ENGINE_load_builtin_engines(void); | 342 | void ENGINE_load_builtin_engines(void); |
| 342 | #ifndef OPENSSL_NO_CAPIENG | 343 | #ifndef OPENSSL_NO_CAPIENG |
diff --git a/src/lib/libssl/src/crypto/evp/evp.h b/src/lib/libssl/src/crypto/evp/evp.h index 51011f2b14..82b5862ce7 100644 --- a/src/lib/libssl/src/crypto/evp/evp.h +++ b/src/lib/libssl/src/crypto/evp/evp.h | |||
| @@ -959,6 +959,7 @@ void ERR_load_EVP_strings(void); | |||
| 959 | /* Error codes for the EVP functions. */ | 959 | /* Error codes for the EVP functions. */ |
| 960 | 960 | ||
| 961 | /* Function codes. */ | 961 | /* Function codes. */ |
| 962 | #define EVP_F_AESNI_INIT_KEY 165 | ||
| 962 | #define EVP_F_AES_INIT_KEY 133 | 963 | #define EVP_F_AES_INIT_KEY 133 |
| 963 | #define EVP_F_ALG_MODULE_INIT 138 | 964 | #define EVP_F_ALG_MODULE_INIT 138 |
| 964 | #define EVP_F_CAMELLIA_INIT_KEY 159 | 965 | #define EVP_F_CAMELLIA_INIT_KEY 159 |
diff --git a/src/lib/libssl/src/crypto/evp/evp_err.c b/src/lib/libssl/src/crypto/evp/evp_err.c index b5b900d4fe..ec2d127cd8 100644 --- a/src/lib/libssl/src/crypto/evp/evp_err.c +++ b/src/lib/libssl/src/crypto/evp/evp_err.c | |||
| @@ -70,6 +70,7 @@ | |||
| 70 | 70 | ||
| 71 | static ERR_STRING_DATA EVP_str_functs[]= | 71 | static ERR_STRING_DATA EVP_str_functs[]= |
| 72 | { | 72 | { |
| 73 | {ERR_FUNC(EVP_F_AESNI_INIT_KEY), "AESNI_INIT_KEY"}, | ||
| 73 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, | 74 | {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, |
| 74 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, | 75 | {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, |
| 75 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, | 76 | {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, |
diff --git a/src/lib/libssl/src/test/test_aesni b/src/lib/libssl/src/test/test_aesni new file mode 100644 index 0000000000..e8fb63ee2b --- /dev/null +++ b/src/lib/libssl/src/test/test_aesni | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | #!/bin/sh | ||
| 2 | |||
| 3 | PROG=$1 | ||
| 4 | |||
| 5 | if [ -x $PROG ]; then | ||
| 6 | if expr "x`$PROG version`" : "xOpenSSL" > /dev/null; then | ||
| 7 | : | ||
| 8 | else | ||
| 9 | echo "$PROG is not OpenSSL executable" | ||
| 10 | exit 1 | ||
| 11 | fi | ||
| 12 | else | ||
| 13 | echo "$PROG is not executable" | ||
| 14 | exit 1; | ||
| 15 | fi | ||
| 16 | |||
| 17 | if $PROG engine aesni | grep -v no-aesni; then | ||
| 18 | |||
| 19 | HASH=`cat $PROG | $PROG dgst -hex` | ||
| 20 | |||
| 21 | AES_ALGS=" aes-128-ecb aes-192-ecb aes-256-ecb \ | ||
| 22 | aes-128-cbc aes-192-cbc aes-256-cbc \ | ||
| 23 | aes-128-cfb aes-192-cfb aes-256-cfb \ | ||
| 24 | aes-128-ofb aes-192-ofb aes-256-ofb" | ||
| 25 | BUFSIZE="16 32 48 64 80 96 128 144 999" | ||
| 26 | |||
| 27 | nerr=0 | ||
| 28 | |||
| 29 | for alg in $AES_ALGS; do | ||
| 30 | echo $alg | ||
| 31 | for bufsize in $BUFSIZE; do | ||
| 32 | TEST=`( cat $PROG | \ | ||
| 33 | $PROG enc -e -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
| 34 | $PROG enc -d -k "$HASH" -$alg | \ | ||
| 35 | $PROG dgst -hex ) 2>/dev/null` | ||
| 36 | if [ "$TEST" != "$HASH" ]; then | ||
| 37 | echo "-$alg/$bufsize encrypt test failed" | ||
| 38 | nerr=`expr $nerr + 1` | ||
| 39 | fi | ||
| 40 | done | ||
| 41 | for bufsize in $BUFSIZE; do | ||
| 42 | TEST=`( cat $PROG | \ | ||
| 43 | $PROG enc -e -k "$HASH" -$alg | \ | ||
| 44 | $PROG enc -d -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
| 45 | $PROG dgst -hex ) 2>/dev/null` | ||
| 46 | if [ "$TEST" != "$HASH" ]; then | ||
| 47 | echo "-$alg/$bufsize decrypt test failed" | ||
| 48 | nerr=`expr $nerr + 1` | ||
| 49 | fi | ||
| 50 | done | ||
| 51 | TEST=`( cat $PROG | \ | ||
| 52 | $PROG enc -e -k "$HASH" -$alg -engine aesni | \ | ||
| 53 | $PROG enc -d -k "$HASH" -$alg -engine aesni | \ | ||
| 54 | $PROG dgst -hex ) 2>/dev/null` | ||
| 55 | if [ "$TEST" != "$HASH" ]; then | ||
| 56 | echo "-$alg en/decrypt test failed" | ||
| 57 | nerr=`expr $nerr + 1` | ||
| 58 | fi | ||
| 59 | done | ||
| 60 | |||
| 61 | if [ $nerr -gt 0 ]; then | ||
| 62 | echo "AESNI engine test failed." | ||
| 63 | exit 1; | ||
| 64 | fi | ||
| 65 | else | ||
| 66 | echo "AESNI engine is not available" | ||
| 67 | fi | ||
| 68 | |||
| 69 | exit 0 | ||
diff --git a/src/lib/libssl/src/util/libeay.num b/src/lib/libssl/src/util/libeay.num index 74eb337227..0bad595233 100644 --- a/src/lib/libssl/src/util/libeay.num +++ b/src/lib/libssl/src/util/libeay.num | |||
| @@ -3725,3 +3725,4 @@ JPAKE_STEP2_release 4110 EXIST::FUNCTION:JPAKE | |||
| 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE | 3725 | JPAKE_STEP3A_init 4111 EXIST::FUNCTION:JPAKE |
| 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE | 3726 | ERR_load_JPAKE_strings 4112 EXIST::FUNCTION:JPAKE |
| 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE | 3727 | JPAKE_STEP2_init 4113 EXIST::FUNCTION:JPAKE |
| 3728 | ENGINE_load_aesni 4114 EXIST::FUNCTION:ENGINE | ||
diff --git a/src/lib/libssl/test/test_aesni b/src/lib/libssl/test/test_aesni new file mode 100644 index 0000000000..e8fb63ee2b --- /dev/null +++ b/src/lib/libssl/test/test_aesni | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | #!/bin/sh | ||
| 2 | |||
| 3 | PROG=$1 | ||
| 4 | |||
| 5 | if [ -x $PROG ]; then | ||
| 6 | if expr "x`$PROG version`" : "xOpenSSL" > /dev/null; then | ||
| 7 | : | ||
| 8 | else | ||
| 9 | echo "$PROG is not OpenSSL executable" | ||
| 10 | exit 1 | ||
| 11 | fi | ||
| 12 | else | ||
| 13 | echo "$PROG is not executable" | ||
| 14 | exit 1; | ||
| 15 | fi | ||
| 16 | |||
| 17 | if $PROG engine aesni | grep -v no-aesni; then | ||
| 18 | |||
| 19 | HASH=`cat $PROG | $PROG dgst -hex` | ||
| 20 | |||
| 21 | AES_ALGS=" aes-128-ecb aes-192-ecb aes-256-ecb \ | ||
| 22 | aes-128-cbc aes-192-cbc aes-256-cbc \ | ||
| 23 | aes-128-cfb aes-192-cfb aes-256-cfb \ | ||
| 24 | aes-128-ofb aes-192-ofb aes-256-ofb" | ||
| 25 | BUFSIZE="16 32 48 64 80 96 128 144 999" | ||
| 26 | |||
| 27 | nerr=0 | ||
| 28 | |||
| 29 | for alg in $AES_ALGS; do | ||
| 30 | echo $alg | ||
| 31 | for bufsize in $BUFSIZE; do | ||
| 32 | TEST=`( cat $PROG | \ | ||
| 33 | $PROG enc -e -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
| 34 | $PROG enc -d -k "$HASH" -$alg | \ | ||
| 35 | $PROG dgst -hex ) 2>/dev/null` | ||
| 36 | if [ "$TEST" != "$HASH" ]; then | ||
| 37 | echo "-$alg/$bufsize encrypt test failed" | ||
| 38 | nerr=`expr $nerr + 1` | ||
| 39 | fi | ||
| 40 | done | ||
| 41 | for bufsize in $BUFSIZE; do | ||
| 42 | TEST=`( cat $PROG | \ | ||
| 43 | $PROG enc -e -k "$HASH" -$alg | \ | ||
| 44 | $PROG enc -d -k "$HASH" -$alg -bufsize $bufsize -engine aesni | \ | ||
| 45 | $PROG dgst -hex ) 2>/dev/null` | ||
| 46 | if [ "$TEST" != "$HASH" ]; then | ||
| 47 | echo "-$alg/$bufsize decrypt test failed" | ||
| 48 | nerr=`expr $nerr + 1` | ||
| 49 | fi | ||
| 50 | done | ||
| 51 | TEST=`( cat $PROG | \ | ||
| 52 | $PROG enc -e -k "$HASH" -$alg -engine aesni | \ | ||
| 53 | $PROG enc -d -k "$HASH" -$alg -engine aesni | \ | ||
| 54 | $PROG dgst -hex ) 2>/dev/null` | ||
| 55 | if [ "$TEST" != "$HASH" ]; then | ||
| 56 | echo "-$alg en/decrypt test failed" | ||
| 57 | nerr=`expr $nerr + 1` | ||
| 58 | fi | ||
| 59 | done | ||
| 60 | |||
| 61 | if [ $nerr -gt 0 ]; then | ||
| 62 | echo "AESNI engine test failed." | ||
| 63 | exit 1; | ||
| 64 | fi | ||
| 65 | else | ||
| 66 | echo "AESNI engine is not available" | ||
| 67 | fi | ||
| 68 | |||
| 69 | exit 0 | ||
