6 files changed, 126 insertions, 76 deletions

diff --git a/src/lib/libcrypto/aes/asm/aes-mips.pl b/src/lib/libcrypto/aes/asm/aes-mips.pl
index 2ce6deffc8..e52395421b 100644
--- a/src/lib/libcrypto/aes/asm/aes-mips.pl
+++ b/src/lib/libcrypto/aes/asm/aes-mips.pl
@@ -1036,9 +1036,9 @@ _mips_AES_set_encrypt_key:
         nop
 .end    _mips_AES_set_encrypt_key
 
-.globl  AES_set_encrypt_key
-.ent    AES_set_encrypt_key
-AES_set_encrypt_key:
+.globl  private_AES_set_encrypt_key
+.ent    private_AES_set_encrypt_key
+private_AES_set_encrypt_key:
         .frame  $sp,$FRAMESIZE,$ra
         .mask   $SAVED_REGS_MASK,-$SZREG
         .set    noreorder
@@ -1060,7 +1060,7 @@ $code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi prologue
 ___
 $code.=<<___ if ($flavour !~ /o32/i);   # non-o32 PIC-ification
         .cplocal        $Tbl
-        .cpsetup        $pf,$zero,AES_set_encrypt_key
+        .cpsetup        $pf,$zero,private_AES_set_encrypt_key
 ___
 $code.=<<___;
         .set    reorder
@@ -1083,7 +1083,7 @@ ___
 $code.=<<___;
         jr      $ra
         $PTR_ADD $sp,$FRAMESIZE
-.end    AES_set_encrypt_key
+.end    private_AES_set_encrypt_key
 ___
 
 my ($head,$tail)=($inp,$bits);
@@ -1091,9 +1091,9 @@ my ($tp1,$tp2,$tp4,$tp8,$tp9,$tpb,$tpd,$tpe)=($a4,$a5,$a6,$a7,$s0,$s1,$s2,$s3);
 my ($m,$x80808080,$x7f7f7f7f,$x1b1b1b1b)=($at,$t0,$t1,$t2);
 $code.=<<___;
 .align  5
-.globl  AES_set_decrypt_key
-.ent    AES_set_decrypt_key
-AES_set_decrypt_key:
+.globl  private_AES_set_decrypt_key
+.ent    private_AES_set_decrypt_key
+private_AES_set_decrypt_key:
         .frame  $sp,$FRAMESIZE,$ra
         .mask   $SAVED_REGS_MASK,-$SZREG
         .set    noreorder
@@ -1115,7 +1115,7 @@ $code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi prologue
 ___
 $code.=<<___ if ($flavour !~ /o32/i);   # non-o32 PIC-ification
         .cplocal        $Tbl
-        .cpsetup        $pf,$zero,AES_set_decrypt_key
+        .cpsetup        $pf,$zero,private_AES_set_decrypt_key
 ___
 $code.=<<___;
         .set    reorder
@@ -1226,7 +1226,7 @@ ___
 $code.=<<___;
         jr      $ra
         $PTR_ADD $sp,$FRAMESIZE
-.end    AES_set_decrypt_key
+.end    private_AES_set_decrypt_key
 ___
 }}}
 
diff --git a/src/lib/libcrypto/aes/asm/aes-parisc.pl b/src/lib/libcrypto/aes/asm/aes-parisc.pl
index c36b6a2270..714dcfbbe3 100644
--- a/src/lib/libcrypto/aes/asm/aes-parisc.pl
+++ b/src/lib/libcrypto/aes/asm/aes-parisc.pl
@@ -1015,7 +1015,8 @@ foreach (split("\n",$code)) {
                 $SIZE_T==4 ? sprintf("extru%s,%d,8,",$1,31-$2)
                 :            sprintf("extrd,u%s,%d,8,",$1,63-$2)/e;
 
-        s/,\*/,/ if ($SIZE_T==4);
+        s/,\*/,/                        if ($SIZE_T==4);
+        s/\bbv\b(.*\(%r2\))/bve$1/      if ($SIZE_T==8);
         print $_,"\n";
 }
 close STDOUT;
diff --git a/src/lib/libcrypto/aes/asm/aes-s390x.pl b/src/lib/libcrypto/aes/asm/aes-s390x.pl
index 445a1e6762..e75dcd0315 100644
--- a/src/lib/libcrypto/aes/asm/aes-s390x.pl
+++ b/src/lib/libcrypto/aes/asm/aes-s390x.pl
@@ -1598,11 +1598,11 @@ $code.=<<___ if(1);
         lghi    $s1,0x7f
         nr      $s1,%r0
         lghi    %r0,0                   # query capability vector
-        la      %r1,2*$SIZE_T($sp)
+        la      %r1,$tweak-16($sp)
         .long   0xb92e0042              # km %r4,%r2
         llihh   %r1,0x8000
         srlg    %r1,%r1,32($s1)         # check for 32+function code
-        ng      %r1,2*$SIZE_T($sp)
+        ng      %r1,$tweak-16($sp)
         lgr     %r0,$s0                 # restore the function code
         la      %r1,0($key1)            # restore $key1
         jz      .Lxts_km_vanilla
@@ -1628,7 +1628,7 @@ $code.=<<___ if(1);
 
         lrvg    $s0,$tweak+0($sp)       # load the last tweak
         lrvg    $s1,$tweak+8($sp)
-        stmg    %r0,%r3,$tweak-32(%r1)  # wipe copy of the key
+        stmg    %r0,%r3,$tweak-32($sp)  # wipe copy of the key
 
         nill    %r0,0xffdf              # switch back to original function code
         la      %r1,0($key1)            # restore pointer to $key1
@@ -1684,11 +1684,9 @@ $code.=<<___;
         lghi    $i1,0x87
         srag    $i2,$s1,63              # broadcast upper bit
         ngr     $i1,$i2                 # rem
-        srlg    $i2,$s0,63              # carry bit from lower half
-        sllg    $s0,$s0,1
-        sllg    $s1,$s1,1
+        algr    $s0,$s0
+        alcgr   $s1,$s1
         xgr     $s0,$i1
-        ogr     $s1,$i2
 .Lxts_km_start:
         lrvgr   $i1,$s0                 # flip byte order
         lrvgr   $i2,$s1
@@ -1745,11 +1743,9 @@ $code.=<<___;
         lghi    $i1,0x87
         srag    $i2,$s1,63              # broadcast upper bit
         ngr     $i1,$i2                 # rem
-        srlg    $i2,$s0,63              # carry bit from lower half
-        sllg    $s0,$s0,1
-        sllg    $s1,$s1,1
+        algr    $s0,$s0
+        alcgr   $s1,$s1
         xgr     $s0,$i1
-        ogr     $s1,$i2
 
         ltr     $len,$len               # clear zero flag
         br      $ra
@@ -1781,8 +1777,8 @@ $code.=<<___ if (!$softonly);
         clr     %r0,%r1
         jl      .Lxts_enc_software
 
+        st${g}  $ra,5*$SIZE_T($sp)
         stm${g} %r6,$s3,6*$SIZE_T($sp)
-        st${g}  $ra,14*$SIZE_T($sp)
 
         sllg    $len,$len,4             # $len&=~15
         slgr    $out,$inp
@@ -1830,9 +1826,9 @@ $code.=<<___ if (!$softonly);
         stg     $i2,8($i3)
 
 .Lxts_enc_km_done:
-        l${g}   $ra,14*$SIZE_T($sp)
-        st${g}  $sp,$tweak($sp)         # wipe tweak
-        st${g}  $sp,$tweak($sp)
+        stg     $sp,$tweak+0($sp)       # wipe tweak
+        stg     $sp,$tweak+8($sp)
+        l${g}   $ra,5*$SIZE_T($sp)
         lm${g}  %r6,$s3,6*$SIZE_T($sp)
         br      $ra
 .align  16
@@ -1843,12 +1839,11 @@ $code.=<<___;
 
         slgr    $out,$inp
 
-        xgr     $s0,$s0                 # clear upper half
-        xgr     $s1,$s1
-        lrv     $s0,$stdframe+4($sp)    # load secno
-        lrv     $s1,$stdframe+0($sp)
-        xgr     $s2,$s2
-        xgr     $s3,$s3
+        l${g}   $s3,$stdframe($sp)      # ivp
+        llgf    $s0,0($s3)              # load iv
+        llgf    $s1,4($s3)
+        llgf    $s2,8($s3)
+        llgf    $s3,12($s3)
         stm${g} %r2,%r5,2*$SIZE_T($sp)
         la      $key,0($key2)
         larl    $tbl,AES_Te
@@ -1864,11 +1859,9 @@ $code.=<<___;
         lghi    %r1,0x87
         srag    %r0,$s3,63              # broadcast upper bit
         ngr     %r1,%r0                 # rem
-        srlg    %r0,$s1,63              # carry bit from lower half
-        sllg    $s1,$s1,1
-        sllg    $s3,$s3,1
+        algr    $s1,$s1
+        alcgr   $s3,$s3
         xgr     $s1,%r1
-        ogr     $s3,%r0
         lrvgr   $s1,$s1                 # flip byte order
         lrvgr   $s3,$s3
         srlg    $s0,$s1,32              # smash the tweak to 4x32-bits 
@@ -1917,11 +1910,9 @@ $code.=<<___;
         lghi    %r1,0x87
         srag    %r0,$s3,63              # broadcast upper bit
         ngr     %r1,%r0                 # rem
-        srlg    %r0,$s1,63              # carry bit from lower half
-        sllg    $s1,$s1,1
-        sllg    $s3,$s3,1
+        algr    $s1,$s1
+        alcgr   $s3,$s3
         xgr     $s1,%r1
-        ogr     $s3,%r0
         lrvgr   $s1,$s1                 # flip byte order
         lrvgr   $s3,$s3
         srlg    $s0,$s1,32              # smash the tweak to 4x32-bits 
@@ -1956,7 +1947,8 @@ $code.=<<___;
 .size   AES_xts_encrypt,.-AES_xts_encrypt
 ___
 # void AES_xts_decrypt(const char *inp,char *out,size_t len,
-#       const AES_KEY *key1, const AES_KEY *key2,u64 secno);
+#       const AES_KEY *key1, const AES_KEY *key2,
+#       const unsigned char iv[16]);
 #
 $code.=<<___;
 .globl  AES_xts_decrypt
@@ -1988,8 +1980,8 @@ $code.=<<___ if (!$softonly);
         clr     %r0,%r1
         jl      .Lxts_dec_software
 
+        st${g}  $ra,5*$SIZE_T($sp)
         stm${g} %r6,$s3,6*$SIZE_T($sp)
-        st${g}  $ra,14*$SIZE_T($sp)
 
         nill    $len,0xfff0             # $len&=~15
         slgr    $out,$inp
@@ -2028,11 +2020,9 @@ $code.=<<___ if (!$softonly);
         lghi    $i1,0x87
         srag    $i2,$s1,63              # broadcast upper bit
         ngr     $i1,$i2                 # rem
-        srlg    $i2,$s0,63              # carry bit from lower half
-        sllg    $s0,$s0,1
-        sllg    $s1,$s1,1
+        algr    $s0,$s0
+        alcgr   $s1,$s1
         xgr     $s0,$i1
-        ogr     $s1,$i2
         lrvgr   $i1,$s0                 # flip byte order
         lrvgr   $i2,$s1
 
@@ -2075,9 +2065,9 @@ $code.=<<___ if (!$softonly);
         stg     $s2,0($i3)
         stg     $s3,8($i3)
 .Lxts_dec_km_done:
-        l${g}   $ra,14*$SIZE_T($sp)
-        st${g}  $sp,$tweak($sp)         # wipe tweak
-        st${g}  $sp,$tweak($sp)
+        stg     $sp,$tweak+0($sp)       # wipe tweak
+        stg     $sp,$tweak+8($sp)
+        l${g}   $ra,5*$SIZE_T($sp)
         lm${g}  %r6,$s3,6*$SIZE_T($sp)
         br      $ra
 .align  16
@@ -2089,12 +2079,11 @@ $code.=<<___;
         srlg    $len,$len,4
         slgr    $out,$inp
 
-        xgr     $s0,$s0                 # clear upper half
-        xgr     $s1,$s1
-        lrv     $s0,$stdframe+4($sp)    # load secno
-        lrv     $s1,$stdframe+0($sp)
-        xgr     $s2,$s2
-        xgr     $s3,$s3
+        l${g}   $s3,$stdframe($sp)      # ivp
+        llgf    $s0,0($s3)              # load iv
+        llgf    $s1,4($s3)
+        llgf    $s2,8($s3)
+        llgf    $s3,12($s3)
         stm${g} %r2,%r5,2*$SIZE_T($sp)
         la      $key,0($key2)
         larl    $tbl,AES_Te
@@ -2113,11 +2102,9 @@ $code.=<<___;
         lghi    %r1,0x87
         srag    %r0,$s3,63              # broadcast upper bit
         ngr     %r1,%r0                 # rem
-        srlg    %r0,$s1,63              # carry bit from lower half
-        sllg    $s1,$s1,1
-        sllg    $s3,$s3,1
+        algr    $s1,$s1
+        alcgr   $s3,$s3
         xgr     $s1,%r1
-        ogr     $s3,%r0
         lrvgr   $s1,$s1                 # flip byte order
         lrvgr   $s3,$s3
         srlg    $s0,$s1,32              # smash the tweak to 4x32-bits 
@@ -2156,11 +2143,9 @@ $code.=<<___;
         lghi    %r1,0x87
         srag    %r0,$s3,63              # broadcast upper bit
         ngr     %r1,%r0                 # rem
-        srlg    %r0,$s1,63              # carry bit from lower half
-        sllg    $s1,$s1,1
-        sllg    $s3,$s3,1
+        algr    $s1,$s1
+        alcgr   $s3,$s3
         xgr     $s1,%r1
-        ogr     $s3,%r0
         lrvgr   $i2,$s1                 # flip byte order
         lrvgr   $i3,$s3
         stmg    $i2,$i3,$tweak($sp)     # save the 1st tweak
@@ -2176,11 +2161,9 @@ $code.=<<___;
         lghi    %r1,0x87
         srag    %r0,$s3,63              # broadcast upper bit
         ngr     %r1,%r0                 # rem
-        srlg    %r0,$s1,63              # carry bit from lower half
-        sllg    $s1,$s1,1
-        sllg    $s3,$s3,1
+        algr    $s1,$s1
+        alcgr   $s3,$s3
         xgr     $s1,%r1
-        ogr     $s3,%r0
         lrvgr   $s1,$s1                 # flip byte order
         lrvgr   $s3,$s3
         srlg    $s0,$s1,32              # smash the tweak to 4x32-bits
diff --git a/src/lib/libcrypto/aes/asm/aesni-sha1-x86_64.pl b/src/lib/libcrypto/aes/asm/aesni-sha1-x86_64.pl
index c6f6b3334a..3c8f6c19e7 100644
--- a/src/lib/libcrypto/aes/asm/aesni-sha1-x86_64.pl
+++ b/src/lib/libcrypto/aes/asm/aesni-sha1-x86_64.pl
@@ -69,7 +69,8 @@ $avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
            `ml64 2>&1` =~ /Version ([0-9]+)\./ &&
            $1>=10);
 
-open STDOUT,"| $^X $xlate $flavour $output";
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
 
 # void aesni_cbc_sha1_enc(const void *inp,
 #                       void *out,
diff --git a/src/lib/libcrypto/aes/asm/bsaes-x86_64.pl b/src/lib/libcrypto/aes/asm/bsaes-x86_64.pl
index c9c6312fa7..41b90f0844 100644
--- a/src/lib/libcrypto/aes/asm/bsaes-x86_64.pl
+++ b/src/lib/libcrypto/aes/asm/bsaes-x86_64.pl
@@ -83,9 +83,9 @@
 # Add decryption procedure. Performance in CPU cycles spent to decrypt
 # one byte out of 4096-byte buffer with 128-bit key is:
 #
-# Core 2        11.0
-# Nehalem       9.16
-# Atom          20.9
+# Core 2        9.83
+# Nehalem       7.74
+# Atom          19.0
 #
 # November 2011.
 #
@@ -105,7 +105,8 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
 die "can't locate x86_64-xlate.pl";
 
-open STDOUT,"| $^X $xlate $flavour $output";
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
 
 my ($inp,$out,$len,$key,$ivp)=("%rdi","%rsi","%rdx","%rcx");
 my @XMM=map("%xmm$_",(15,0..14));       # best on Atom, +10% over (0..15)
@@ -455,6 +456,7 @@ sub MixColumns {
 # modified to emit output in order suitable for feeding back to aesenc[last]
 my @x=@_[0..7];
 my @t=@_[8..15];
+my $inv=@_[16]; # optional
 $code.=<<___;
         pshufd  \$0x93, @x[0], @t[0]    # x0 <<< 32
         pshufd  \$0x93, @x[1], @t[1]
@@ -496,7 +498,8 @@ $code.=<<___;
         pxor    @t[4], @t[0]
          pshufd \$0x4E, @x[2], @x[6]
         pxor    @t[5], @t[1]
-
+___
+$code.=<<___ if (!$inv);
         pxor    @t[3], @x[4]
         pxor    @t[7], @x[5]
         pxor    @t[6], @x[3]
@@ -504,9 +507,20 @@ $code.=<<___;
         pxor    @t[2], @x[6]
          movdqa @t[1], @x[7]
 ___
+$code.=<<___ if ($inv);
+        pxor    @x[4], @t[3]
+        pxor    @t[7], @x[5]
+        pxor    @x[3], @t[6]
+         movdqa @t[0], @x[3]
+        pxor    @t[2], @x[6]
+         movdqa @t[6], @x[2]
+         movdqa @t[1], @x[7]
+         movdqa @x[6], @x[4]
+         movdqa @t[3], @x[6]
+___
 }
 
-sub InvMixColumns {
+sub InvMixColumns_orig {
 my @x=@_[0..7];
 my @t=@_[8..15];
 
@@ -660,6 +674,54 @@ $code.=<<___;
 ___
 }
 
+sub InvMixColumns {
+my @x=@_[0..7];
+my @t=@_[8..15];
+
+# Thanks to Jussi Kivilinna for providing pointer to
+#
+# | 0e 0b 0d 09 |   | 02 03 01 01 |   | 05 00 04 00 |
+# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 |
+# | 0d 09 0e 0b |   | 01 01 02 03 |   | 04 00 05 00 |
+# | 0b 0d 09 0e |   | 03 01 01 02 |   | 00 04 00 05 |
+
+$code.=<<___;
+        # multiplication by 0x05-0x00-0x04-0x00
+        pshufd  \$0x4E, @x[0], @t[0]
+        pshufd  \$0x4E, @x[6], @t[6]
+        pxor    @x[0], @t[0]
+        pshufd  \$0x4E, @x[7], @t[7]
+        pxor    @x[6], @t[6]
+        pshufd  \$0x4E, @x[1], @t[1]
+        pxor    @x[7], @t[7]
+        pshufd  \$0x4E, @x[2], @t[2]
+        pxor    @x[1], @t[1]
+        pshufd  \$0x4E, @x[3], @t[3]
+        pxor    @x[2], @t[2]
+         pxor   @t[6], @x[0]
+         pxor   @t[6], @x[1]
+        pshufd  \$0x4E, @x[4], @t[4]
+        pxor    @x[3], @t[3]
+         pxor   @t[0], @x[2]
+         pxor   @t[1], @x[3]
+        pshufd  \$0x4E, @x[5], @t[5]
+        pxor    @x[4], @t[4]
+         pxor   @t[7], @x[1]
+         pxor   @t[2], @x[4]
+        pxor    @x[5], @t[5]
+
+         pxor   @t[7], @x[2]
+         pxor   @t[6], @x[3]
+         pxor   @t[6], @x[4]
+         pxor   @t[3], @x[5]
+         pxor   @t[4], @x[6]
+         pxor   @t[7], @x[4]
+         pxor   @t[7], @x[5]
+         pxor   @t[5], @x[7]
+___
+        &MixColumns     (@x,@t,1);      # flipped 2<->3 and 4<->6
+}
+
 sub aesenc {                            # not used
 my @b=@_[0..7];
 my @t=@_[8..15];
@@ -2027,6 +2089,8 @@ ___
 #       const unsigned char iv[16]);
 #
 my ($twmask,$twres,$twtmp)=@XMM[13..15];
+$arg6=~s/d$//;
+
 $code.=<<___;
 .globl  bsaes_xts_encrypt
 .type   bsaes_xts_encrypt,\@abi-omnipotent
diff --git a/src/lib/libcrypto/aes/asm/vpaes-x86_64.pl b/src/lib/libcrypto/aes/asm/vpaes-x86_64.pl
index 37998db5e1..bd7f45b850 100644
--- a/src/lib/libcrypto/aes/asm/vpaes-x86_64.pl
+++ b/src/lib/libcrypto/aes/asm/vpaes-x86_64.pl
@@ -56,7 +56,8 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
 die "can't locate x86_64-xlate.pl";
 
-open STDOUT,"| $^X $xlate $flavour $output";
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
 
 $PREFIX="vpaes";
 
@@ -1059,7 +1060,7 @@ _vpaes_consts:
 .Lk_dsbo:       # decryption sbox final output
         .quad   0x1387EA537EF94000, 0xC7AA6DB9D4943E2D
         .quad   0x12D7560F93441D00, 0xCA4B8159D8C58E9C
-.asciz  "Vector Permutaion AES for x86_64/SSSE3, Mike Hamburg (Stanford University)"
+.asciz  "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)"
 .align  64
 .size   _vpaes_consts,.-_vpaes_consts
 ___