Enable optimized crypto operations for x86_64

This adds initial support for assembly crypto acceleration on x86_64 for
ELF (Linux, *BSD, Solaris) and Mach-O (OS-X) systems.

The build method is a little different than OpenSSL and OpenBSD.  All
the .s files are generated ahead of time when the tarball is generated,
so there are no complicated makefile rules at configure/build time. This
also means the builds are faster and perl is not required on the build
system.

Thanks to Wouter Clarie for providing the initial cleanup and patch
that this is based on.

6 files changed, 168 insertions, 25 deletions

diff --git a/.gitignore b/.gitignore
index 03ff4da..e964021 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,10 @@
 # C stuff
 *.o
 
+# Assembly stuff
+*.S
+*.s
+
 # Windows stuff
 *.obj
 *.exe
diff --git a/configure.ac b/configure.ac
index 2cc7477..4b3d209 100644
--- a/configure.ac
+++ b/configure.ac
@@ -14,22 +14,21 @@ CFLAGS="$CFLAGS -Wall -std=gnu99 -g"
 
 case $host_os in
         *darwin*)
-                HOST_OS=darwin;
+                HOST_OS=darwin
+                HOST_ABI=macosx
                 ;;
         *freebsd*)
-                HOST_OS=freebsd;
+                HOST_OS=freebsd
+                HOST_ABI=elf
                 AC_SUBST([PROG_LDADD], ['-lthr'])
                 ;;
         *linux*)
-                HOST_OS=linux;
+                HOST_OS=linux
+                HOST_ABI=elf
                 CFLAGS="$CFLAGS -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_POSIX_SOURCE -D_GNU_SOURCE"
                 ;;
-        *solaris*)
-                HOST_OS=solaris;
-                CFLAGS="$CFLAGS -D__EXTENSIONS__ -D_XOPEN_SOURCE=600 -DBSD_COMP"
-                AC_SUBST([PLATFORM_LDADD], ['-lnsl -lsocket'])
-                ;;
         *openbsd*)
+                HOST_ABI=elf
                 AC_DEFINE([HAVE_ATTRIBUTE__BOUNDED__], [1], [OpenBSD gcc has bounded])
                 ;;
         *mingw*)
@@ -37,14 +36,20 @@ case $host_os in
                 CFLAGS="$CFLAGS -D_GNU_SOURCE -D_POSIX -D_POSIX_SOURCE -D_REENTRANT -D_POSIX_THREAD_SAFE_FUNCTIONS -DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x0600 -DOPENSSL_NO_SPEED -D__USE_MINGW_ANSI_STDIO"
                 AC_SUBST([PLATFORM_LDADD], ['-lws2_32'])
                 ;;
+        *solaris*)
+                HOST_OS=solaris
+                HOST_ABI=elf
+                CFLAGS="$CFLAGS -D__EXTENSIONS__ -D_XOPEN_SOURCE=600 -DBSD_COMP"
+                AC_SUBST([PLATFORM_LDADD], ['-lnsl -lsocket'])
+                ;;
         *) ;;
 esac
 
-AM_CONDITIONAL(HOST_DARWIN, test x$HOST_OS = xdarwin)
-AM_CONDITIONAL(HOST_FREEBSD, test x$HOST_OS = xfreebsd)
-AM_CONDITIONAL(HOST_LINUX, test x$HOST_OS = xlinux)
-AM_CONDITIONAL(HOST_SOLARIS, test x$HOST_OS = xsolaris)
-AM_CONDITIONAL(HOST_WIN, test x$HOST_OS = xwin)
+AM_CONDITIONAL([HOST_DARWIN],  [test x$HOST_OS = xdarwin])
+AM_CONDITIONAL([HOST_FREEBSD], [test x$HOST_OS = xfreebsd])
+AM_CONDITIONAL([HOST_LINUX],   [test x$HOST_OS = xlinux])
+AM_CONDITIONAL([HOST_SOLARIS], [test x$HOST_OS = xsolaris])
+AM_CONDITIONAL([HOST_WIN],     [test x$HOST_OS = xwin])
 
 AC_CHECK_FUNC([clock_gettime],,
         [AC_SEARCH_LIBS([clock_gettime],[rt posix4])])
@@ -52,6 +57,7 @@ AC_CHECK_FUNC([clock_gettime],,
 AC_CHECK_FUNC([dl_iterate_phdr],,
         [AC_SEARCH_LIBS([dl_iterate_phdr],[dl])])
 
+AM_PROG_AS
 AC_PROG_CC
 AC_PROG_LIBTOOL
 AC_PROG_CC_STDC
@@ -134,18 +140,25 @@ fi
 AC_CHECK_HEADERS([sys/sysctl.h err.h])
 
 AC_ARG_WITH([openssldir],
-        AS_HELP_STRING([--with-openssldir], [Set the default openssl directory]),
+        AS_HELP_STRING([--with-openssldir],
+                       [Set the default openssl directory]),
         AC_DEFINE_UNQUOTED(OPENSSLDIR, "$withval")
 )
 
 AC_ARG_WITH([enginesdir],
-        AS_HELP_STRING([--with-enginesdir], [Set the default engines directory (use with openssldir)]),
+        AS_HELP_STRING([--with-enginesdir],
+                       [Set the default engines directory (use with openssldir)]),
         AC_DEFINE_UNQUOTED(ENGINESDIR, "$withval")
 )
 
 AC_ARG_ENABLE([asm],
         AS_HELP_STRING([--disable-asm], [Disable assembly]))
-AS_IF([test "x$enable_asm" = "xno"], [CFLAGS="$CFLAGS -DOPENSSL_NO_ASM"])
+AM_CONDITIONAL([OPENSSL_NO_ASM], [test "x$enable_asm" = "xno"])
+
+AM_CONDITIONAL([HOST_ASM_ELF_X86_64],
+    [test "x$HOST_ABI" = "xelf" -a "$host_cpu" = "x86_64" -a "x$enable_asm" != "xno"])
+AM_CONDITIONAL([HOST_ASM_MACOSX_X86_64],
+    [test "x$HOST_ABI" = "xmacosx" -a "$host_cpu" = "x86_64" -a "x$enable_asm" != "xno"])
 
 AC_ARG_ENABLE([libtls],
         AS_HELP_STRING([--enable-libtls], [Enable building the libtls library]))
diff --git a/crypto/Makefile.am b/crypto/Makefile.am
index 39b143d..91c58db 100644
--- a/crypto/Makefile.am
+++ b/crypto/Makefile.am
@@ -10,7 +10,11 @@ EXTRA_DIST = VERSION
 
 libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@
 libcrypto_la_LIBADD = libcompat.la libcompatnoopt.la
-libcrypto_la_CFLAGS = $(CFLAGS) $(USER_CFLAGS) -DOPENSSL_NO_HW_PADLOCK
+libcrypto_la_CFLAGS = $(CFLAGS) $(USER_CFLAGS)
+libcrypto_la_CFLAGS += -DOPENSSL_NO_HW_PADLOCK
+if OPENSSL_NO_ASM
+libcrypto_la_CFLAGS += -DOPENSSL_NO_ASM
+endif
 
 noinst_LTLIBRARIES = libcompat.la libcompatnoopt.la
 
@@ -103,6 +107,21 @@ noinst_HEADERS += compat/chacha_private.h
 libcrypto_la_SOURCES =
 EXTRA_libcrypto_la_SOURCES =
 
+include Makefile.am.elf-x86_64
+include Makefile.am.macosx-x86_64
+
+if !HOST_ASM_ELF_X86_64
+if !HOST_ASM_MACOSX_X86_64
+libcrypto_la_SOURCES += aes/aes_cbc.c
+libcrypto_la_SOURCES += aes/aes_core.c
+libcrypto_la_SOURCES += camellia/camellia.c
+libcrypto_la_SOURCES += camellia/cmll_cbc.c
+libcrypto_la_SOURCES += rc4/rc4_enc.c
+libcrypto_la_SOURCES += rc4/rc4_skey.c
+libcrypto_la_SOURCES += whrlpool/wp_block.c
+endif
+endif
+
 libcrypto_la_SOURCES += cpt_err.c
 libcrypto_la_SOURCES += cryptlib.c
 libcrypto_la_SOURCES += cversion.c
@@ -118,9 +137,7 @@ noinst_HEADERS += md32_common.h
 noinst_HEADERS += o_time.h
 
 # aes
-libcrypto_la_SOURCES += aes/aes_cbc.c
 libcrypto_la_SOURCES += aes/aes_cfb.c
-libcrypto_la_SOURCES += aes/aes_core.c
 libcrypto_la_SOURCES += aes/aes_ctr.c
 libcrypto_la_SOURCES += aes/aes_ecb.c
 libcrypto_la_SOURCES += aes/aes_ige.c
@@ -284,8 +301,6 @@ libcrypto_la_SOURCES += buffer/buf_str.c
 libcrypto_la_SOURCES += buffer/buffer.c
 
 # camellia
-libcrypto_la_SOURCES += camellia/camellia.c
-libcrypto_la_SOURCES += camellia/cmll_cbc.c
 libcrypto_la_SOURCES += camellia/cmll_cfb.c
 libcrypto_la_SOURCES += camellia/cmll_ctr.c
 libcrypto_la_SOURCES += camellia/cmll_ecb.c
@@ -666,8 +681,6 @@ libcrypto_la_SOURCES += rc2/rc2ofb64.c
 noinst_HEADERS += rc2/rc2_locl.h
 
 # rc4
-libcrypto_la_SOURCES += rc4/rc4_enc.c
-libcrypto_la_SOURCES += rc4/rc4_skey.c
 noinst_HEADERS += rc4/rc4_locl.h
 
 # ripemd
@@ -739,7 +752,6 @@ libcrypto_la_SOURCES += ui/ui_util.c
 noinst_HEADERS += ui/ui_locl.h
 
 # whrlpool
-libcrypto_la_SOURCES += whrlpool/wp_block.c
 libcrypto_la_SOURCES += whrlpool/wp_dgst.c
 noinst_HEADERS += whrlpool/wp_locl.h
 
diff --git a/crypto/Makefile.am.elf-x86_64 b/crypto/Makefile.am.elf-x86_64
new file mode 100644
index 0000000..6257c40
--- /dev/null
+++ b/crypto/Makefile.am.elf-x86_64
@@ -0,0 +1,41 @@
+
+ASM_X86_64_ELF = aes/aes-elf-x86_64.s
+ASM_X86_64_ELF += aes/bsaes-elf-x86_64.s
+ASM_X86_64_ELF += aes/vpaes-elf-x86_64.s
+ASM_X86_64_ELF += aes/aesni-elf-x86_64.s
+ASM_X86_64_ELF += aes/aesni-sha1-elf-x86_64.s
+ASM_X86_64_ELF += bn/modexp512-elf-x86_64.s
+ASM_X86_64_ELF += bn/mont-elf-x86_64.s
+ASM_X86_64_ELF += bn/mont5-elf-x86_64.s
+ASM_X86_64_ELF += bn/gf2m-elf-x86_64.s
+ASM_X86_64_ELF += camellia/cmll-elf-x86_64.s
+ASM_X86_64_ELF += md5/md5-elf-x86_64.s
+ASM_X86_64_ELF += modes/ghash-elf-x86_64.s
+ASM_X86_64_ELF += rc4/rc4-elf-x86_64.s
+ASM_X86_64_ELF += rc4/rc4-md5-elf-x86_64.s
+ASM_X86_64_ELF += sha/sha1-elf-x86_64.s
+ASM_X86_64_ELF += sha/sha256-elf-x86_64.S
+ASM_X86_64_ELF += sha/sha512-elf-x86_64.S
+ASM_X86_64_ELF += whrlpool/wp-elf-x86_64.s
+ASM_X86_64_ELF += cpuid-elf-x86_64.S
+
+EXTRA_DIST += $(ASM_X86_64_ELF)
+
+if HOST_ASM_ELF_X86_64
+libcrypto_la_CFLAGS += -DAES_ASM
+libcrypto_la_CFLAGS += -DBSAES_ASM
+libcrypto_la_CFLAGS += -DVPAES_ASM
+libcrypto_la_CFLAGS += -DOPENSSL_IA32_SSE2
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_MONT
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_MONT5
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_GF2m
+libcrypto_la_CFLAGS += -DMD5_ASM
+libcrypto_la_CFLAGS += -DGHASH_ASM
+libcrypto_la_CFLAGS += -DRSA_ASM
+libcrypto_la_CFLAGS += -DSHA1_ASM
+libcrypto_la_CFLAGS += -DSHA256_ASM
+libcrypto_la_CFLAGS += -DSHA512_ASM
+libcrypto_la_CFLAGS += -DWHIRLPOOL_ASM
+libcrypto_la_CFLAGS += -DOPENSSL_CPUID_OBJ
+libcrypto_la_SOURCES += $(ASM_X86_64_ELF)
+endif
diff --git a/crypto/Makefile.am.macosx-x86_64 b/crypto/Makefile.am.macosx-x86_64
new file mode 100644
index 0000000..e361aae
--- /dev/null
+++ b/crypto/Makefile.am.macosx-x86_64
@@ -0,0 +1,41 @@
+
+ASM_X86_64_MACOSX = aes/aes-macosx-x86_64.s
+ASM_X86_64_MACOSX += aes/bsaes-macosx-x86_64.s
+ASM_X86_64_MACOSX += aes/vpaes-macosx-x86_64.s
+ASM_X86_64_MACOSX += aes/aesni-macosx-x86_64.s
+ASM_X86_64_MACOSX += aes/aesni-sha1-macosx-x86_64.s
+ASM_X86_64_MACOSX += bn/modexp512-macosx-x86_64.s
+ASM_X86_64_MACOSX += bn/mont-macosx-x86_64.s
+ASM_X86_64_MACOSX += bn/mont5-macosx-x86_64.s
+ASM_X86_64_MACOSX += bn/gf2m-macosx-x86_64.s
+ASM_X86_64_MACOSX += camellia/cmll-macosx-x86_64.s
+ASM_X86_64_MACOSX += md5/md5-macosx-x86_64.s
+ASM_X86_64_MACOSX += modes/ghash-macosx-x86_64.s
+ASM_X86_64_MACOSX += rc4/rc4-macosx-x86_64.s
+ASM_X86_64_MACOSX += rc4/rc4-md5-macosx-x86_64.s
+ASM_X86_64_MACOSX += sha/sha1-macosx-x86_64.s
+ASM_X86_64_MACOSX += sha/sha256-macosx-x86_64.S
+ASM_X86_64_MACOSX += sha/sha512-macosx-x86_64.S
+ASM_X86_64_MACOSX += whrlpool/wp-macosx-x86_64.s
+ASM_X86_64_MACOSX += cpuid-macosx-x86_64.S
+
+EXTRA_DIST += $(ASM_X86_64_MACOSX)
+
+if HOST_ASM_MACOSX_X86_64
+libcrypto_la_CFLAGS += -DAES_ASM
+libcrypto_la_CFLAGS += -DBSAES_ASM
+libcrypto_la_CFLAGS += -DVPAES_ASM
+libcrypto_la_CFLAGS += -DOPENSSL_IA32_SSE2
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_MONT
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_MONT5
+libcrypto_la_CFLAGS += -DOPENSSL_BN_ASM_GF2m
+libcrypto_la_CFLAGS += -DMD5_ASM
+libcrypto_la_CFLAGS += -DGHASH_ASM
+libcrypto_la_CFLAGS += -DRSA_ASM
+libcrypto_la_CFLAGS += -DSHA1_ASM
+libcrypto_la_CFLAGS += -DSHA256_ASM
+libcrypto_la_CFLAGS += -DSHA512_ASM
+libcrypto_la_CFLAGS += -DWHIRLPOOL_ASM
+libcrypto_la_CFLAGS += -DOPENSSL_CPUID_OBJ
+libcrypto_la_SOURCES += $(ASM_X86_64_MACOSX)
+endif
diff --git a/update.sh b/update.sh
index 47e2e22..348616f 100755
--- a/update.sh
+++ b/update.sh
@@ -5,6 +5,7 @@ openbsd_branch=`cat OPENBSD_BRANCH`
 libressl_version=`cat VERSION`
 
 # pull in latest upstream code
+echo "pulling upstream openbsd source"
 if [ ! -d openbsd ]; then
         if [ -z "$LIBRESSL_GIT" ]; then
                 git clone https://github.com/libressl-portable/openbsd.git
@@ -16,7 +17,7 @@ fi
  git checkout $openbsd_branch
  git pull --rebase)
 
-# setup source paths 
+# setup source paths
 dir=`pwd`
 libc_src=$dir/openbsd/src/lib/libc
 libc_regress=$dir/openbsd/src/regress/lib/libc
@@ -93,6 +94,7 @@ copy_hdrs crypto "stack/stack.h lhash/lhash.h stack/safestack.h opensslv.h
 copy_hdrs ssl "srtp.h ssl.h ssl2.h ssl3.h ssl23.h tls1.h dtls1.h"
 
 # copy libcrypto source
+echo copying libcrypto source
 rm -f crypto/*.c crypto/*.h
 for i in `awk '/SOURCES|HEADERS/ { print $3 }' crypto/Makefile.am` ; do
         dir=`dirname $i`
@@ -106,13 +108,40 @@ done
 $CP crypto/compat/b_win.c crypto/bio
 $CP crypto/compat/ui_openssl_win.c crypto/ui
 
+# generate assembly crypto algorithms
+asm_src=$libssl_src/src/crypto
+for abi in elf macosx; do
+        echo generating ASM source for $abi
+        perl $asm_src/aes/asm/aes-x86_64.pl $abi        > crypto/aes/aes-${abi}-x86_64.s
+        perl $asm_src/aes/asm/vpaes-x86_64.pl $abi      > crypto/aes/vpaes-${abi}-x86_64.s
+        perl $asm_src/aes/asm/bsaes-x86_64.pl $abi      > crypto/aes/bsaes-${abi}-x86_64.s
+        perl $asm_src/aes/asm/aesni-x86_64.pl $abi      > crypto/aes/aesni-${abi}-x86_64.s
+        perl $asm_src/aes/asm/aesni-sha1-x86_64.pl $abi > crypto/aes/aesni-sha1-${abi}-x86_64.s
+        perl $asm_src/bn/asm/modexp512-x86_64.pl $abi   > crypto/bn/modexp512-${abi}-x86_64.s
+        perl $asm_src/bn/asm/x86_64-mont.pl $abi        > crypto/bn/mont-${abi}-x86_64.s
+        perl $asm_src/bn/asm/x86_64-mont5.pl $abi       > crypto/bn/mont5-${abi}-x86_64.s
+        perl $asm_src/bn/asm/x86_64-gf2m.pl $abi        > crypto/bn/gf2m-${abi}-x86_64.s
+        perl $asm_src/camellia/asm/cmll-x86_64.pl $abi  > crypto/camellia/cmll-${abi}-x86_64.s
+        perl $asm_src/md5/asm/md5-x86_64.pl $abi        > crypto/md5/md5-${abi}-x86_64.s
+        perl $asm_src/modes/asm/ghash-x86_64.pl $abi    > crypto/modes/ghash-${abi}-x86_64.s
+        perl $asm_src/rc4/asm/rc4-x86_64.pl $abi        > crypto/rc4/rc4-${abi}-x86_64.s
+        perl $asm_src/rc4/asm/rc4-md5-x86_64.pl $abi    > crypto/rc4/rc4-md5-${abi}-x86_64.s
+        perl $asm_src/sha/asm/sha1-x86_64.pl $abi       > crypto/sha/sha1-${abi}-x86_64.s
+        perl $asm_src/sha/asm/sha512-x86_64.pl $abi     crypto/sha/sha256-${abi}-x86_64.S
+        perl $asm_src/sha/asm/sha512-x86_64.pl $abi     crypto/sha/sha512-${abi}-x86_64.S
+        perl $asm_src/whrlpool/asm/wp-x86_64.pl $abi    > crypto/whrlpool/wp-${abi}-x86_64.s
+        perl $asm_src/x86_64cpuid.pl $abi               crypto/cpuid-${abi}-x86_64.S
+done
+
 # copy libtls source
+echo copying libtls source
 rm -f tls/*.c tls/*.h
 for i in `awk '/SOURCES|HEADERS/ { print $3 }' tls/Makefile.am` ; do
         cp $libtls_src/$i tls
 done
 
 # copy openssl(1) source
+echo "copying openssl(1) source"
 $CP $libc_src/stdlib/strtonum.c apps
 $CP $libcrypto_src/openssl.cnf apps
 for i in `awk '/SOURCES|HEADERS/ { print $3 }' apps/Makefile.am` ; do
@@ -122,12 +151,14 @@ for i in `awk '/SOURCES|HEADERS/ { print $3 }' apps/Makefile.am` ; do
 done
 
 # copy libssl source
+echo "copying libssl source"
 rm -f ssl/*.c ssl/*.h
 for i in `awk '/SOURCES|HEADERS/ { print $3 }' ssl/Makefile.am` ; do
         cp $libssl_src/src/ssl/$i ssl
 done
 
 # copy libcrypto tests
+echo "copying tests"
 rm -f tests/biotest.c
 for i in aead/aeadtest.c aeswrap/aes_wrap.c base64/base64test.c bf/bftest.c \
         bn/general/bntest.c bn/mont/mont.c \
@@ -223,6 +254,7 @@ echo "EXTRA_DIST += testssl ca.pem server.pem" >> tests/Makefile.am
         done
 )
 
+echo "copying manpages"
 # copy manpages
 (cd man
         $CP Makefile.am.tpl Makefile.am