add fix for issue #17

1 files changed, 148 insertions, 7 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 3465922..163ba7f 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -33,7 +33,9 @@
 #include <math.h>         /* INFINITY fabs(3) floor(3) frexp(3) fmod(3) round(3) isfinite(3) */
 #include <time.h>         /* struct tm time_t strptime(3) time(2) */
 #include <ctype.h>        /* tolower(3) */
+#include <signal.h>       /* sig_atomic_t */
 #include <errno.h>        /* ENOMEM errno */
+#include <assert.h>       /* assert */
 
 #include <sys/types.h>    /* ssize_t pid_t */
 #if !defined __sun && !defined _AIX
@@ -44,16 +46,11 @@
 #include <sys/socket.h>   /* AF_INET AF_INET6 */
 #include <sys/resource.h> /* RUSAGE_SELF struct rusage getrusage(2) */
 #include <sys/utsname.h>  /* struct utsname uname(3) */
-
 #include <fcntl.h>        /* O_RDONLY O_CLOEXEC open(2) */
-
 #include <unistd.h>       /* close(2) getpid(2) */
-
 #include <netinet/in.h>   /* struct in_addr struct in6_addr */
 #include <arpa/inet.h>    /* inet_pton(3) */
-
 #include <pthread.h>      /* pthread_mutex_init(3) pthread_mutex_lock(3) pthread_mutex_unlock(3) */
-
 #include <dlfcn.h>        /* dladdr(3) dlopen(3) */
 
 #if __APPLE__
@@ -576,6 +573,22 @@ static void lib_setintegers(lua_State *L, const integer_Reg *l) {
 } /* lib_setintegers() */
 
 
+#define COMPAT_X509_STORE_FREE_BUG 0x01
+
+static struct {
+        int flags;
+
+        int SSL_CTX_ex_index;
+        void (*X509_STORE_free)(X509_STORE *);
+
+        struct {
+                X509_STORE *store;
+        } tmp;
+} compat = {
+        .flags = 0,
+        .SSL_CTX_ex_index = -1,
+        .X509_STORE_free = &X509_STORE_free,
+};
 
 #if !HAVE_EVP_PKEY_base_id
 #define EVP_PKEY_base_id(key) compat_EVP_PKEY_base_id((key))
@@ -621,6 +634,126 @@ static void *compat_EVP_PKEY_get0(EVP_PKEY *key) {
 } /* compat_EVP_PKEY_get0() */
 #endif
 
+/* X509_STORE_free in OpenSSL versions < 1.0.2 doesn't obey reference count */
+#define X509_STORE_free(store) \
+        (compat.X509_STORE_free)((store))
+
+static void compat_X509_STORE_free(X509_STORE *store) {
+        int i;
+
+        i = CRYPTO_add(&store->references, -1, CRYPTO_LOCK_X509_STORE);
+
+        if (i > 0)
+                return;
+
+        (X509_STORE_free)(store);
+} /* compat_X509_STORE_free() */
+
+#if !HAVE_SSL_CTX_set1_cert_store
+#define SSL_CTX_set1_cert_store(ctx, store) \
+        compat_SSL_CTX_set1_cert_store((ctx), (store))
+
+static void compat_SSL_CTX_set1_cert_store(SSL_CTX *ctx, X509_STORE *store) {
+        int n;
+
+        /*
+         * This isn't thead-safe, but using X509_STORE or SSL_CTX objects
+         * from different threads isn't safe generally.
+         */
+        if (ctx->cert_store) {
+                X509_STORE_free(ctx->cert_store);
+                ctx->cert_store = NULL;
+        }
+
+        n = store->references;
+
+        SSL_CTX_set_cert_store(ctx, store);
+
+        if (n == store->references)
+                CRYPTO_add(&store->references, 1, CRYPTO_LOCK_X509_STORE);
+} /* compat_SSL_CTX_set1_cert_store() */
+#endif
+
+static void compat_SSL_CTX_onfree(void *_ctx, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+        SSL_CTX *ctx = _ctx;
+
+        if (ctx->cert_store) {
+                X509_STORE_free(ctx->cert_store);
+                ctx->cert_store = NULL;
+        }
+} /* compat_SSL_CTX_onfree() */
+
+/* helper routine to determine if X509_STORE_free obeys reference count */
+static void compat_init_X509_STORE_onfree(void *store, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+        /* unfortunately there's no way to remove a handler */
+        if (store != compat.tmp.store)
+                return;
+
+        /* signal that we were freed by nulling our reference */
+        compat.tmp.store = NULL;
+} /* compat_init_X509_STORE_onfree() */
+
+static int compat_init(void) {
+        static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+        static volatile sig_atomic_t done;
+        int error;
+
+        if ((error = pthread_mutex_lock(&mutex)))
+                return error;
+
+        if (!done) {
+                /*
+                 * Test if X509_STORE_free obeys reference counts.
+                 */
+                if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, 0, NULL, NULL, NULL, &compat_init_X509_STORE_onfree))
+                        goto sslerr;
+
+                if (!(compat.tmp.store = X509_STORE_new()))
+                        goto sslerr;
+
+                CRYPTO_add(&compat.tmp.store->references, 1, CRYPTO_LOCK_X509_STORE);
+                X509_STORE_free(compat.tmp.store);
+
+                if (compat.tmp.store) {
+                        X509_STORE_free(compat.tmp.store);
+                        assert(compat.tmp.store == NULL);
+                        compat.tmp.store = NULL;
+                } else {
+                        /*
+                         * If X509_STORE_free does NOT obey reference
+                         * counts, then make sure that our fixed version is
+                         * called on SSL_CTX destruction. Note that this
+                         * won't fix code which doesn't properly obey the
+                         * reference counts when setting the cert_store
+                         * member.
+                         */
+                        if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_SSL_CTX, 0, NULL, NULL, NULL, &compat_SSL_CTX_onfree))
+                                goto sslerr;
+
+                        compat.flags |= COMPAT_X509_STORE_FREE_BUG;
+                }
+
+                done = 1;
+        }
+epilog:
+        if (compat.tmp.store) {
+                X509_STORE_free(compat.tmp.store);
+                compat.tmp.store = NULL;
+        }
+
+        pthread_mutex_unlock(&mutex);
+
+        return 0;
+sslerr:
+        error = -1;
+
+        goto epilog;
+syserr:
+        error = errno;
+
+        goto epilog;
+} /* compat_init() */
+
 
 typedef int auxref_t;
 typedef int auxtype_t;
@@ -4699,8 +4832,7 @@ static int sx_setStore(lua_State *L) {
         SSL_CTX *ctx = checksimple(L, 1, SSL_CTX_CLASS);
         X509_STORE *store = checksimple(L, 2, X509_STORE_CLASS);
 
-        SSL_CTX_set_cert_store(ctx, store);
-        CRYPTO_add(&store->references, 1, CRYPTO_LOCK_X509_STORE);
+        SSL_CTX_set1_cert_store(ctx, store);
 
         lua_pushboolean(L, 1);
 
@@ -6251,6 +6383,15 @@ static void initall(lua_State *L) {
                 }
         }
 
+        /* TODO: Move down to after SSL_load_error_strings */
+        if ((error = compat_init())) {
+                if (error == -1) {
+                        throwssl(L, "openssl.init");
+                } else {
+                        luaL_error(L, "openssl.init: %s", xstrerror(error));
+                }
+        }
+
         pthread_mutex_lock(&mutex);
 
         if (!initssl) {