pin the module in memory when installing external app data callbacks, as these can be never be uninstalled

1 files changed, 234 insertions, 124 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 90db798..0894fcf 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -23,9 +23,6 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  * ==========================================================================
  */
-#ifndef LUAOSSL_H
-#define LUAOSSL_H
-
 #include <limits.h>       /* INT_MAX INT_MIN UCHAR_MAX */
 #include <stdint.h>       /* uintptr_t */
 #include <string.h>       /* memset(3) strerror_r(3) */
@@ -33,8 +30,7 @@
 #include <math.h>         /* INFINITY fabs(3) floor(3) frexp(3) fmod(3) round(3) isfinite(3) */
 #include <time.h>         /* struct tm time_t strptime(3) time(2) */
 #include <ctype.h>        /* tolower(3) */
-#include <signal.h>       /* sig_atomic_t */
-#include <errno.h>        /* ENOMEM errno */
+#include <errno.h>        /* ENOMEM ENOTSUP errno */
 #include <assert.h>       /* assert */
 
 #include <sys/types.h>    /* ssize_t pid_t */
@@ -80,6 +76,10 @@
 #include "compat52.h"
 #endif
 
+#ifndef HAVE_DLADDR
+#define HAVE_DLADDR (!defined _AIX) /* TODO: https://root.cern.ch/drupal/content/aix-and-dladdr */
+#endif
+
 #ifndef HAVE_SSL_CTX_SET_ALPN_PROTOS
 #define HAVE_SSL_CTX_SET_ALPN_PROTOS (OPENSSL_VERSION_NUMBER >= 0x1000200fL)
 #endif
@@ -573,6 +573,83 @@ static void lib_setintegers(lua_State *L, const integer_Reg *l) {
 } /* lib_setintegers() */
 
 
+/*
+ * Auxiliary Lua API routines
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+typedef int auxref_t;
+typedef int auxtype_t;
+
+static void auxL_unref(lua_State *L, auxref_t *ref) {
+        luaL_unref(L, LUA_REGISTRYINDEX, *ref);
+        *ref = LUA_NOREF;
+} /* auxL_unref() */
+
+static void auxL_ref(lua_State *L, int index, auxref_t *ref) {
+        auxL_unref(L, ref);
+        lua_pushvalue(L, index);
+        *ref = luaL_ref(L, LUA_REGISTRYINDEX);
+} /* auxL_ref() */
+
+static auxtype_t auxL_getref(lua_State *L, auxref_t ref) {
+        if (ref == LUA_NOREF || ref == LUA_REFNIL) {
+                lua_pushnil(L);
+        } else {
+                lua_rawgeti(L, LUA_REGISTRYINDEX, ref);
+        }
+
+        return lua_type(L, -1);
+} /* auxL_getref() */
+
+
+/*
+ * dl - dynamically loaded module management
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Prevent loader from unlinking us if we've registered a callback with
+ * OpenSSL by taking another reference to ourselves.
+ */
+static int dl_anchor(void) {
+#if HAVE_DLADDR
+        extern int luaopen__openssl(lua_State *);
+        static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+        static void *anchor;
+        Dl_info info;
+        int error = 0;
+
+        if ((error = pthread_mutex_lock(&mutex)))
+                return error;
+
+        if (anchor)
+                goto epilog;
+
+        if (!dladdr((void *)&luaopen__openssl, &info))
+                goto dlerr;
+
+        if (!(anchor = dlopen(info.dli_fname, RTLD_NOW|RTLD_LOCAL)))
+                goto dlerr;
+epilog:
+        (void)pthread_mutex_unlock(&mutex);
+
+        return error;
+dlerr:
+        error = -2;
+
+        goto epilog;
+#else
+        return 0;//ENOTSUP;
+#endif
+} /* dl_anchor() */
+
+
+/*
+ * compat - OpenSSL API compatibility and bug workarounds
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 #define COMPAT_X509_STORE_FREE_BUG 0x01
 
 static struct {
@@ -632,7 +709,9 @@ static void *compat_EVP_PKEY_get0(EVP_PKEY *key) {
 } /* compat_EVP_PKEY_get0() */
 #endif
 
-/* X509_STORE_free in OpenSSL versions < 1.0.2 doesn't obey reference count */
+/*
+ * X509_STORE_free in OpenSSL versions < 1.0.2 doesn't obey reference count
+ */
 #define X509_STORE_free(store) \
         (compat.X509_STORE_free)((store))
 
@@ -672,14 +751,14 @@ static void compat_SSL_CTX_set1_cert_store(SSL_CTX *ctx, X509_STORE *store) {
 } /* compat_SSL_CTX_set1_cert_store() */
 #endif
 
-static void compat_SSL_CTX_onfree(void *_ctx, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+static void compat_init_SSL_CTX_onfree(void *_ctx, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
         SSL_CTX *ctx = _ctx;
 
         if (ctx->cert_store) {
                 X509_STORE_free(ctx->cert_store);
                 ctx->cert_store = NULL;
         }
-} /* compat_SSL_CTX_onfree() */
+} /* compat_init_SSL_CTX_onfree() */
 
 /* helper routine to determine if X509_STORE_free obeys reference count */
 static void compat_init_X509_STORE_onfree(void *store, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
@@ -693,90 +772,89 @@ static void compat_init_X509_STORE_onfree(void *store, void *data NOTUSED, CRYPT
 
 static int compat_init(void) {
         static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-        static volatile sig_atomic_t done;
-        int error;
+        static int store_index = -1, ssl_ctx_index = -1, done;
+        X509_STORE *store;
+        int error = 0;
 
         if ((error = pthread_mutex_lock(&mutex)))
                 return error;
 
-        if (!done) {
-                /*
-                 * Test if X509_STORE_free obeys reference counts.
-                 */
-                if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, 0, NULL, NULL, NULL, &compat_init_X509_STORE_onfree))
-                        goto sslerr;
+        if (done)
+                goto epilog;
 
-                if (!(compat.tmp.store = X509_STORE_new()))
-                        goto sslerr;
+        /*
+         * We need to unconditionally install at least one external
+         * application data callback. Because these can never be
+         * uninstalled, we can never be unloaded.
+         */
+        if ((error = dl_anchor()))
+                goto epilog;
 
-                CRYPTO_add(&compat.tmp.store->references, 1, CRYPTO_LOCK_X509_STORE);
-                X509_STORE_free(compat.tmp.store);
+        /*
+         * Test if X509_STORE_free obeys reference counts by installing an
+         * onfree callback.
+         */
+        if (store_index == -1
+        &&  -1 == (store_index = CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, 0, NULL, NULL, NULL, &compat_init_X509_STORE_onfree)))
+                goto sslerr;
 
-                if (compat.tmp.store) {
-                        X509_STORE_free(compat.tmp.store);
-                        assert(compat.tmp.store == NULL);
-                        compat.tmp.store = NULL;
-                } else {
-                        /*
-                         * If X509_STORE_free does NOT obey reference
-                         * counts, then make sure that our fixed version is
-                         * called on SSL_CTX destruction. Note that this
-                         * won't fix code which doesn't properly obey the
-                         * reference counts when setting the cert_store
-                         * member.
-                         */
-                        if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_SSL_CTX, 0, NULL, NULL, NULL, &compat_SSL_CTX_onfree))
-                                goto sslerr;
-
-                        compat.flags |= COMPAT_X509_STORE_FREE_BUG;
-                }
+        if (!(compat.tmp.store = X509_STORE_new()))
+                goto sslerr;
+
+        CRYPTO_add(&compat.tmp.store->references, 1, CRYPTO_LOCK_X509_STORE);
+        X509_STORE_free(compat.tmp.store);
 
-                done = 1;
+        if (compat.tmp.store) {
+                /*
+                 * Because our onfree callback didn't execute, we assume
+                 * X509_STORE_free obeys reference counts. Alternatively,
+                 * our callback might not have executed for some other
+                 * reason. We assert the truth of our assumption by checking
+                 * again after calling X509_STORE_free once more.
+                 */
+                X509_STORE_free(compat.tmp.store);
+                assert(compat.tmp.store == NULL);
+                compat.tmp.store = NULL; /* in case assertions disabled */
+        } else {
+                /*
+                 * Because our onfree callback was invoked, X509_STORE_free
+                 * appears not to obey reference counts. Ensure that our
+                 * fixed version is called on SSL_CTX destruction.
+                 *
+                 * NB: We depend on the coincidental order of operations in
+                 * SSL_CTX_free that user data destruction occurs before
+                 * free'ing the cert_store member. Ruby's OpenSSL bindings
+                 * also depend on this order as we both use the onfree
+                 * callback to clear the member.
+                 */
+                if (ssl_ctx_index == -1
+                &&  -1 == (ssl_ctx_index = CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_SSL_CTX, 0, NULL, NULL, NULL, &compat_init_SSL_CTX_onfree)))
+                        goto sslerr;
+
+                compat.flags |= COMPAT_X509_STORE_FREE_BUG;
         }
+
+        done = 1;
 epilog:
         if (compat.tmp.store) {
                 X509_STORE_free(compat.tmp.store);
                 compat.tmp.store = NULL;
         }
 
-        pthread_mutex_unlock(&mutex);
+        (void)pthread_mutex_unlock(&mutex);
 
-        return 0;
+        return error;
 sslerr:
         error = -1;
 
         goto epilog;
-syserr:
-        error = errno;
-
-        goto epilog;
 } /* compat_init() */
 
 
-typedef int auxref_t;
-typedef int auxtype_t;
-
-static void auxL_unref(lua_State *L, auxref_t *ref) {
-        luaL_unref(L, LUA_REGISTRYINDEX, *ref);
-        *ref = LUA_NOREF;
-} /* auxL_unref() */
-
-static void auxL_ref(lua_State *L, int index, auxref_t *ref) {
-        auxL_unref(L, ref);
-        lua_pushvalue(L, index);
-        *ref = luaL_ref(L, LUA_REGISTRYINDEX);
-} /* auxL_ref() */
-
-static auxtype_t auxL_getref(lua_State *L, auxref_t ref) {
-        if (ref == LUA_NOREF || ref == LUA_REFNIL) {
-                lua_pushnil(L);
-        } else {
-                lua_rawgeti(L, LUA_REGISTRYINDEX, ref);
-        }
-
-        return lua_type(L, -1);
-} /* auxL_getref() */
-
+/*
+ * External Application Data Hooks
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 struct ex_state {
         lua_State *L;
@@ -835,16 +913,43 @@ static void ex_onfree(void *parent NOTUSED, void *_data, CRYPTO_EX_DATA *ad NOTU
         free(data);
 } /* ex_onfree() */
 
-static int ex_initonce(void) {
+static int ex_init(void) {
+        static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+        static int done;
         struct ex_type *type;
+        int error = 0;
+
+        if ((error = pthread_mutex_lock(&mutex)))
+                return error;
+
+        if (done)
+                goto epilog;
+
+        /*
+         * Our callbacks can never be uninstalled, so ensure we're never
+         * unloaded.
+         */
+        if ((error = dl_anchor()))
+                goto epilog;
 
         for (type = ex_type; type < endof(ex_type); type++) {
+                if (type->index != -1)
+                        continue;
+
                 if (-1 == (type->index = CRYPTO_get_ex_new_index(type->class_index, 0, NULL, NULL, &ex_ondup, &ex_onfree)))
-                        return -1;
+                        goto sslerr;
         };
 
-        return 0;
-} /* ex_initonce() */
+        done = 1;
+epilog:
+        (void)pthread_mutex_unlock(&mutex);
+
+        return error;
+sslerr:
+        error = -1;
+
+        goto epilog;
+} /* ex_init() */
 
 static int ex__gc(lua_State *L) {
         struct ex_state *state = lua_touserdata(L, 1);
@@ -967,6 +1072,25 @@ static void initall(lua_State *L);
 
 
 /*
+ * compat - Lua OpenSSL
+ *
+ * Bindings to our internal feature detection, compatability, and workaround
+ * code.
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+int luaopen__openssl_compat(lua_State *L) {
+        initall(L);
+
+        lua_newtable(L);
+        lua_pushboolean(L, !!(compat.flags & COMPAT_X509_STORE_FREE_BUG));
+        lua_setfield(L, -2, "X509_STORE_FREE_BUG");
+
+        return 1;
+} /* luaopen__openssl_compat() */
+
+
+/*
  * OPENSSL - openssl
  *
  * Miscellaneous global interfaces.
@@ -6257,18 +6381,11 @@ int luaopen__openssl_des(lua_State *L) {
  *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-#ifndef HAVE_DLADDR
-#define HAVE_DLADDR (!defined _AIX) /* TODO: https://root.cern.ch/drupal/content/aix-and-dladdr */
-#endif
-
 static struct {
         pthread_mutex_t *lock;
         int nlock;
-
-        void *dlref;
 } mt_state;
 
-
 static void mt_lock(int mode, int type, const char *file NOTUSED, int line NOTUSED) {
         if (mode & CRYPTO_LOCK)
                 pthread_mutex_lock(&mt_state.lock[type]);
@@ -6276,7 +6393,6 @@ static void mt_lock(int mode, int type, const char *file NOTUSED, int line NOTUS
                 pthread_mutex_unlock(&mt_state.lock[type]);
 } /* mt_lock() */
 
-
 /*
  * Sources include Google and especially the Wine Project. See get_unix_tid
  * at http://source.winehq.org/git/wine.git/?a=blob;f=dlls/ntdll/server.c.
@@ -6309,12 +6425,16 @@ static unsigned long mt_gettid(void) {
 #endif
 } /* mt_gettid() */
 
-
 static int mt_init(void) {
         static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-        int bound = 0, error = 0;
+        static int done, bound;
+        int error = 0;
 
-        pthread_mutex_lock(&mutex);
+        if ((error = pthread_mutex_lock(&mutex)))
+                return error;
+
+        if (done)
+                goto epilog;
 
         if (!CRYPTO_get_locking_callback()) {
                 if (!mt_state.lock) {
@@ -6324,11 +6444,20 @@ static int mt_init(void) {
                 
                         if (!(mt_state.lock = malloc(mt_state.nlock * sizeof *mt_state.lock))) {
                                 error = errno;
-                                goto leave;
+                                goto epilog;
                         }
 
                         for (i = 0; i < mt_state.nlock; i++) {
-                                pthread_mutex_init(&mt_state.lock[i], NULL);
+                                if ((error = pthread_mutex_init(&mt_state.lock[i], NULL))) {
+                                        while (i > 0) {
+                                                pthread_mutex_destroy(&mt_state.lock[--i]);
+                                        }
+
+                                        free(mt_state.lock);
+                                        mt_state.lock = NULL;
+
+                                        goto epilog;
+                                }
                         }
                 }
 
@@ -6341,27 +6470,11 @@ static int mt_init(void) {
                 bound = 1;
         }
 
-        /*
-         * Prevent loader from unlinking us if we've registered a callback
-         * with OpenSSL by taking another reference to ourselves.
-         */
-#if HAVE_DLADDR
-        if (bound && !mt_state.dlref) {
-                Dl_info info;
-
-                if (!dladdr((void *)&luaopen__openssl_rand, &info)) {
-                        error = -1;
-                        goto leave;
-                }
+        if (bound && (error = dl_anchor()))
+                goto epilog;
 
-                if (!(mt_state.dlref = dlopen(info.dli_fname, RTLD_NOW|RTLD_LOCAL))) {
-                        error = -1;
-                        goto leave;
-                }
-        }
-#endif
-
-leave:
+        done = 1;
+epilog:
         pthread_mutex_unlock(&mutex);
 
         return error;
@@ -6381,15 +6494,6 @@ static void initall(lua_State *L) {
                 }
         }
 
-        /* TODO: Move down to after SSL_load_error_strings */
-        if ((error = compat_init())) {
-                if (error == -1) {
-                        throwssl(L, "openssl.init");
-                } else {
-                        luaL_error(L, "openssl.init: %s", xstrerror(error));
-                }
-        }
-
         pthread_mutex_lock(&mutex);
 
         if (!initssl) {
@@ -6404,17 +6508,25 @@ static void initall(lua_State *L) {
                  * already been configured.
                  */
                 OPENSSL_config(NULL);
+        }
 
-                if ((error = ex_initonce())) {
-                        if (error == -1) {
-                                throwssl(L, "openssl.init");
-                        } else {
-                                luaL_error(L, "openssl.init: %s", xstrerror(error));
-                        }
+        pthread_mutex_unlock(&mutex);
+
+        if ((error = compat_init())) {
+                if (error == -1) {
+                        throwssl(L, "openssl.init");
+                } else {
+                        luaL_error(L, "openssl.init: %s", xstrerror(error));
                 }
         }
 
-        pthread_mutex_unlock(&mutex);
+        if ((error = ex_init())) {
+                if (error == -1) {
+                        throwssl(L, "openssl.init");
+                } else {
+                        luaL_error(L, "openssl.init: %s", xstrerror(error));
+                }
+        }
 
         ex_newstate(L);
 
@@ -6436,5 +6548,3 @@ static void initall(lua_State *L) {
         addclass(L, CIPHER_CLASS, cipher_methods, cipher_metatable);
 } /* initall() */
 
-
-#endif /* LUAOSSL_H */