refactor and fixup some interfaces, and begin to flesh out ALPN selection callback

1 files changed, 148 insertions, 39 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 4e0b898..3465922 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -424,6 +424,24 @@ static void checkprotos(luaL_Buffer *B, lua_State *L, int index) {
         }
 } /* checkprotos() */
 
+static void pushprotos(lua_State *L, const unsigned char *p, size_t n) {
+        const unsigned char *pe = &p[n];
+        int i = 0;
+
+        lua_newtable(L);
+
+        while (p < pe) {
+                n = *p++;
+
+                if ((size_t)(pe - p) < n)
+                        luaL_error(L, "corrupt ALPN protocol list (%zu > %zu)", n, (size_t)(pe - p));
+
+                lua_pushlstring(L, (const void *)p, n);
+                lua_rawseti(L, -2, ++i);
+                p += n;
+        }
+} /* pushprotos() */
+
 
 static _Bool getfield(lua_State *L, int index, const char *k) {
         lua_getfield(L, index, k);
@@ -604,15 +622,44 @@ static void *compat_EVP_PKEY_get0(EVP_PKEY *key) {
 #endif
 
 
+typedef int auxref_t;
+typedef int auxtype_t;
+
+static void auxL_unref(lua_State *L, auxref_t *ref) {
+        luaL_unref(L, LUA_REGISTRYINDEX, *ref);
+        *ref = LUA_NOREF;
+} /* auxL_unref() */
+
+static void auxL_ref(lua_State *L, int index, auxref_t *ref) {
+        auxL_unref(L, ref);
+        lua_pushvalue(L, index);
+        *ref = luaL_ref(L, LUA_REGISTRYINDEX);
+} /* auxL_ref() */
+
+static auxtype_t auxL_getref(lua_State *L, auxref_t ref) {
+        if (ref == LUA_NOREF || ref == LUA_REFNIL) {
+                lua_pushnil(L);
+        } else {
+                lua_rawgeti(L, LUA_REGISTRYINDEX, ref);
+        }
+
+        return lua_type(L, -1);
+} /* auxL_getref() */
+
+
 struct ex_state {
-        lua_State *mainthread;
+        lua_State *L;
         LIST_HEAD(, ex_data) data;
 }; /* struct ex_state */
 
+#ifndef EX_DATA_MAXARGS
+#define EX_DATA_MAXARGS 4
+#endif
+
 struct ex_data {
         struct ex_state *state;
         int refs;
-        int arg[4];
+        auxref_t arg[EX_DATA_MAXARGS];
         LIST_ENTRY(ex_data) le;
 }; /* struct ex_data */
 
@@ -621,40 +668,47 @@ enum {
 };
 
 static struct ex_type {
-        int class_index;
-        int index;
+        int class_index; /* OpenSSL object type identifier */
+        int index; /* OpenSSL-allocated external data identifier */
         void *(*get_ex_data)();
         int (*set_ex_data)();
 } ex_type[] = {
         [EX_SSL_CTX_ALPN_SELECT_CB] = { CRYPTO_EX_INDEX_SSL_CTX, -1, &SSL_CTX_get_ex_data, &SSL_CTX_set_ex_data },
 };
 
-static int ex_data_dup(CRYPTO_EX_DATA *to NOTUSED, CRYPTO_EX_DATA *from NOTUSED, void *from_d, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+static int ex_ondup(CRYPTO_EX_DATA *to NOTUSED, CRYPTO_EX_DATA *from NOTUSED, void *from_d, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
         struct ex_data **data = from_d;
 
         if (*data)
                 (*data)->refs++;
 
         return 1;
-} /* ex_data_dup() */
+} /* ex_ondup() */
 
-static void ex_data_free(void *parent NOTUSED, void *_data, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+static void ex_onfree(void *parent NOTUSED, void *_data, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
         struct ex_data *data = _data;
 
         if (!data || --data->refs > 0)
                 return;
 
-        if (data->state)
+        if (data->state) {
+                int i;
+
+                for (i = 0; i < (int)countof(data->arg); i++) {
+                        auxL_unref(data->state->L, &data->arg[i]);
+                }
+
                 LIST_REMOVE(data, le);
+        }
 
         free(data);
-} /* ex_data_free() */
+} /* ex_onfree() */
 
 static int ex_initonce(void) {
         struct ex_type *type;
 
         for (type = ex_type; type < endof(ex_type); type++) {
-                if (-1 == (type->index = CRYPTO_get_ex_new_index(type->class_index, 0, NULL, NULL, &ex_data_dup, &ex_data_free)))
+                if (-1 == (type->index = CRYPTO_get_ex_new_index(type->class_index, 0, NULL, NULL, &ex_ondup, &ex_onfree)))
                         return -1;
         };
 
@@ -676,22 +730,27 @@ static int ex__gc(lua_State *L) {
         return 0;
 } /* ex__gc() */
 
-static void ex_init(lua_State *L) {
+static void ex_newstate(lua_State *L) {
         struct ex_state *state;
         struct lua_State *thr;
 
         state = prepudata(L, sizeof *state, NULL, &ex__gc);
         LIST_INIT(&state->data);
 
+        /*
+         * XXX: Don't reuse mainthread because if an error occurs in a
+         * callback Lua might longjmp across the OpenSSL call stack.
+         * Instead, we'll install our own panic handlers.
+         */
 #if defined LUA_RIDX_MAINTHREAD
         lua_rawgeti(L, LUA_REGISTRYINDEX, LUA_RIDX_MAINTHREAD);
-        state->mainthread = lua_tothread(L, -1);
+        state->L = lua_tothread(L, -1);
         lua_pop(L, 1);
 #else
         lua_pushvalue(L, -1);
         thr = lua_newthread(L);
         lua_settable(L, LUA_REGISTRYINDEX);
-        state->mainthread = thr;
+        state->L = thr;
 #endif
 
         lua_pushcfunction(L, &ex__gc);
@@ -699,9 +758,9 @@ static void ex_init(lua_State *L) {
         lua_settable(L, LUA_REGISTRYINDEX);
 
         lua_pop(L, 1);
-} /* ex_init() */
+} /* ex_newstate() */
 
-static struct ex_state *ex_get(lua_State *L) {
+static struct ex_state *ex_getstate(lua_State *L) {
         struct ex_state *state;
 
         lua_pushcfunction(L, &ex__gc);
@@ -712,12 +771,12 @@ static struct ex_state *ex_get(lua_State *L) {
         lua_pop(L, 1);
 
         return state;
-} /* ex_get() */
+} /* ex_getstate() */
 
-static int ex_data_get(lua_State **L, int _type, void *obj) {
+static size_t ex_getdata(lua_State **L, int _type, void *obj) {
         struct ex_type *type = &ex_type[_type];
         struct ex_data *data;
-        int i;
+        size_t i;
 
         if (!(data = type->get_ex_data(obj, type->index)))
                 return 0;
@@ -725,28 +784,31 @@ static int ex_data_get(lua_State **L, int _type, void *obj) {
                 return 0;
 
         if (!*L)
-                *L = data->state->mainthread;
+                *L = data->state->L;
+
+        if (!lua_checkstack(*L, countof(data->arg)))
+                return 0;
 
-        for (i = 0; i < (int)countof(data->arg); i++) {
+        for (i = 0; i < countof(data->arg) && data->arg[i] != LUA_NOREF; i++) {
                 lua_rawgeti(*L, LUA_REGISTRYINDEX, data->arg[i]);
         }
 
         return i;
-} /* ex_data_get() */
+} /* ex_getdata() */
 
-static int ex_data_set(lua_State *L, int _type, void *obj, int n) {
+/* returns 0 on success, otherwise error (>0 == errno, -1 == OpenSSL error) */
+static int ex_setdata(lua_State *L, int _type, void *obj, size_t n) {
         struct ex_type *type = &ex_type[_type];
         struct ex_state *state;
         struct ex_data *data;
-        int i, j;
+        size_t i, j;
 
         if ((data = type->get_ex_data(obj, type->index)) && data->state) {
-                for (i = 0; i < (int)countof(data->arg); i++) {
-                        luaL_unref(L, LUA_REGISTRYINDEX, data->arg[i]);
-                        data->arg[i] = LUA_NOREF;
+                for (i = 0; i < countof(data->arg); i++) {
+                        auxL_unref(L, &data->arg[i]);
                 }
         } else {
-                state = ex_get(L);
+                state = ex_getstate(L);
 
                 if (!(data = malloc(sizeof *data)))
                         return errno;
@@ -756,20 +818,19 @@ static int ex_data_set(lua_State *L, int _type, void *obj, int n) {
 
                 data->state = state;
                 data->refs = 1;
-                for (i = 0; i < (int)countof(data->arg); i++)
+                for (i = 0; i < countof(data->arg); i++)
                         data->arg[i] = LUA_NOREF;
                 LIST_INSERT_HEAD(&state->data, data, le);
         }
 
-        for (i = n, j = 0; i > 0 && j < (int)countof(data->arg); i--, j++) {
-                lua_pushvalue(L, -i);
-                data->arg[j] = luaL_ref(L, LUA_REGISTRYINDEX);
+        for (i = n, j = 0; i > 0 && j < countof(data->arg); i--, j++) {
+                auxL_ref(L, -(int)i, &data->arg[j]);
         }
 
         lua_pop(L, n);
 
         return 0;
-} /* ex_data_set() */
+} /* ex_setdata() */
 
 static void initall(lua_State *L);
 
@@ -4800,13 +4861,44 @@ static int sx_setAlpnProtos(lua_State *L) {
 #endif
 
 #if HAVE_SSL_CTX_SET_ALPN_SELECT_CB
-static int sx_setAlpnSelect_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) {
+static SSL *ssl_push(lua_State *, SSL *);
+
+static int sx_setAlpnSelect_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *_ctx) {
+        SSL_CTX *ctx = _ctx;
         lua_State *L = NULL;
-        int n;
+        size_t n;
+        int top, status;
 
-        n = ex_data_get(&L, EX_SSL_CTX_ALPN_SELECT_CB, ssl);
+        if (0 == (n = ex_getdata(&L, EX_SSL_CTX_ALPN_SELECT_CB, ctx)))
+                return SSL_TLSEXT_ERR_ALERT_FATAL;
 
-        return 0;
+        top = lua_gettop(L) - n;
+
+        /* TODO: Install temporary panic handler to catch OOM errors */
+
+        /* pass the SSL object as first argument */
+        ssl_push(L, ssl);
+        pushprotos(L, in, inlen);
+
+        /* TODO: lua_rotate ssl and protocols table into position. */
+
+        if (LUA_OK != (status = lua_pcall(L, 2 + (n - 1), 1, 0)))
+                goto fatal;
+
+        /* TODO: check return value */
+        (void)out; (void)outlen;
+
+        lua_settop(L, top);
+
+        return SSL_TLSEXT_ERR_OK;
+fatal:
+        lua_settop(L, top);
+
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
+noack:
+        lua_settop(L, top);
+
+        return SSL_TLSEXT_ERR_NOACK;
 } /* sx_setAlpnSelect_cb() */
 
 static int sx_setAlpnSelect(lua_State *L) {
@@ -4815,9 +4907,17 @@ static int sx_setAlpnSelect(lua_State *L) {
         int error;
 
         luaL_checktype(L, 2, LUA_TFUNCTION);
-        error = ex_data_set(L, EX_SSL_CTX_ALPN_SELECT_CB, ctx, 1);
+        if ((error = ex_setdata(L, EX_SSL_CTX_ALPN_SELECT_CB, ctx, 1))) {
+                if (error > 0) {
+                        return luaL_error(L, "unable to set ALPN protocol selection callback: %s", xstrerror(error));
+                } else if (!ERR_peek_error()) {
+                        return luaL_error(L, "unable to set ALPN protocol selection callback: Unknown internal error");
+                } else {
+                        return throwssl(L, "ssl.context:setAlpnSelect");
+                }
+        }
 
-        SSL_CTX_set_alpn_select_cb(ctx, &sx_setAlpnSelect_cb, NULL);
+        SSL_CTX_set_alpn_select_cb(ctx, &sx_setAlpnSelect_cb, ctx);
 
         lua_pushboolean(L, 1);
 
@@ -4936,6 +5036,15 @@ int luaopen__openssl_ssl_context(lua_State *L) {
  *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static SSL *ssl_push(lua_State *L, SSL *ssl) {
+        SSL **ud = prepsimple(L, SSL_CLASS);
+
+        CRYPTO_add(&(ssl)->references, 1, CRYPTO_LOCK_SSL);
+        *ud = ssl;
+
+        return *ud;
+} /* ssl_push() */
+
 static int ssl_new(lua_State *L) {
         lua_pushnil(L);
 
@@ -6168,7 +6277,7 @@ static void initall(lua_State *L) {
 
         pthread_mutex_unlock(&mutex);
 
-        ex_init(L);
+        ex_newstate(L);
 
         addclass(L, BIGNUM_CLASS, bn_methods, bn_metatable);
         addclass(L, PKEY_CLASS, pk_methods, pk_metatable);