1 files changed, 142 insertions, 0 deletions

diff --git a/src/openssl.c b/src/openssl.c
index c33d934..38c9888 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -193,6 +193,10 @@
 #define HAVE_EVP_PKEY_BASE_ID OPENSSL_PREREQ(1,1,0)
 #endif
 
+#ifndef HAVE_EVP_PKEY_CTX_NEW
+#define HAVE_EVP_PKEY_CTX_NEW (OPENSSL_PREREQ(1,0,0) || LIBRESSL_PREREQ(2,0,0))
+#endif
+
 #ifndef HAVE_EVP_PKEY_GET0
 #define HAVE_EVP_PKEY_GET0 OPENSSL_PREREQ(1,1,0)
 #endif
@@ -225,6 +229,10 @@
 #define HAVE_RSA_GET0_KEY OPENSSL_PREREQ(1,1,0)
 #endif
 
+#ifndef HAVE_RSA_PKCS1_PSS_PADDING
+#define HAVE_RSA_PKCS1_PSS_PADDING (defined RSA_PKCS1_PSS_PADDING || OPENSSL_PREREQ(1,0,0) || LIBRESSL_PREREQ(2,0,0))
+#endif
+
 #ifndef HAVE_RSA_SET0_CRT_PARAMS
 #define HAVE_RSA_SET0_CRT_PARAMS OPENSSL_PREREQ(1,1,0)
 #endif
@@ -3281,6 +3289,123 @@ static int pk_setPrivateKey(lua_State *L) {
         return 1;
 } /* pk_setPrivateKey() */
 
+#if HAVE_EVP_PKEY_CTX_NEW
+static int pk_decrypt(lua_State *L) {
+        size_t outlen, inlen;
+        EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+        EVP_PKEY_CTX *ctx;
+        const char *str = luaL_checklstring(L, 2, &inlen);
+        BIO *bio;
+        BUF_MEM *buf;
+        int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+        int base_type = EVP_PKEY_base_id(key);
+
+        if (lua_istable(L, 3)) {
+                if (base_type == EVP_PKEY_RSA) {
+                        lua_getfield(L, 3, "rsaPadding");
+                        rsaPadding = luaL_optint(L, -1, rsaPadding);
+                        lua_pop(L, 1);
+                }
+        }
+
+        bio = getbio(L);
+        BIO_get_mem_ptr(bio, &buf);
+
+        if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt_init(ctx) <= 0)
+                goto sslerr;
+
+        if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt(ctx, NULL, &outlen, (const unsigned char *)str, inlen) <= 0)
+                goto sslerr;
+
+        if (!BUF_MEM_grow_clean(buf, outlen))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt(ctx, (unsigned char *)buf->data, &outlen, (const unsigned char *)str, inlen) <= 0)
+                goto sslerr;
+
+        EVP_PKEY_CTX_free(ctx);
+        ctx = NULL;
+
+        lua_pushlstring(L, buf->data, outlen);
+
+        BIO_reset(bio);
+
+        return 1;
+sslerr:
+        if (ctx) {
+                EVP_PKEY_CTX_free(ctx);
+                ctx = NULL;
+        }
+        BIO_reset(bio);
+
+        return auxL_error(L, auxL_EOPENSSL, "pkey:decrypt");
+} /* pk_decrypt() */
+#endif
+
+#if HAVE_EVP_PKEY_CTX_NEW
+static int pk_encrypt(lua_State *L) {
+        size_t outlen, inlen;
+        EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+        EVP_PKEY_CTX *ctx;
+        const char *str = luaL_checklstring(L, 2, &inlen);
+        BIO *bio;
+        BUF_MEM *buf;
+        int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+        int base_type = EVP_PKEY_base_id(key);
+
+        if (lua_istable(L, 3)) {
+                if (base_type == EVP_PKEY_RSA) {
+                        lua_getfield(L, 3, "rsaPadding");
+                        rsaPadding = luaL_optint(L, -1, rsaPadding);
+                        lua_pop(L, 1);
+                }
+        }
+
+        bio = getbio(L);
+        BIO_get_mem_ptr(bio, &buf);
+
+        if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt_init(ctx) <= 0)
+                goto sslerr;
+
+        if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt(ctx, NULL, &outlen, (const unsigned char *)str, inlen) <= 0)
+                goto sslerr;
+
+        if (!BUF_MEM_grow_clean(buf, outlen))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt(ctx, (unsigned char *)buf->data, &outlen, (const unsigned char *)str, inlen) <= 0)
+                goto sslerr;
+
+        EVP_PKEY_CTX_free(ctx);
+        ctx = NULL;
+
+        lua_pushlstring(L, buf->data, outlen);
+
+        BIO_reset(bio);
+
+        return 1;
+sslerr:
+        if (ctx) {
+                EVP_PKEY_CTX_free(ctx);
+                ctx = NULL;
+        }
+        BIO_reset(bio);
+
+        return auxL_error(L, auxL_EOPENSSL, "pkey:encrypt");
+} /* pk_encrypt() */
+#endif
 
 static int pk_sign(lua_State *L) {
         EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
@@ -4001,6 +4126,10 @@ static const auxL_Reg pk_methods[] = {
         { "type",          &pk_type },
         { "setPublicKey",  &pk_setPublicKey },
         { "setPrivateKey", &pk_setPrivateKey },
+#if HAVE_EVP_PKEY_CTX_NEW
+        { "decrypt",       &pk_decrypt },
+        { "encrypt",       &pk_encrypt },
+#endif
         { "sign",          &pk_sign },
         { "verify",        &pk_verify },
         { "getDefaultDigestName", &pk_getDefaultDigestName },
@@ -4039,10 +4168,23 @@ static void pk_luainit(lua_State *L, _Bool reset) {
         lua_pop(L, 2);
 } /* pk_luainit() */
 
+static const auxL_IntegerReg pk_rsa_pad_opts[] = {
+        { "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING }, // PKCS#1 padding
+        { "RSA_SSLV23_PADDING", RSA_SSLV23_PADDING }, // SSLv23 padding
+        { "RSA_NO_PADDING", RSA_NO_PADDING }, // no padding
+        { "RSA_PKCS1_OAEP_PADDING", RSA_PKCS1_OAEP_PADDING }, // OAEP padding (encrypt and decrypt only)
+        { "RSA_X931_PADDING", RSA_X931_PADDING }, // (signature operations only)
+#if HAVE_RSA_PKCS1_PSS_PADDING
+        { "RSA_PKCS1_PSS_PADDING", RSA_PKCS1_PSS_PADDING }, // (sign and verify only)
+#endif
+        { NULL, 0 },
+};
+
 int luaopen__openssl_pkey(lua_State *L) {
         initall(L);
 
         auxL_newlib(L, pk_globals, 0);
+        auxL_setintegers(L, pk_rsa_pad_opts);
 
         return 1;
 } /* luaopen__openssl_pkey() */