Merge branch '58-pkey-encrypt+decrypt' of https://github.com/daurnimator/luaossl into daurnimator-58-pkey-encrypt+decrypt

1 files changed, 129 insertions, 0 deletions

diff --git a/src/openssl.c b/src/openssl.c
index c33d934..8cf79c0 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -3282,6 +3282,122 @@ static int pk_setPrivateKey(lua_State *L) {
 } /* pk_setPrivateKey() */
 
 
+static int pk_decrypt(lua_State *L) {
+        size_t outlen, inlen;
+        EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+        EVP_PKEY_CTX *ctx;
+        const char *str = luaL_checklstring(L, 2, &inlen);
+        BIO *bio;
+        BUF_MEM *buf;
+        int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+        int base_type = EVP_PKEY_base_id(key);
+
+        if (lua_istable(L, 3)) {
+                if (base_type == EVP_PKEY_RSA) {
+                        lua_getfield(L, 3, "rsaPadding");
+                        rsaPadding = luaL_optint(L, -1, rsaPadding);
+                        lua_pop(L, 1);
+                }
+        }
+
+        bio = getbio(L);
+        BIO_get_mem_ptr(bio, &buf);
+
+        if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt_init(ctx) <= 0)
+                goto sslerr;
+
+        if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt(ctx, NULL, &outlen, str, inlen) <= 0)
+                goto sslerr;
+
+        if (!BUF_MEM_grow_clean(buf, outlen))
+                goto sslerr;
+
+        if (EVP_PKEY_decrypt(ctx, buf->data, &outlen, str, inlen) <= 0)
+                goto sslerr;
+
+        EVP_PKEY_CTX_free(ctx);
+        ctx = NULL;
+
+        lua_pushlstring(L, buf->data, outlen);
+
+        BIO_reset(*bio);
+
+        return 1;
+sslerr:
+        if (ctx) {
+                EVP_PKEY_CTX_free(ctx);
+                ctx = NULL;
+        }
+        BIO_reset(*bio);
+
+        return auxL_error(L, auxL_EOPENSSL, "pkey:decrypt");
+} /* pk_decrypt() */
+
+
+static int pk_encrypt(lua_State *L) {
+        size_t outlen, inlen;
+        EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+        EVP_PKEY_CTX *ctx;
+        const char *str = luaL_checklstring(L, 2, &inlen);
+        BIO *bio;
+        BUF_MEM *buf;
+        int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+        int base_type = EVP_PKEY_base_id(key);
+
+        if (lua_istable(L, 3)) {
+                if (base_type == EVP_PKEY_RSA) {
+                        lua_getfield(L, 3, "rsaPadding");
+                        rsaPadding = luaL_optint(L, -1, rsaPadding);
+                        lua_pop(L, 1);
+                }
+        }
+
+        bio = getbio(L);
+        BIO_get_mem_ptr(bio, &buf);
+
+        if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt_init(ctx) <= 0)
+                goto sslerr;
+
+        if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt(ctx, NULL, &outlen, str, inlen) <= 0)
+                goto sslerr;
+
+        if (!BUF_MEM_grow_clean(buf, outlen))
+                goto sslerr;
+
+        if (EVP_PKEY_encrypt(ctx, buf->data, &outlen, str, inlen) <= 0)
+                goto sslerr;
+
+        EVP_PKEY_CTX_free(ctx);
+        ctx = NULL;
+
+        lua_pushlstring(L, buf->data, outlen);
+
+        BIO_reset(*bio);
+
+        return 1;
+sslerr:
+        if (ctx) {
+                EVP_PKEY_CTX_free(ctx);
+                ctx = NULL;
+        }
+        BIO_reset(*bio);
+
+        return auxL_error(L, auxL_EOPENSSL, "pkey:encrypt");
+} /* pk_encrypt() */
+
+
 static int pk_sign(lua_State *L) {
         EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
         EVP_MD_CTX *md = checksimple(L, 2, DIGEST_CLASS);
@@ -4001,6 +4117,8 @@ static const auxL_Reg pk_methods[] = {
         { "type",          &pk_type },
         { "setPublicKey",  &pk_setPublicKey },
         { "setPrivateKey", &pk_setPrivateKey },
+        { "decrypt",       &pk_decrypt },
+        { "encrypt",       &pk_encrypt },
         { "sign",          &pk_sign },
         { "verify",        &pk_verify },
         { "getDefaultDigestName", &pk_getDefaultDigestName },
@@ -4039,10 +4157,21 @@ static void pk_luainit(lua_State *L, _Bool reset) {
         lua_pop(L, 2);
 } /* pk_luainit() */
 
+static const auxL_IntegerReg pk_rsa_pad_opts[] = {
+        { "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING }, // PKCS#1 padding
+        { "RSA_SSLV23_PADDING", RSA_SSLV23_PADDING }, // SSLv23 padding
+        { "RSA_NO_PADDING", RSA_NO_PADDING }, // no padding
+        { "RSA_PKCS1_OAEP_PADDING", RSA_PKCS1_OAEP_PADDING }, // OAEP padding (encrypt and decrypt only)
+        { "RSA_X931_PADDING", RSA_X931_PADDING }, // (signature operations only)
+        { "RSA_PKCS1_PSS_PADDING", RSA_PKCS1_PSS_PADDING }, // (sign and verify only)
+        { NULL, 0 },
+};
+
 int luaopen__openssl_pkey(lua_State *L) {
         initall(L);
 
         auxL_newlib(L, pk_globals, 0);
+        auxL_setintegers(L, pk_rsa_pad_opts);
 
         return 1;
 } /* luaopen__openssl_pkey() */