add and use pkey:getDefaultDigestName because the old digest type names used in examples/vrfy.sig are not accepted by OpenSSL 1.1

2 files changed, 38 insertions, 16 deletions

diff --git a/examples/vrfy.sig b/examples/vrfy.sig
index 258490a..f6cc927 100755
--- a/examples/vrfy.sig
+++ b/examples/vrfy.sig
@@ -13,16 +13,18 @@ local digest = require"openssl.digest"
 local function genkey(type)
         type = string.upper(type or (not openssl.NO_EC and "EC") or "RSA")
 
+        local key
         if type == "RSA" then
-                return pkey.new{ type = "RSA", bits = 1024 }, "sha256"
+                return pkey.new{ type = "RSA", bits = 1024 }
         elseif type == "DSA" then
-                return pkey.new{ type = "DSA", bits = 1024 }, "dss1"
+                return pkey.new{ type = "DSA", bits = 1024 }
         else
-                return pkey.new{ type = "EC", curve = "prime192v1" }, "ecdsa-with-SHA1"
+                return pkey.new{ type = "EC", curve = "prime192v1" }
         end
 end
 
-local key, hash = genkey(keytype)
+local key = genkey(keytype)
+local hash = key:getDefaultDigestName()
 
 -- digest our message using an appropriate digest ("ecdsa-with-SHA1" for EC;
 -- "dss1" for DSA; and "sha1", "sha256", etc for RSA).
@@ -45,6 +47,7 @@ local function tohex(b)
         return x
 end
 
-print("okay", pub:verify(sig, data))
-print("type", pub:type())
-print("sig", tohex(sig))
+print("verified", pub:verify(sig, data))
+print("key-type", pub:type())
+print("hash-type", hash)
+print("signature", tohex(sig))
diff --git a/src/openssl.c b/src/openssl.c
index 2fb7367..88c34d8 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -1336,7 +1336,7 @@ static int compat_EVP_PKEY_get_default_digest_nid(EVP_PKEY *key, int *nid) {
                 *nid = EVP_MD_nid(EVP_ecdsa());
                 break;
         default:
-                *nid = EVP_MD_nid(EVP_md_null());
+                *nid = EVP_MD_nid(EVP_sha1());
                 break;
         }
 
@@ -3391,6 +3391,26 @@ static int pk_toPEM(lua_State *L) {
 } /* pk_toPEM() */
 
 
+static int pk_getDefaultDigestName(lua_State *L) {
+        EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+        int nid;
+        char txt[256];
+        size_t len;
+
+        if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
+                return auxL_error(L, auxL_EOPENSSL, "pkey:getDefaultDigestName");
+
+        if (!(len = auxS_nid2txt(txt, sizeof txt, nid)))
+                return auxL_error(L, auxL_EOPENSSL, "pkey:getDefaultDigestName");
+        if (len > sizeof txt)
+                return auxL_error(L, EOVERFLOW, "pkey:getDefaultDigestName");
+
+        lua_pushlstring(L, txt, len);
+
+        return 1;
+} /* pk_getDefaultDigestName() */
+
+
 enum pk_param  {
 #define PK_RSA_OPTLIST { "n", "e", "d", "p", "q", "dmp1", "dmq1", "iqmp", NULL }
 #define PK_RSA_OPTOFFSET PK_RSA_N
@@ -3944,6 +3964,7 @@ static const auxL_Reg pk_methods[] = {
         { "setPrivateKey", &pk_setPrivateKey },
         { "sign",          &pk_sign },
         { "verify",        &pk_verify },
+        { "getDefaultDigestName", &pk_getDefaultDigestName },
         { "toPEM",         &pk_toPEM },
         { "getParameters", &pk_getParameters },
         { "setParameters", &pk_setParameters },
@@ -5730,18 +5751,16 @@ static const EVP_MD *xc_signature(lua_State *L, int index, EVP_PKEY *key) {
         if ((id = luaL_optstring(L, index, NULL))) {
                 if (!(md = EVP_get_digestbyname(id)))
                         goto unknown;
-
-                return md;
+        } else {
+                if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
+                        goto unknown;
+                if (!(md = EVP_get_digestbynid(nid)))
+                        goto unknown;
         }
 
-        if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
-                goto unknown;
-        if (!(md = EVP_get_digestbynid(nid)))
-                goto unknown;
-
         return md;
 unknown:
-        return EVP_md_null();
+        return EVP_sha1();
 } /* xc_signature() */
 
 static int xc_sign(lua_State *L) {