Merge a few additional X509error(ERR_R_MALLOC_FAILURE) calls

and various style improvements from the OpenSSL 1.1.1 branch,
which is still under a free license.

- No need to #include <openssl/lhash.h>.
- BUF_MEM_free(3) and sk_pop_free(3) can handle NULL.
- sk_value(3) can handle -1.
- Test pointers with "== NULL" rather than with "!".
- Use the safer "p = malloc(sizeof(*p))" idiom.
- return is not a function.
- Delete very wrong commented out code.

Including parts of the these commits from the 2015 to 2018 time range:
25aaa98a b4faea50 90945fa3 f32b0abe 26a7d938 7fcdbd83 208056b2 5b37fef0

Requested by and OK tb@.

1 files changed, 28 insertions, 39 deletions

diff --git a/src/lib/libcrypto/x509/by_dir.c b/src/lib/libcrypto/x509/by_dir.c
index 0ff60644f5..fa05f552f9 100644
--- a/src/lib/libcrypto/x509/by_dir.c
+++ b/src/lib/libcrypto/x509/by_dir.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: by_dir.c,v 1.40 2021/11/01 20:53:08 tb Exp $ */
+/* $OpenBSD: by_dir.c,v 1.41 2021/11/10 14:34:21 schwarze Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -68,7 +68,6 @@
 #include <openssl/opensslconf.h>
 
 #include <openssl/err.h>
-#include <openssl/lhash.h>
 #include <openssl/x509.h>
 
 #include "x509_lcl.h"
@@ -116,7 +115,7 @@ static X509_LOOKUP_METHOD x509_dir_lookup = {
 X509_LOOKUP_METHOD *
 X509_LOOKUP_hash_dir(void)
 {
-        return (&x509_dir_lookup);
+        return &x509_dir_lookup;
 }
 
 static int
@@ -140,7 +139,7 @@ dir_ctrl(X509_LOOKUP *ctx, int cmd, const char *argp, long argl,
                         ret = add_cert_dir(ld, argp, (int)argl);
                 break;
         }
-        return (ret);
+        return ret;
 }
 
 static int
@@ -148,15 +147,18 @@ new_dir(X509_LOOKUP *lu)
 {
         BY_DIR *a;
 
-        if ((a = malloc(sizeof(BY_DIR))) == NULL)
-                return (0);
+        if ((a = malloc(sizeof(*a))) == NULL) {
+                X509error(ERR_R_MALLOC_FAILURE);
+                return 0;
+        }
         if ((a->buffer = BUF_MEM_new()) == NULL) {
+                X509error(ERR_R_MALLOC_FAILURE);
                 free(a);
-                return (0);
+                return 0;
         }
         a->dirs = NULL;
         lu->method_data = (char *)a;
-        return (1);
+        return 1;
 }
 
 static void
@@ -180,8 +182,7 @@ static void
 by_dir_entry_free(BY_DIR_ENTRY *ent)
 {
         free(ent->dir);
-        if (ent->hashes)
-                sk_BY_DIR_HASH_pop_free(ent->hashes, by_dir_hash_free);
+        sk_BY_DIR_HASH_pop_free(ent->hashes, by_dir_hash_free);
         free(ent);
 }
 
@@ -191,10 +192,8 @@ free_dir(X509_LOOKUP *lu)
         BY_DIR *a;
 
         a = (BY_DIR *)lu->method_data;
-        if (a->dirs != NULL)
-                sk_BY_DIR_ENTRY_pop_free(a->dirs, by_dir_entry_free);
-        if (a->buffer != NULL)
-                BUF_MEM_free(a->buffer);
+        sk_BY_DIR_ENTRY_pop_free(a->dirs, by_dir_entry_free);
+        BUF_MEM_free(a->buffer);
         free(a);
 }
 
@@ -215,6 +214,7 @@ add_cert_dir(BY_DIR *ctx, const char *dir, int type)
         do {
                 if ((*p == ':') || (*p == '\0')) {
                         BY_DIR_ENTRY *ent;
+
                         ss = s;
                         s = p + 1;
                         len = p - ss;
@@ -230,20 +230,20 @@ add_cert_dir(BY_DIR *ctx, const char *dir, int type)
                                 continue;
                         if (ctx->dirs == NULL) {
                                 ctx->dirs = sk_BY_DIR_ENTRY_new_null();
-                                if (!ctx->dirs) {
+                                if (ctx->dirs == NULL) {
                                         X509error(ERR_R_MALLOC_FAILURE);
                                         return 0;
                                 }
                         }
-                        ent = malloc(sizeof(BY_DIR_ENTRY));
-                        if (!ent) {
+                        ent = malloc(sizeof(*ent));
+                        if (ent == NULL) {
                                 X509error(ERR_R_MALLOC_FAILURE);
                                 return 0;
                         }
                         ent->dir_type = type;
                         ent->hashes = sk_BY_DIR_HASH_new(by_dir_hash_cmp);
                         ent->dir = strndup(ss, (size_t)len);
-                        if (!ent->dir || !ent->hashes) {
+                        if (ent->dir == NULL || ent->hashes == NULL) {
                                 X509error(ERR_R_MALLOC_FAILURE);
                                 by_dir_entry_free(ent);
                                 return 0;
@@ -281,7 +281,7 @@ get_cert_by_subject(X509_LOOKUP *xl, int type, X509_NAME *name,
         const char *postfix="";
 
         if (name == NULL)
-                return (0);
+                return 0;
 
         stmp.type = type;
         if (type == X509_LU_X509) {
@@ -311,6 +311,7 @@ get_cert_by_subject(X509_LOOKUP *xl, int type, X509_NAME *name,
                 BY_DIR_ENTRY *ent;
                 int idx;
                 BY_DIR_HASH htmp, *hent;
+
                 ent = sk_BY_DIR_ENTRY_value(ctx->dirs, i);
                 j = strlen(ent->dir) + 1 + 8 + 6 + 1 + 1;
                 if (!BUF_MEM_grow(b, j)) {
@@ -359,10 +360,7 @@ get_cert_by_subject(X509_LOOKUP *xl, int type, X509_NAME *name,
                 /* we have added it to the cache so now pull it out again */
                 CRYPTO_w_lock(CRYPTO_LOCK_X509_STORE);
                 j = sk_X509_OBJECT_find(xl->store_ctx->objs, &stmp);
-                if (j != -1)
-                        tmp = sk_X509_OBJECT_value(xl->store_ctx->objs, j);
-                else
-                        tmp = NULL;
+                tmp = sk_X509_OBJECT_value(xl->store_ctx->objs, j);
                 CRYPTO_w_unlock(CRYPTO_LOCK_X509_STORE);
 
                 /* If a CRL, update the last file suffix added for this */
@@ -372,16 +370,14 @@ get_cert_by_subject(X509_LOOKUP *xl, int type, X509_NAME *name,
                          * Look for entry again in case another thread added
                          * an entry first.
                          */
-                        if (!hent) {
+                        if (hent == NULL) {
                                 htmp.hash = h;
                                 idx = sk_BY_DIR_HASH_find(ent->hashes, &htmp);
-                                if (idx >= 0)
-                                        hent = sk_BY_DIR_HASH_value(
-                                            ent->hashes, idx);
+                                hent = sk_BY_DIR_HASH_value(ent->hashes, idx);
                         }
-                        if (!hent) {
-                                hent = malloc(sizeof(BY_DIR_HASH));
-                                if (!hent) {
+                        if (hent == NULL) {
+                                hent = malloc(sizeof(*hent));
+                                if (hent == NULL) {
                                         X509error(ERR_R_MALLOC_FAILURE);
                                         CRYPTO_w_unlock(CRYPTO_LOCK_X509_STORE);
                                         ok = 0;
@@ -407,17 +403,10 @@ get_cert_by_subject(X509_LOOKUP *xl, int type, X509_NAME *name,
                         ok = 1;
                         ret->type = tmp->type;
                         memcpy(&ret->data, &tmp->data, sizeof(ret->data));
-                        /*
-                         * If we were going to up the reference count,
-                         * we would need to do it on a perl 'type' basis
-                         */
-        /*              CRYPTO_add(&tmp->data.x509->references,1,
-                                CRYPTO_LOCK_X509);*/
                         goto finish;
                 }
         }
 finish:
-        if (b != NULL)
-                BUF_MEM_free(b);
-        return (ok);
+        BUF_MEM_free(b);
+        return ok;
 }