From 7d5e6890156b686b6b3e771d9efd343ce8df5fa9 Mon Sep 17 00:00:00 2001
From: tb <>
Date: Sat, 10 May 2025 05:25:43 +0000
Subject: cms: disallow AEAD ciphers and AES XTS

The CMS code doesn't support RFC 5083/5084 authenticated enveloped data
and outputs garbage that even itself can't decrypt for a reason that I
have not tried to pinpoint. So refuse using AEAD ciphers and AES XTS for
enveloped data from the cms "app" and throw an error pointing out that
this isn't supported.

OpenSSL have since added incorrect support for AuthEnvelopedData (ASN.1
and code review are hard), so doing this right will need both correct and
interoperable code, which I doubt anyone will bother to write anytime soon.

Reported by Ben Cooper in https://github.com/libressl/portable/issues/1157

ok beck jsing
---
 src/usr.bin/openssl/cms.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/usr.bin/openssl/cms.c b/src/usr.bin/openssl/cms.c
index 7420d0ab8c..8e5015feba 100644
--- a/src/usr.bin/openssl/cms.c
+++ b/src/usr.bin/openssl/cms.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: cms.c,v 1.36 2024/08/12 15:34:58 job Exp $ */
+/* $OpenBSD: cms.c,v 1.37 2025/05/10 05:25:43 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project.
  */
@@ -193,15 +193,33 @@ get_cipher_by_name(char *name)
 static int
 cms_opt_cipher(int argc, char **argv, int *argsused)
 {
+	const EVP_CIPHER *cipher;
 	char *name = argv[0];
 
 	if (*name++ != '-')
 		return (1);
 
-	if ((cfg.cipher = get_cipher_by_name(name)) == NULL)
-		if ((cfg.cipher = EVP_get_cipherbyname(name)) == NULL)
+	if ((cipher = get_cipher_by_name(name)) == NULL)
+		if ((cipher = EVP_get_cipherbyname(name)) == NULL)
 			return (1);
 
+	/*
+	 * XXX - this should really be done in CMS_{encrypt,decrypt}() until
+	 * we have proper support for AuthEnvelopedData (RFC 5084), but this
+	 * is good enough for now to avoid outputting garbage with this rusty
+	 * swiss army knife.
+	 */
+	if ((EVP_CIPHER_flags(cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) != 0) {
+		BIO_printf(bio_err, "AuthEnvelopedData is not supported\n");
+		return (1);
+	}
+	if (EVP_CIPHER_mode(cipher) == EVP_CIPH_XTS_MODE) {
+		BIO_printf(bio_err, "XTS mode not supported\n");
+		return (1);
+	}
+
+	cfg.cipher = cipher;
+
 	*argsused = 1;
 	return (0);
 }
-- 
cgit v1.2.3-55-g6feb