Use C functions for RC4 public API.

Rather than having public API switch between C and assembly, always
use C functions as entry points, which then call an assembly
implementation (if available). This makes it significantly easier
to deal with symbol aliasing/namespaces and it also means we
benefit from vulnerability prevention provided by the C compiler.

Rename the assembly generated functions from RC4() to rc4_internal()
and RC4_set_key() to rc4_set_key_internal(). Always include rc4.c
and change it to use defines that are similar to those used in BN.

ok beck@ joshua@ tb@

5 files changed, 52 insertions, 21 deletions

diff --git a/src/lib/libcrypto/arch/amd64/Makefile.inc b/src/lib/libcrypto/arch/amd64/Makefile.inc
index b04923b596..617b9feb9a 100644
--- a/src/lib/libcrypto/arch/amd64/Makefile.inc
+++ b/src/lib/libcrypto/arch/amd64/Makefile.inc
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile.inc,v 1.16 2024/03/28 01:41:16 jsing Exp $
+# $OpenBSD: Makefile.inc,v 1.17 2024/03/28 01:49:29 jsing Exp $
 
 # amd64-specific libcrypto build rules
 
@@ -46,7 +46,10 @@ SSLASM+= md5 md5-x86_64
 CFLAGS+= -DGHASH_ASM
 SSLASM+= modes ghash-x86_64
 # rc4
+CFLAGS+= -DHAVE_RC4_INTERNAL
+CFLAGS+= -DHAVE_RC4_SET_KEY_INTERNAL
 SSLASM+= rc4 rc4-x86_64
+SRCS+=   rc4.c
 # ripemd
 # sha
 CFLAGS+= -DSHA1_ASM
diff --git a/src/lib/libcrypto/arch/i386/Makefile.inc b/src/lib/libcrypto/arch/i386/Makefile.inc
index c1a52e786a..911f02073b 100644
--- a/src/lib/libcrypto/arch/i386/Makefile.inc
+++ b/src/lib/libcrypto/arch/i386/Makefile.inc
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile.inc,v 1.13 2024/03/28 01:41:16 jsing Exp $
+# $OpenBSD: Makefile.inc,v 1.14 2024/03/28 01:49:29 jsing Exp $
 
 # i386-specific libcrypto build rules
 
@@ -26,7 +26,10 @@ SSLASM+= md5 md5-586
 CFLAGS+= -DGHASH_ASM
 SSLASM+= modes ghash-x86
 # rc4
+CFLAGS+= -DHAVE_RC4_INTERNAL
+CFLAGS+= -DHAVE_RC4_SET_KEY_INTERNAL
 SSLASM+= rc4 rc4-586
+SRCS+=   rc4.c
 # sha
 CFLAGS+= -DSHA1_ASM
 SSLASM+= sha sha1-586
diff --git a/src/lib/libcrypto/rc4/asm/rc4-586.pl b/src/lib/libcrypto/rc4/asm/rc4-586.pl
index bb30f9b88f..8fffe91e74 100644
--- a/src/lib/libcrypto/rc4/asm/rc4-586.pl
+++ b/src/lib/libcrypto/rc4/asm/rc4-586.pl
@@ -152,8 +152,9 @@ if ($alt=0) {
 
 &external_label("OPENSSL_ia32cap_P");
 
-# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
-&function_begin("RC4");
+# void rc4_internal(RC4_KEY *key, size_t len, const unsigned char *inp,
+#     unsigned char *out);
+&function_begin("rc4_internal");
         &mov    ($dat,&wparam(0));      # load key schedule pointer
         &mov    ($ty, &wparam(1));      # load len
         &mov    ($inp,&wparam(2));      # load inp
@@ -291,7 +292,7 @@ if ($alt=0) {
         &mov    (&DWP(-4,$dat),$yy);            # save key->y
         &mov    (&BP(-8,$dat),&LB($xx));        # save key->x
 &set_label("abort");
-&function_end("RC4");
+&function_end("rc4_internal");
 
 ########################################################################
 
@@ -301,8 +302,8 @@ $idi="ebp";
 $ido="ecx";
 $idx="edx";
 
-# void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data);
-&function_begin("RC4_set_key");
+# void rc4_set_key_internal(RC4_KEY *key,int len,const unsigned char *data);
+&function_begin("rc4_set_key_internal");
         &mov    ($out,&wparam(0));              # load key
         &mov    ($idi,&wparam(1));              # load len
         &mov    ($inp,&wparam(2));              # load data
@@ -382,6 +383,6 @@ $idx="edx";
         &xor    ("eax","eax");
         &mov    (&DWP(-8,$out),"eax");          # key->x=0;
         &mov    (&DWP(-4,$out),"eax");          # key->y=0;
-&function_end("RC4_set_key");
+&function_end("rc4_set_key_internal");
 
 &asm_finish();
diff --git a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
index a9cf9d1bdf..4dfce6a9ad 100755
--- a/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
+++ b/src/lib/libcrypto/rc4/asm/rc4-x86_64.pl
@@ -124,10 +124,10 @@ $code=<<___;
 .extern OPENSSL_ia32cap_P
 .hidden OPENSSL_ia32cap_P
 
-.globl  RC4
-.type   RC4,\@function,4
+.globl  rc4_internal
+.type   rc4_internal,\@function,4
 .align  16
-RC4:
+rc4_internal:
         _CET_ENDBR
         or      $len,$len
         jne     .Lentry
@@ -423,7 +423,7 @@ $code.=<<___;
         add     \$24,%rsp
 .Lepilogue:
         ret
-.size   RC4,.-RC4
+.size   rc4_internal,.-rc4_internal
 ___
 }
 
@@ -431,10 +431,10 @@ $idx="%r8";
 $ido="%r9";
 
 $code.=<<___;
-.globl  RC4_set_key
-.type   RC4_set_key,\@function,3
+.globl  rc4_set_key_internal
+.type   rc4_set_key_internal,\@function,3
 .align  16
-RC4_set_key:
+rc4_set_key_internal:
         _CET_ENDBR
         lea     8($dat),$dat
         lea     ($inp,$len),$inp
@@ -502,7 +502,7 @@ RC4_set_key:
         mov     %eax,-8($dat)
         mov     %eax,-4($dat)
         ret
-.size   RC4_set_key,.-RC4_set_key
+.size   rc4_set_key_internal,.-rc4_set_key_internal
 ___
 
 sub reg_part {
diff --git a/src/lib/libcrypto/rc4/rc4.c b/src/lib/libcrypto/rc4/rc4.c
index bbf7c3ae4e..8ff8191a51 100644
--- a/src/lib/libcrypto/rc4/rc4.c
+++ b/src/lib/libcrypto/rc4/rc4.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rc4.c,v 1.8 2024/03/27 12:54:42 jsing Exp $ */
+/* $OpenBSD: rc4.c,v 1.9 2024/03/28 01:49:29 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -68,8 +68,13 @@
  * Date: Wed, 14 Sep 1994 06:35:31 GMT
  */
 
-void
-RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
+#ifdef HAVE_RC4_INTERNAL
+void rc4_internal(RC4_KEY *key, size_t len, const unsigned char *indata,
+    unsigned char *outdata);
+
+#else
+static void
+rc4_internal(RC4_KEY *key, size_t len, const unsigned char *indata,
     unsigned char *outdata)
 {
         RC4_INT *d;
@@ -251,9 +256,14 @@ RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
         key->x = x;
         key->y = y;
 }
+#endif
 
-void
-RC4_set_key(RC4_KEY *key, int len, const unsigned char *data)
+#ifdef HAVE_RC4_SET_KEY_INTERNAL
+void rc4_set_key_internal(RC4_KEY *key, int len, const unsigned char *data);
+
+#else
+static void
+rc4_set_key_internal(RC4_KEY *key, int len, const unsigned char *data)
 {
         RC4_INT tmp;
         int id1, id2;
@@ -281,3 +291,17 @@ RC4_set_key(RC4_KEY *key, int len, const unsigned char *data)
                 SK_LOOP(d, i + 3);
         }
 }
+#endif
+
+void
+RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
+    unsigned char *outdata)
+{
+        rc4_internal(key, len, indata, outdata);
+}
+
+void
+RC4_set_key(RC4_KEY *key, int len, const unsigned char *data)
+{
+        rc4_set_key_internal(key, len, data);
+}