Enable GOST_SIG_FORMAT_RS_LE when verifying certificate signatures.

GOST cipher suites requires that CertVerify signatures be generated in a
special way (see ssl3_send_client_kex_gost(), ssl3_get_cert_verify()).
However, the GOST_SIG_FORMAT_RS_LE flag was not passed in case of TLS 1.2
connections (because they use different code path). Set this flag on
GOST PKEYs.

Diff from Dmitry Baryshkov <dbaryshkov@gmail.com>

Sponsored by ROSA Linux

ok inoguchi@ tb@

2 files changed, 15 insertions, 2 deletions

diff --git a/src/lib/libssl/ssl_clnt.c b/src/lib/libssl/ssl_clnt.c
index 4d003466c4..0a1b6ea241 100644
--- a/src/lib/libssl/ssl_clnt.c
+++ b/src/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.68 2020/05/31 16:36:35 jsing Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.69 2020/06/05 17:53:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -2341,6 +2341,12 @@ ssl3_send_client_verify_sigalgs(SSL *s, CBB *cert_verify)
                 SSLerror(s, ERR_R_EVP_LIB);
                 goto err;
         }
+        if (sigalg->key_type == EVP_PKEY_GOSTR01 &&
+            EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_SIGN,
+            EVP_PKEY_CTRL_GOST_SIG_FORMAT, GOST_SIG_FORMAT_RS_LE, NULL) <= 0) {
+                SSLerror(s, ERR_R_EVP_LIB);
+                goto err;
+        }
         if ((sigalg->flags & SIGALG_FLAG_RSA_PSS) &&
             (!EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) ||
             !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index 15dfdc35b1..706ad1453b 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.78 2020/06/01 08:04:02 tb Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.79 2020/06/05 17:53:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -2199,6 +2199,13 @@ ssl3_get_cert_verify(SSL *s)
                         al = SSL_AD_INTERNAL_ERROR;
                         goto f_err;
                 }
+                if (sigalg->key_type == EVP_PKEY_GOSTR01 &&
+                    EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_VERIFY,
+                    EVP_PKEY_CTRL_GOST_SIG_FORMAT, GOST_SIG_FORMAT_RS_LE,
+                    NULL) <= 0) {
+                        al = SSL_AD_INTERNAL_ERROR;
+                        goto f_err;
+                }
                 if (!EVP_DigestVerifyUpdate(&mctx, hdata, hdatalen)) {
                         SSLerror(s, ERR_R_EVP_LIB);
                         al = SSL_AD_INTERNAL_ERROR;