Create a new default RSA engine instead of patching the existing one

if none is available.  Fixes SSL/TLS and a possible fatalx() on
machines without a default RSA engine.

Thanks to Bjorn Ketelaars for reporting and testing.

ok gilles@ (for the relayd part)

5 files changed, 84 insertions, 29 deletions

diff --git a/usr.sbin/relayd/ca.c b/usr.sbin/relayd/ca.c
index b478d843e4d..609764aec1d 100644
--- a/usr.sbin/relayd/ca.c
+++ b/usr.sbin/relayd/ca.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ca.c,v 1.7 2014/04/22 08:04:23 reyk Exp $	*/
+/*	$OpenBSD: ca.c,v 1.8 2014/05/04 16:38:19 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -420,17 +420,37 @@ rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
 	return (rsa_default->rsa_keygen(rsa, bits, e, cb));
 }
 
-int
+void
 ca_engine_init(struct relayd *x_env)
 {
-	ENGINE	*e;
+	ENGINE		*e;
+	const char	*errstr, *name;
 
 	if (env == NULL)
 		env = x_env;
 
-	if ((e = ENGINE_get_default_RSA()) == NULL ||
-	    (rsa_default = ENGINE_get_RSA(e)) == NULL)
-		return (-1);
+	if ((e = ENGINE_get_default_RSA()) == NULL) {
+		if ((e = ENGINE_new()) == NULL) {
+			errstr = "ENGINE_new";
+			goto fail;
+		}
+		if (!ENGINE_set_name(e, rsae_method.name)) {
+			errstr = "ENGINE_set_name";
+			goto fail;
+		}
+		if ((rsa_default = RSA_get_default_method()) == NULL) {
+			errstr = "RSA_get_default_method";
+			goto fail;
+		}
+	} else if ((rsa_default = ENGINE_get_RSA(e)) == NULL) {
+		errstr = "ENGINE_get_RSA";
+		goto fail;
+	}
+
+	if ((name = ENGINE_get_name(e)) == NULL)
+		name = "unknown RSA engine";
+
+	log_debug("%s: using %s", __func__, name);
 
 	if (rsa_default->flags & RSA_FLAG_SIGN_VER)
 		fatalx("unsupported RSA engine");
@@ -447,9 +467,18 @@ ca_engine_init(struct relayd *x_env)
 	    RSA_METHOD_FLAG_NO_CHECK;
 	rsae_method.app_data = rsa_default->app_data;
 
-	if (!ENGINE_set_RSA(e, &rsae_method) ||
-	    !ENGINE_set_default_RSA(e))
-		return (-1);
+	if (!ENGINE_set_RSA(e, &rsae_method)) {
+		errstr = "ENGINE_set_RSA";
+		goto fail;
+	}
+	if (!ENGINE_set_default_RSA(e)) {
+		errstr = "ENGINE_set_default_RSA";
+		goto fail;
+	}
 
-	return (0);
+	return;
+
+ fail:
+	ssl_error(__func__, errstr);
+	fatalx(errstr);
 }
diff --git a/usr.sbin/relayd/config.c b/usr.sbin/relayd/config.c
index 609317a5680..51adfe950b7 100644
--- a/usr.sbin/relayd/config.c
+++ b/usr.sbin/relayd/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.13 2014/04/22 08:04:23 reyk Exp $	*/
+/*	$OpenBSD: config.c,v 1.14 2014/05/04 16:38:19 reyk Exp $	*/
 
 /*
  * Copyright (c) 2011 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -284,9 +284,8 @@ config_getcfg(struct relayd *env, struct imsg *imsg)
 
 	if (env->sc_flags & (F_SSL|F_SSLCLIENT)) {
 		ssl_init(env);
-		if ((what & CONFIG_CA_ENGINE) &&
-		    (ca_engine_init(env)) == -1)
-			fatal("CA engine failed");
+		if (what & CONFIG_CA_ENGINE)
+			ca_engine_init(env);
 	}
 
 	if (privsep_process != PROC_PARENT)
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 23ddafdbee5..74e13d54fd5 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.177 2014/04/22 08:04:23 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.178 2014/05/04 16:38:19 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -1118,7 +1118,7 @@ int	 ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t);
 
 /* ca.c */
 pid_t	 ca(struct privsep *, struct privsep_proc *);
-int	 ca_engine_init(struct relayd *);
+void	 ca_engine_init(struct relayd *);
 
 /* relayd.c */
 struct host	*host_find(struct relayd *, objid_t);
diff --git a/usr.sbin/smtpd/ca.c b/usr.sbin/smtpd/ca.c
index 719c7579f02..b50169f8f7e 100644
--- a/usr.sbin/smtpd/ca.c
+++ b/usr.sbin/smtpd/ca.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ca.c,v 1.6 2014/05/01 15:50:20 reyk Exp $	*/
+/*	$OpenBSD: ca.c,v 1.7 2014/05/04 16:38:19 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -521,16 +521,34 @@ rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
 	return (rsa_default->rsa_keygen(rsa, bits, e, cb));
 }
 
-int
+void
 ca_engine_init(void)
 {
-	ENGINE	*e;
+	ENGINE		*e;
+	const char	*errstr, *name;
 
-	log_debug("debug: %s: %s", proc_name(smtpd_process), __func__);
+	if ((e = ENGINE_get_default_RSA()) == NULL) {
+		if ((e = ENGINE_new()) == NULL) {
+			errstr = "ENGINE_new";
+			goto fail;
+		}
+		if (!ENGINE_set_name(e, rsae_method.name)) {
+			errstr = "ENGINE_set_name";
+			goto fail;
+		}
+		if ((rsa_default = RSA_get_default_method()) == NULL) {
+			errstr = "RSA_get_default_method";
+			goto fail;
+		}
+	} else if ((rsa_default = ENGINE_get_RSA(e)) == NULL) {
+		errstr = "ENGINE_get_RSA";
+		goto fail;
+	}
+
+	if ((name = ENGINE_get_name(e)) == NULL)
+		name = "unknown RSA engine";
 
-	if ((e = ENGINE_get_default_RSA()) == NULL ||
-	    (rsa_default = ENGINE_get_RSA(e)) == NULL)
-		return (-1);
+	log_debug("debug: %s: using %s", __func__, name);
 
 	if (rsa_default->flags & RSA_FLAG_SIGN_VER)
 		fatalx("unsupported RSA engine");
@@ -547,9 +565,18 @@ ca_engine_init(void)
 	    RSA_METHOD_FLAG_NO_CHECK;
 	rsae_method.app_data = rsa_default->app_data;
 
-	if (!ENGINE_set_RSA(e, &rsae_method) ||
-	    !ENGINE_set_default_RSA(e))
-		return (-1);
+	if (!ENGINE_set_RSA(e, &rsae_method)) {
+		errstr = "ENGINE_set_RSA";
+		goto fail;
+	}
+	if (!ENGINE_set_default_RSA(e)) {
+		errstr = "ENGINE_set_default_RSA";
+		goto fail;
+	}
+
+	return;
 
-	return (0);
+ fail:
+	ssl_error(errstr);
+	fatalx(errstr);
 }
diff --git a/usr.sbin/smtpd/smtpd.h b/usr.sbin/smtpd/smtpd.h
index c1db503ef63..7023a9fe6b1 100644
--- a/usr.sbin/smtpd/smtpd.h
+++ b/usr.sbin/smtpd/smtpd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: smtpd.h,v 1.460 2014/05/01 15:50:20 reyk Exp $	*/
+/*	$OpenBSD: smtpd.h,v 1.461 2014/05/04 16:38:19 reyk Exp $	*/
 
 /*
  * Copyright (c) 2008 Gilles Chehade <gilles@poolp.org>
@@ -1078,7 +1078,7 @@ pid_t	 ca(void);
 int	 ca_X509_verify(void *, void *, const char *, const char *, const char **);
 void	 ca_imsg(struct mproc *, struct imsg *);
 void	 ca_init(void);
-int	 ca_engine_init(void);
+void	 ca_engine_init(void);
 
 /* compress_backend.c */
 struct compress_backend *compress_backend_lookup(const char *);