Move the relay keys/certs into a separate global list and look them up by id.

Moving the certs out of the relay struct will help to add multiple SNI certs.

Tested by many users (thanks!)

Feedback & OK rob@

6 files changed, 274 insertions, 143 deletions

diff --git a/usr.sbin/relayd/ca.c b/usr.sbin/relayd/ca.c
index f72c900e80f..91bf346fe5a 100644
--- a/usr.sbin/relayd/ca.c
+++ b/usr.sbin/relayd/ca.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ca.c,v 1.34 2018/09/19 11:28:02 reyk Exp $	*/
+/*	$OpenBSD: ca.c,v 1.35 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -108,56 +108,60 @@ hash_x509(X509 *cert, char *hash, size_t hashlen)
 void
 ca_launch(void)
 {
-	char		 hash[TLS_CERT_HASH_SIZE];
-	char		*buf;
-	BIO		*in = NULL;
-	EVP_PKEY	*pkey = NULL;
-	struct relay	*rlay;
-	X509		*cert = NULL;
-	off_t		 len;
-
-	TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) {
-		if ((rlay->rl_conf.flags & (F_TLS|F_TLSCLIENT)) == 0)
+	char			 hash[TLS_CERT_HASH_SIZE];
+	char			*buf;
+	BIO			*in = NULL;
+	EVP_PKEY		*pkey = NULL;
+	struct relay		*rlay;
+	struct relay_cert	*cert;
+	X509			*x509 = NULL;
+	off_t			 len;
+
+	TAILQ_FOREACH(cert, env->sc_certs, cert_entry) {
+		if (cert->cert_fd == -1 || cert->cert_key_fd == -1)
 			continue;
 
-		if (rlay->rl_tls_cert_fd != -1) {
-			if ((buf = relay_load_fd(rlay->rl_tls_cert_fd,
-			    &len)) == NULL)
-				fatal("ca_launch: cert relay_load_fd");
+		if ((buf = relay_load_fd(cert->cert_fd, &len)) == NULL)
+			fatal("ca_launch: cert relay_load_fd");
 
-			if ((in = BIO_new_mem_buf(buf, len)) == NULL)
-				fatalx("ca_launch: cert BIO_new_mem_buf");
+		if ((in = BIO_new_mem_buf(buf, len)) == NULL)
+			fatalx("ca_launch: cert BIO_new_mem_buf");
 
-			if ((cert = PEM_read_bio_X509(in, NULL,
-			    NULL, NULL)) == NULL)
-				fatalx("ca_launch: cert PEM_read_bio_X509");
+		if ((x509 = PEM_read_bio_X509(in, NULL,
+		    NULL, NULL)) == NULL)
+			fatalx("ca_launch: cert PEM_read_bio_X509");
 
-			hash_x509(cert, hash, sizeof(hash));
+		hash_x509(x509, hash, sizeof(hash));
 
-			BIO_free(in);
-			X509_free(cert);
-			purge_key(&buf, len);
-		}
-		if (rlay->rl_conf.tls_key_len) {
-			if ((in = BIO_new_mem_buf(rlay->rl_tls_key,
-			    rlay->rl_conf.tls_key_len)) == NULL)
-				fatalx("%s: key", __func__);
+		BIO_free(in);
+		X509_free(x509);
+		purge_key(&buf, len);
 
-			if ((pkey = PEM_read_bio_PrivateKey(in,
-			    NULL, NULL, NULL)) == NULL)
-				fatalx("%s: PEM", __func__);
-			BIO_free(in);
+		if ((buf = relay_load_fd(cert->cert_key_fd, &len)) == NULL)
+			fatal("ca_launch: key relay_load_fd");
 
-			rlay->rl_tls_pkey = pkey;
+		if ((in = BIO_new_mem_buf(buf, len)) == NULL)
+			fatalx("%s: key", __func__);
 
-			if (pkey_add(env, pkey, hash) == NULL)
-				fatalx("tls pkey");
+		if ((pkey = PEM_read_bio_PrivateKey(in,
+		    NULL, NULL, NULL)) == NULL)
+			fatalx("%s: PEM", __func__);
 
-			purge_key(&rlay->rl_tls_key,
-			    rlay->rl_conf.tls_key_len);
-		}
+		cert->cert_pkey = pkey;
+
+		if (pkey_add(env, pkey, hash) == NULL)
+			fatalx("tls pkey");
+
+		BIO_free(in);
+		purge_key(&buf, len);
+	}
 
-		if (rlay->rl_tls_cacert_fd != -1) {
+	TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) {
+		if ((rlay->rl_conf.flags & (F_TLS|F_TLSCLIENT)) == 0)
+			continue;
+
+		if (rlay->rl_tls_cacert_fd != -1 &&
+		    rlay->rl_conf.tls_cakey_len) {
 			if ((buf = relay_load_fd(rlay->rl_tls_cacert_fd,
 			    &len)) == NULL)
 				fatal("ca_launch: cacert relay_load_fd");
@@ -165,17 +169,16 @@ ca_launch(void)
 			if ((in = BIO_new_mem_buf(buf, len)) == NULL)
 				fatalx("ca_launch: cacert BIO_new_mem_buf");
 
-			if ((cert = PEM_read_bio_X509(in, NULL,
+			if ((x509 = PEM_read_bio_X509(in, NULL,
 			    NULL, NULL)) == NULL)
 				fatalx("ca_launch: cacert PEM_read_bio_X509");
 
-			hash_x509(cert, hash, sizeof(hash));
+			hash_x509(x509, hash, sizeof(hash));
 
 			BIO_free(in);
-			X509_free(cert);
+			X509_free(x509);
 			purge_key(&buf, len);
-		}
-		if (rlay->rl_conf.tls_cakey_len) {
+
 			if ((in = BIO_new_mem_buf(rlay->rl_tls_cakey,
 			    rlay->rl_conf.tls_cakey_len)) == NULL)
 				fatalx("%s: key", __func__);
diff --git a/usr.sbin/relayd/config.c b/usr.sbin/relayd/config.c
index a8e88baac37..9b71565d956 100644
--- a/usr.sbin/relayd/config.c
+++ b/usr.sbin/relayd/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.36 2017/11/29 15:24:50 benno Exp $	*/
+/*	$OpenBSD: config.c,v 1.37 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2011 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -81,6 +81,12 @@ config_init(struct relayd *env)
 		    calloc(1, sizeof(*env->sc_relays))) == NULL)
 			return (-1);
 		TAILQ_INIT(env->sc_relays);
+
+		if ((env->sc_certs =
+		    calloc(1, sizeof(*env->sc_certs))) == NULL)
+			return (-1);
+		TAILQ_INIT(env->sc_certs);
+
 		if ((env->sc_pkeys =
 		    calloc(1, sizeof(*env->sc_pkeys))) == NULL)
 			return (-1);
@@ -773,12 +779,13 @@ config_getrule(struct relayd *env, struct imsg *imsg)
 }
 
 static int
-config_setrelayfd(struct privsep *ps, int id, int n, int rlay_id, int type,
-    int ofd)
+config_setrelayfd(struct privsep *ps, int id, int n,
+    objid_t obj_id, objid_t rlay_id, enum fd_type type, int ofd)
 {
 	struct ctl_relayfd	rfd;
 	int			fd;
 
+	rfd.id = obj_id;
 	rfd.relayid = rlay_id;
 	rfd.type = type;
 
@@ -798,6 +805,7 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 	struct ctl_relaytable	 crt;
 	struct relay_table	*rlt;
 	struct relay_config	 rl;
+	struct relay_cert	*cert;
 	int			 id;
 	int			 fd, n, m;
 	struct iovec		 iov[6];
@@ -824,12 +832,6 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 		iov[c++].iov_len = sizeof(rl);
 
 		if ((what & CONFIG_CA_ENGINE) == 0 &&
-		    rl.tls_key_len) {
-			iov[c].iov_base = rlay->rl_tls_key;
-			iov[c++].iov_len = rl.tls_key_len;
-		} else
-			rl.tls_key_len = 0;
-		if ((what & CONFIG_CA_ENGINE) == 0 &&
 		    rl.tls_cakey_len) {
 			iov[c].iov_base = rlay->rl_tls_cakey;
 			iov[c++].iov_len = rl.tls_cakey_len;
@@ -868,22 +870,42 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 			}
 		}
 
-
-		if (what & CONFIG_CERTS) {
+		/* cert keypairs */
+		TAILQ_FOREACH(cert, env->sc_certs, cert_entry) {
+			if (cert->cert_relayid != rlay->rl_conf.id)
+				continue;
 			n = -1;
 			proc_range(ps, id, &n, &m);
-			for (n = 0; n < m; n++) {
-				if (rlay->rl_tls_cert_fd != -1 &&
+			for (n = 0; (what & CONFIG_CERTS) && n < m; n++) {
+				if (cert->cert_fd != -1 &&
 				    config_setrelayfd(ps, id, n,
-				    rlay->rl_conf.id, RELAY_FD_CERT,
-				    rlay->rl_tls_cert_fd) == -1) {
+				    cert->cert_id, cert->cert_relayid,
+				    RELAY_FD_CERT, cert->cert_fd) == -1) {
 					log_warn("%s: fd passing failed for "
 					    "`%s'", __func__,
 					    rlay->rl_conf.name);
 					return (-1);
 				}
-				if (rlay->rl_tls_ca_fd != -1 &&
+				if (id == PROC_CA &&
+				    cert->cert_key_fd != -1 &&
 				    config_setrelayfd(ps, id, n,
+				    cert->cert_id, cert->cert_relayid,
+				    RELAY_FD_KEY, cert->cert_key_fd) == -1) {
+					log_warn("%s: fd passing failed for "
+					    "`%s'", __func__,
+					    rlay->rl_conf.name);
+					return (-1);
+				}
+			}
+		}
+
+		/* CA certs */
+		if (what & CONFIG_CERTS) {
+			n = -1;
+			proc_range(ps, id, &n, &m);
+			for (n = 0; n < m; n++) {
+				if (rlay->rl_tls_ca_fd != -1 &&
+				    config_setrelayfd(ps, id, n, 0,
 				    rlay->rl_conf.id, RELAY_FD_CACERT,
 				    rlay->rl_tls_ca_fd) == -1) {
 					log_warn("%s: fd passing failed for "
@@ -892,7 +914,7 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 					return (-1);
 				}
 				if (rlay->rl_tls_cacert_fd != -1 &&
-				    config_setrelayfd(ps, id, n,
+				    config_setrelayfd(ps, id, n, 0,
 				    rlay->rl_conf.id, RELAY_FD_CAFILE,
 				    rlay->rl_tls_cacert_fd) == -1) {
 					log_warn("%s: fd passing failed for "
@@ -933,10 +955,6 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 		close(rlay->rl_s);
 		rlay->rl_s = -1;
 	}
-	if (rlay->rl_tls_cert_fd != -1) {
-		close(rlay->rl_tls_cert_fd);
-		rlay->rl_tls_cert_fd = -1;
-	}
 	if (rlay->rl_tls_cacert_fd != -1) {
 		close(rlay->rl_tls_cacert_fd);
 		rlay->rl_tls_cacert_fd = -1;
@@ -945,6 +963,19 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 		close(rlay->rl_tls_ca_fd);
 		rlay->rl_tls_ca_fd = -1;
 	}
+	TAILQ_FOREACH(cert, env->sc_certs, cert_entry) {
+		if (cert->cert_relayid != rlay->rl_conf.id)
+			continue;
+
+		if (cert->cert_fd != -1) {
+			close(cert->cert_fd);
+			cert->cert_fd = -1;
+		}
+		if (cert->cert_key_fd != -1) {
+			close(cert->cert_key_fd);
+			cert->cert_key_fd = -1;
+		}
+	}
 
 	return (0);
 }
@@ -965,7 +996,6 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 	s = sizeof(rlay->rl_conf);
 
 	rlay->rl_s = imsg->fd;
-	rlay->rl_tls_cert_fd = -1;
 	rlay->rl_tls_ca_fd = -1;
 	rlay->rl_tls_cacert_fd = -1;
 
@@ -980,17 +1010,11 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 	}
 
 	if ((off_t)(IMSG_DATA_SIZE(imsg) - s) <
-	    (rlay->rl_conf.tls_key_len + rlay->rl_conf.tls_cakey_len)) {
+	    (rlay->rl_conf.tls_cakey_len)) {
 		log_debug("%s: invalid message length", __func__);
 		goto fail;
 	}
 
-	if (rlay->rl_conf.tls_key_len) {
-		if ((rlay->rl_tls_key = get_data(p + s,
-		    rlay->rl_conf.tls_key_len)) == NULL)
-			goto fail;
-		s += rlay->rl_conf.tls_key_len;
-	}
 	if (rlay->rl_conf.tls_cakey_len) {
 		if ((rlay->rl_tls_cakey = get_data(p + s,
 		    rlay->rl_conf.tls_cakey_len)) == NULL)
@@ -1010,7 +1034,6 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 	return (0);
 
  fail:
-	free(rlay->rl_tls_key);
 	free(rlay->rl_tls_cakey);
 	close(rlay->rl_s);
 	free(rlay);
@@ -1062,22 +1085,37 @@ config_getrelaytable(struct relayd *env, struct imsg *imsg)
 int
 config_getrelayfd(struct relayd *env, struct imsg *imsg)
 {
-	struct relay_table	*rlt = NULL;
 	struct ctl_relayfd	 crfd;
-	struct relay		*rlay;
+	struct relay		*rlay = NULL;
+	struct relay_cert	*cert;
 	u_int8_t		*p = imsg->data;
 
 	IMSG_SIZE_CHECK(imsg, &crfd);
 	memcpy(&crfd, p, sizeof(crfd));
 
-	if ((rlay = relay_find(env, crfd.relayid)) == NULL) {
-		log_debug("%s: unknown relay", __func__);
-		goto fail;
+	switch (crfd.type) {
+	case RELAY_FD_CERT:
+	case RELAY_FD_KEY:
+		if ((cert = cert_find(env, crfd.id)) == NULL) {
+			if ((cert = cert_add(env, crfd.id)) == NULL)
+				return (-1);
+			cert->cert_relayid = crfd.relayid;
+		}
+		/* FALLTHROUGH */
+	default:
+		if ((rlay = relay_find(env, crfd.relayid)) == NULL) {
+			log_debug("%s: unknown relay", __func__);
+			return (-1);
+		}
+		break;
 	}
 
 	switch (crfd.type) {
 	case RELAY_FD_CERT:
-		rlay->rl_tls_cert_fd = imsg->fd;
+		cert->cert_fd = imsg->fd;
+		break;
+	case RELAY_FD_KEY:
+		cert->cert_key_fd = imsg->fd;
 		break;
 	case RELAY_FD_CACERT:
 		rlay->rl_tls_ca_fd = imsg->fd;
@@ -1092,8 +1130,4 @@ config_getrelayfd(struct relayd *env, struct imsg *imsg)
 	    imsg->fd, crfd.type, rlay->rl_conf.name);
 
 	return (0);
-
- fail:
-	free(rlt);
-	return (-1);
 }
diff --git a/usr.sbin/relayd/parse.y b/usr.sbin/relayd/parse.y
index a2533749d23..bb1765ebe33 100644
--- a/usr.sbin/relayd/parse.y
+++ b/usr.sbin/relayd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.236 2019/05/29 11:52:56 reyk Exp $	*/
+/*	$OpenBSD: parse.y,v 1.237 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -1710,7 +1710,6 @@ relay		: RELAY STRING	{
 			r->rl_proto = NULL;
 			r->rl_conf.proto = EMPTY_ID;
 			r->rl_conf.dstretry = 0;
-			r->rl_tls_cert_fd = -1;
 			r->rl_tls_ca_fd = -1;
 			r->rl_tls_cacert_fd = -1;
 			TAILQ_INIT(&r->rl_tables);
@@ -3269,11 +3268,8 @@ relay_inherit(struct relay *ra, struct relay *rb)
 	rb->rl_conf.flags =
 	    (ra->rl_conf.flags & ~F_TLS) | (rc.flags & F_TLS);
 	if (!(rb->rl_conf.flags & F_TLS)) {
-		rb->rl_tls_cert_fd = -1;
 		rb->rl_tls_cacert_fd = -1;
 		rb->rl_tls_ca_fd = -1;
-		rb->rl_tls_key = NULL;
-		rb->rl_conf.tls_key_len = 0;
 	}
 	TAILQ_INIT(&rb->rl_tables);
 
diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index 44d4ef9e737..2148fdf3817 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.246 2019/05/29 11:48:28 reyk Exp $	*/
+/*	$OpenBSD: relay.c,v 1.247 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -2127,8 +2127,9 @@ relay_tls_ctx_create(struct relay *rlay)
 {
 	struct tls_config	*tls_cfg, *tls_client_cfg;
 	struct tls		*tls = NULL;
+	struct relay_cert	*cert;
 	const char		*fake_key;
-	int			 fake_keylen;
+	int			 fake_keylen, keyfound = 0;
 	char			*buf = NULL, *cabuf = NULL;
 	off_t			 len = 0, calen = 0;
 
@@ -2160,6 +2161,7 @@ relay_tls_ctx_create(struct relay *rlay)
 				log_warn("failed to read root certificates");
 				goto err;
 			}
+			rlay->rl_tls_ca_fd = -1;
 
 			if (tls_config_set_ca_mem(tls_client_cfg, buf, len) !=
 			    0) {
@@ -2188,24 +2190,47 @@ relay_tls_ctx_create(struct relay *rlay)
 		 */
 		tls_config_skip_private_key_check(tls_cfg);
 
-		if ((buf = relay_load_fd(rlay->rl_tls_cert_fd, &len)) == NULL) {
-			log_warn("failed to load tls certificate");
-			goto err;
-		}
+		TAILQ_FOREACH(cert, env->sc_certs, cert_entry) {
+			if (cert->cert_relayid != rlay->rl_conf.id ||
+			    cert->cert_fd == -1)
+				continue;
+			keyfound++;
 
-		if ((fake_keylen = ssl_ctx_fake_private_key(buf, len,
-		    &fake_key)) == -1) {
-			/* error already printed */
-			goto err;
-		}
+			if ((buf = relay_load_fd(cert->cert_fd,
+			    &len)) == NULL) {
+				log_warn("failed to load tls certificate");
+				goto err;
+			}
+			cert->cert_fd = -1;
 
-		if (tls_config_set_keypair_ocsp_mem(tls_cfg, buf, len,
-		    fake_key, fake_keylen, NULL, 0) != 0) {
-			log_warnx("failed to set tls certificate: %s",
-			    tls_config_error(tls_cfg));
-			goto err;
-		}
+			if ((fake_keylen = ssl_ctx_fake_private_key(buf, len,
+			    &fake_key)) == -1) {
+				/* error already printed */
+				goto err;
+			}
 
+			if (keyfound == 1 &&
+			    tls_config_set_keypair_ocsp_mem(tls_cfg, buf, len,
+			    fake_key, fake_keylen, NULL, 0) != 0) {
+				log_warnx("failed to set tls certificate: %s",
+				    tls_config_error(tls_cfg));
+				goto err;
+			}
+
+			/* loading certificate public key */
+			if (keyfound == 1 &&
+			    !ssl_load_pkey(buf, len, NULL, &rlay->rl_tls_pkey))
+				goto err;
+
+			if (tls_config_add_keypair_ocsp_mem(tls_cfg, buf, len,
+			    fake_key, fake_keylen, NULL, 0) != 0) {
+				log_warnx("failed to add tls certificate: %s",
+				    tls_config_error(tls_cfg));
+				goto err;
+			}
+
+			purge_key(&buf, len);
+		}
 
 		if (rlay->rl_tls_cacert_fd != -1) {
 			if ((cabuf = relay_load_fd(rlay->rl_tls_cacert_fd,
@@ -2217,11 +2242,8 @@ relay_tls_ctx_create(struct relay *rlay)
 			if (!ssl_load_pkey(cabuf, calen,
 			    &rlay->rl_tls_cacertx509, &rlay->rl_tls_capkey))
 				goto err;
-			/* loading certificate public key */
-			log_debug("%s: loading certificate", __func__);
-			if (!ssl_load_pkey(buf, len, NULL, &rlay->rl_tls_pkey))
-				goto err;
 		}
+		rlay->rl_tls_cacert_fd = -1;
 
 		tls = tls_server();
 		if (tls == NULL) {
@@ -2238,14 +2260,8 @@ relay_tls_ctx_create(struct relay *rlay)
 		rlay->rl_tls_ctx = tls;
 
 		purge_key(&cabuf, calen);
-		purge_key(&buf, len);
 	}
 
-	/* The fd for the keys/certs are not needed anymore */
-	close(rlay->rl_tls_cert_fd);
-	close(rlay->rl_tls_cacert_fd);
-	close(rlay->rl_tls_ca_fd);
-
 	if (rlay->rl_tls_client_cfg == NULL)
 		tls_config_free(tls_client_cfg);
 	if (rlay->rl_tls_cfg == NULL)
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index c884588c773..74ecc21a360 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.c,v 1.177 2019/05/29 11:48:29 reyk Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.178 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007 - 2016 Reyk Floeter <reyk@openbsd.org>
@@ -555,6 +555,7 @@ purge_relay(struct relayd *env, struct relay *rlay)
 {
 	struct rsession		*con;
 	struct relay_table	*rlt;
+	struct relay_cert	*cert, *tmpcert;
 
 	/* shutdown and remove relay */
 	if (event_initialized(&rlay->rl_ev))
@@ -573,7 +574,6 @@ purge_relay(struct relayd *env, struct relay *rlay)
 	if (rlay->rl_dstbev != NULL)
 		bufferevent_free(rlay->rl_dstbev);
 
-	purge_key(&rlay->rl_tls_key, rlay->rl_conf.tls_key_len);
 	purge_key(&rlay->rl_tls_cakey, rlay->rl_conf.tls_cakey_len);
 
 	if (rlay->rl_tls_pkey != NULL) {
@@ -598,6 +598,19 @@ purge_relay(struct relayd *env, struct relay *rlay)
 		free(rlt);
 	}
 
+	TAILQ_FOREACH_SAFE(cert, env->sc_certs, cert_entry, tmpcert) {
+		if (rlay->rl_conf.id != cert->cert_relayid)
+			continue;
+		if (cert->cert_fd != -1)
+			close(cert->cert_fd);
+		if (cert->cert_key_fd != -1)
+			close(cert->cert_key_fd);
+		if (cert->cert_pkey != NULL)
+			EVP_PKEY_free(cert->cert_pkey);
+		TAILQ_REMOVE(env->sc_certs, cert, cert_entry);
+		free(cert);
+	}
+
 	free(rlay);
 }
 
@@ -1237,6 +1250,43 @@ pkey_add(struct relayd *env, EVP_PKEY *pkey, char *hash)
 	return (ca_pkey);
 }
 
+struct relay_cert *
+cert_add(struct relayd *env, objid_t id)
+{
+	static objid_t		 last_cert_id = 0;
+	struct relay_cert	*cert;
+
+	if ((cert = calloc(1, sizeof(*cert))) == NULL)
+		return (NULL);
+
+	if (id == 0)
+		id = ++last_cert_id;
+	if (id == INT_MAX) {
+		log_warnx("too many tls keypairs defined");
+		free(cert);
+		return (NULL);
+	}
+
+	cert->cert_id = id;
+	cert->cert_fd = -1;
+	cert->cert_key_fd = -1;
+
+	TAILQ_INSERT_TAIL(env->sc_certs, cert, cert_entry);
+
+	return (cert);
+}
+
+struct relay_cert *
+cert_find(struct relayd *env, objid_t id)
+{
+	struct relay_cert	*cert;
+
+	TAILQ_FOREACH(cert, env->sc_certs, cert_entry)
+		if (cert->cert_id == id)
+			return (cert);
+	return (NULL);
+}
+
 char *
 relay_load_fd(int fd, off_t *len)
 {
@@ -1273,23 +1323,26 @@ relay_load_certfiles(struct relayd *env, struct relay *rlay)
 	char	 certfile[PATH_MAX];
 	char	 hbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")];
 	struct protocol *proto = rlay->rl_proto;
+	struct relay_cert *cert;
 	int	 useport = htons(rlay->rl_conf.port);
+	int	 cert_fd = -1, key_fd = -1;
 
 	if (rlay->rl_conf.flags & F_TLSCLIENT) {
-		if (strlen(proto->tlsca)) {
+		if (strlen(proto->tlsca) && rlay->rl_tls_ca_fd == -1) {
 			if ((rlay->rl_tls_ca_fd =
 			    open(proto->tlsca, O_RDONLY)) == -1)
 				return (-1);
 			log_debug("%s: using ca %s", __func__, proto->tlsca);
 		}
-		if (strlen(proto->tlscacert)) {
+		if (strlen(proto->tlscacert) && rlay->rl_tls_cacert_fd == -1) {
 			if ((rlay->rl_tls_cacert_fd =
 			    open(proto->tlscacert, O_RDONLY)) == -1)
 				return (-1);
 			log_debug("%s: using ca certificate %s", __func__,
 			    proto->tlscacert);
 		}
-		if (strlen(proto->tlscakey) && proto->tlscapass != NULL) {
+		if (strlen(proto->tlscakey) && !rlay->rl_conf.tls_cakey_len &&
+		    proto->tlscapass != NULL) {
 			if ((rlay->rl_tls_cakey =
 			    ssl_load_key(env, proto->tlscakey,
 			    &rlay->rl_conf.tls_cakey_len,
@@ -1304,17 +1357,17 @@ relay_load_certfiles(struct relayd *env, struct relay *rlay)
 		return (0);
 
 	if (print_host(&rlay->rl_conf.ss, hbuf, sizeof(hbuf)) == NULL)
-		return (-1);
+		goto fail;
 
 	if (snprintf(certfile, sizeof(certfile),
 	    "/etc/ssl/%s:%u.crt", hbuf, useport) == -1)
-		return (-1);
-	if ((rlay->rl_tls_cert_fd = open(certfile, O_RDONLY)) == -1) {
+		goto fail;
+	if ((cert_fd = open(certfile, O_RDONLY)) == -1) {
 		if (snprintf(certfile, sizeof(certfile),
 		    "/etc/ssl/%s.crt", hbuf) == -1)
-			return (-1);
-		if ((rlay->rl_tls_cert_fd = open(certfile, O_RDONLY)) == -1)
-			return (-1);
+			goto fail;
+		if ((cert_fd = open(certfile, O_RDONLY)) == -1)
+			goto fail;
 		useport = 0;
 	}
 	log_debug("%s: using certificate %s", __func__, certfile);
@@ -1322,18 +1375,32 @@ relay_load_certfiles(struct relayd *env, struct relay *rlay)
 	if (useport) {
 		if (snprintf(certfile, sizeof(certfile),
 		    "/etc/ssl/private/%s:%u.key", hbuf, useport) == -1)
-			return -1;
+			goto fail;
 	} else {
 		if (snprintf(certfile, sizeof(certfile),
 		    "/etc/ssl/private/%s.key", hbuf) == -1)
-			return -1;
+			goto fail;
 	}
-	if ((rlay->rl_tls_key = ssl_load_key(env, certfile,
-	    &rlay->rl_conf.tls_key_len, NULL)) == NULL)
-		return (-1);
+	if ((key_fd = open(certfile, O_RDONLY)) == -1)
+		goto fail;
 	log_debug("%s: using private key %s", __func__, certfile);
 
+	if ((cert = cert_add(env, 0)) == NULL)
+		goto fail;
+
+	cert->cert_relayid = rlay->rl_conf.id;
+	cert->cert_fd = cert_fd;
+	cert->cert_key_fd = key_fd;
+
 	return (0);
+
+ fail:
+	if (cert_fd != -1)
+		close(cert_fd);
+	if (key_fd != -1)
+		close(key_fd);
+
+	return (-1);
 }
 
 void
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index d3c03da9f38..ec76c38c795 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.256 2019/05/29 11:48:29 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.257 2019/05/31 15:15:37 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2016 Reyk Floeter <reyk@openbsd.org>
@@ -137,13 +137,18 @@ struct ctl_relaytable {
 	u_int32_t	 flags;
 };
 
+enum fd_type {
+	RELAY_FD_CERT	= 1,
+	RELAY_FD_CACERT	= 2,
+	RELAY_FD_CAFILE	= 3,
+	RELAY_FD_KEY	= 4
+};
+
 struct ctl_relayfd {
+	objid_t		 id;
 	objid_t		 relayid;
-	int		 type;
+	enum fd_type	 type;
 };
-#define RELAY_FD_CERT	1
-#define RELAY_FD_CACERT	2
-#define RELAY_FD_CAFILE	3
 
 struct ctl_script {
 	objid_t		 host;
@@ -764,6 +769,16 @@ struct ca_pkey {
 };
 TAILQ_HEAD(ca_pkeylist, ca_pkey);
 
+struct relay_cert {
+	objid_t			 cert_id;
+	objid_t			 cert_relayid;
+	int			 cert_fd;
+	int			 cert_key_fd;
+	EVP_PKEY		*cert_pkey;
+	TAILQ_ENTRY(relay_cert)	 cert_entry;
+};
+TAILQ_HEAD(relaycertlist, relay_cert);
+
 struct relay_config {
 	objid_t			 id;
 	u_int32_t		 flags;
@@ -778,7 +793,6 @@ struct relay_config {
 	struct timeval		 timeout;
 	enum forwardmode	 fwdmode;
 	union hashkey		 hashkey;
-	off_t			 tls_key_len;
 	off_t			 tls_cakey_len;
 };
 
@@ -803,10 +817,8 @@ struct relay {
 	struct tls_config	*rl_tls_client_cfg;
 	struct tls		*rl_tls_ctx;
 
-	int			rl_tls_cert_fd;
-	int			rl_tls_ca_fd;
-	int			rl_tls_cacert_fd;
-	char			*rl_tls_key;
+	int			 rl_tls_ca_fd;
+	int			 rl_tls_cacert_fd;
 	EVP_PKEY		*rl_tls_pkey;
 	X509			*rl_tls_cacertx509;
 	char			*rl_tls_cakey;
@@ -1088,6 +1100,7 @@ struct relayd {
 	struct routerlist	*sc_rts;
 	struct netroutelist	*sc_routes;
 	struct ca_pkeylist	*sc_pkeys;
+	struct relaycertlist	*sc_certs;
 	struct sessionlist	 sc_sessions;
 	char			 sc_demote_group[IFNAMSIZ];
 	u_int16_t		 sc_id;
@@ -1296,6 +1309,8 @@ struct relay	*relay_findbyname(struct relayd *, const char *);
 struct relay	*relay_findbyaddr(struct relayd *, struct relay_config *);
 EVP_PKEY	*pkey_find(struct relayd *, char *hash);
 struct ca_pkey	*pkey_add(struct relayd *, EVP_PKEY *, char *hash);
+struct relay_cert *cert_add(struct relayd *, objid_t);
+struct relay_cert *cert_find(struct relayd *, objid_t);
 char		*relay_load_fd(int, off_t *);
 int		 relay_load_certfiles(struct relayd *, struct relay *);
 int		 expand_string(char *, size_t, const char *, const char *);