Implement socket server code that properly handles async I/O, partial

messages, multiple messages per buffer and important things like
connection limits and file descriptor accounting.  It works with TCP
connections as well as switch(4).  The ofrelay.c part replaces
networking that was in ofp.c and will soon handle all socket
connections of switchd.  It is called "ofrelay" because it will be
used as client, server, and forwarder.

OK rzalamena@

12 files changed, 698 insertions, 233 deletions

diff --git a/usr.sbin/switchd/Makefile b/usr.sbin/switchd/Makefile
index e6b3d60c124..7798638705b 100644
--- a/usr.sbin/switchd/Makefile
+++ b/usr.sbin/switchd/Makefile
@@ -1,10 +1,10 @@
-# $OpenBSD: Makefile,v 1.3 2016/07/20 14:04:51 reyk Exp $
+# $OpenBSD: Makefile,v 1.4 2016/09/30 11:57:57 reyk Exp $
 
 PROG=		switchd
 MAN=		switchd.8 switchd.conf.5
 
-SRCS=		imsg_util.c log.c packet.c switch.c timer.c util.c
-SRCS+=		switchd.c ofp.c ofp10.c ofp13.c control.c proc.c
+SRCS=		imsg_util.c log.c packet.c proc.c switch.c timer.c util.c
+SRCS+=		switchd.c control.c ofp.c ofp10.c ofp13.c ofrelay.c
 SRCS+=		${.OBJDIR}/ofp_map.c ${.OBJDIR}/ofp10_map.c
 SRCS+=		parse.y ofcconn.c
 
diff --git a/usr.sbin/switchd/imsg_util.c b/usr.sbin/switchd/imsg_util.c
index dccc01b8def..e7e79fdee1d 100644
--- a/usr.sbin/switchd/imsg_util.c
+++ b/usr.sbin/switchd/imsg_util.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: imsg_util.c,v 1.4 2016/09/29 17:03:00 reyk Exp $	*/
+/*	$OpenBSD: imsg_util.c,v 1.5 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2010-2016 Reyk Floeter <reyk@openbsd.org>
@@ -208,6 +208,15 @@ ibuf_setsize(struct ibuf *buf, size_t len)
 }
 
 int
+ibuf_setmax(struct ibuf *buf, size_t len)
+{
+	if (len > buf->size)
+		return (-1);
+	buf->max = len;
+	return (0);
+}
+
+int
 ibuf_prepend(struct ibuf *buf, void *data, size_t len)
 {
 	struct ibuf	*new;
diff --git a/usr.sbin/switchd/ofcconn.c b/usr.sbin/switchd/ofcconn.c
index 859be960cc2..522309f4bea 100644
--- a/usr.sbin/switchd/ofcconn.c
+++ b/usr.sbin/switchd/ofcconn.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ofcconn.c,v 1.9 2016/09/14 13:46:51 rzalamena Exp $	*/
+/*	$OpenBSD: ofcconn.c,v 1.10 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2016 YASUOKA Masahiko <yasuoka@openbsd.org>
@@ -40,7 +40,6 @@ int	 ofcconn_dispatch_parent(int, struct privsep_proc *, struct imsg *);
 static struct privsep_proc procs[] = {
 	{ "parent",	PROC_PARENT,	ofcconn_dispatch_parent },
 	{ "control",	PROC_CONTROL,	NULL },
-	{ "ofp",	PROC_OFP,	NULL }
 };
 
 struct ofcconn;
diff --git a/usr.sbin/switchd/ofp.c b/usr.sbin/switchd/ofp.c
index 6bd4944bad4..ac1bb57f3d6 100644
--- a/usr.sbin/switchd/ofp.c
+++ b/usr.sbin/switchd/ofp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ofp.c,v 1.9 2016/09/29 18:25:54 reyk Exp $	*/
+/*	$OpenBSD: ofp.c,v 1.10 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -38,45 +38,27 @@
 #include "switchd.h"
 #include "ofp_map.h"
 
-int	 ofp_dispatch_control(int, struct privsep_proc *, struct imsg *);
 int	 ofp_dispatch_parent(int, struct privsep_proc *, struct imsg *);
+int	 ofp_dispatch_control(int, struct privsep_proc *, struct imsg *);
 void	 ofp_run(struct privsep *, struct privsep_proc *, void *);
 int	 ofp_add_device(struct switchd *, int, const char *);
 
-static unsigned int	 id = 0;
-
 static struct privsep_proc procs[] = {
 	{ "control",	PROC_CONTROL,	ofp_dispatch_control },
-	{ "parent",	PROC_PARENT,	ofp_dispatch_parent },
-	{ "ofcconn",	PROC_OFCCONN,	NULL }
+	{ "parent",	PROC_PARENT,	ofp_dispatch_parent }
 };
 
-static TAILQ_HEAD(, switch_connection) conn_head =
-    TAILQ_HEAD_INITIALIZER(conn_head);
-
 void
 ofp(struct privsep *ps, struct privsep_proc *p)
 {
-	struct switchd		*sc = ps->ps_env;
-	struct switch_server	*srv = &sc->sc_server;
-
-	if ((sc->sc_tap = switchd_tap()) == -1)
-		fatal("tap");
-
-	log_info("listen on %s", print_host(&srv->srv_addr, NULL, 0));
-
-	if ((srv->srv_fd = switchd_listen((struct sockaddr *)
-	    &srv->srv_addr)) == -1)
-		fatal("listen");
-
+	ofrelay(ps, p);
 	proc_run(ps, p, procs, nitems(procs), ofp_run, NULL);
 }
 
 void
 ofp_run(struct privsep *ps, struct privsep_proc *p, void *arg)
 {
-	struct switchd		*sc = ps->ps_env;
-	struct switch_server	*srv = &sc->sc_server;
+	struct switchd	*sc = ps->ps_env;
 
 	/*
 	 * pledge in the ofp process:
@@ -87,8 +69,10 @@ ofp_run(struct privsep *ps, struct privsep_proc *p, void *arg)
 	if (pledge("stdio inet recvfd", NULL) == -1)
 		fatal("pledge");
 
-	event_set(&srv->srv_ev, srv->srv_fd, EV_READ, ofp_accept, srv);
-	event_add(&srv->srv_ev, NULL);
+	TAILQ_INIT(&sc->sc_conns);
+	sc->sc_tap = -1;
+
+	ofrelay_run(ps, p, NULL);
 }
 
 int
@@ -107,51 +91,45 @@ ofp_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 int
 ofp_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
-	struct switchd			*sc = p->p_ps->ps_env;
+	struct privsep			*ps = p->p_ps;
+	struct switchd			*sc = ps->ps_env;
+	struct sockaddr_un		 un;
 	struct switch_device		*sdv;
-	struct switch_connection	*c;
+	struct switch_connection	*con;
 
 	switch (imsg->hdr.type) {
+	case IMSG_TAPFD:
+		if (sc->sc_tap != -1)
+			close(sc->sc_tap);
+		sc->sc_tap = imsg->fd;
+		return (0);
 	case IMSG_CTL_DEVICE_CONNECT:
 	case IMSG_CTL_DEVICE_DISCONNECT:
-		if (IMSG_DATA_SIZE(imsg) < sizeof(*sdv)) {
-			log_warnx("%s: IMSG_CTL_DEVICE_CONNECT: "
-			    "message size is wrong", __func__);
+		IMSG_SIZE_CHECK(imsg, sdv);
+		sdv = imsg->data;
+
+		if (strlcpy(un.sun_path, sdv->sdv_device,
+		    sizeof(un.sun_path)) >= sizeof(un.sun_path)) {
+			log_warnx("invalid device: %s", sdv->sdv_device);
 			return (0);
 		}
-		sdv = imsg->data;
+		un.sun_family = AF_UNIX;
+		un.sun_len = sizeof(un);
+
 		if (imsg->hdr.type == IMSG_CTL_DEVICE_CONNECT)
-			ofp_add_device(sc, imsg->fd, sdv->sdv_device);
-		else {
-			TAILQ_FOREACH(c, &conn_head, con_next) {
-				if (c->con_peer.ss_family == AF_UNIX &&
-				    strcmp(sdv->sdv_device,
-				    ((struct sockaddr_un *)&c->con_peer)
-				    ->sun_path) == 0)
-					break;
-			}
-			if (c)
-				ofp_close(c);
-		}
+			ofrelay_attach(&sc->sc_server,
+			    imsg->fd, (struct sockaddr *)&un);
+		else if ((con =
+		    switchd_connbyaddr(sc, (struct sockaddr *)&un)) != NULL)
+			ofp_close(con);
 		return (0);
 	default:
-
 		break;
 	}
 
 	return (-1);
 }
 
-void
-ofp_close(struct switch_connection *con)
-{
-	log_info("%s: connection %u closed", __func__, con->con_id);
-	event_del(&con->con_ev);
-	switch_remove(con->con_sc, con->con_switch);
-	close(con->con_fd);
-	TAILQ_REMOVE(&conn_head, con, con_next);
-}
-
 int
 ofp_validate_header(struct switchd *sc,
     struct sockaddr_storage *src, struct sockaddr_storage *dst,
@@ -186,46 +164,25 @@ ofp_validate_header(struct switchd *sc,
 	return (0);
 }
 
-void
-ofp_read(int fd, short event, void *arg)
+int
+ofp_input(struct switch_connection *con, struct ibuf *ibuf)
 {
-	uint8_t			 buf[SWITCHD_READ_BUFFER];
-	struct switch_connection *con = arg;
-	struct switch_control	*sw;
 	struct switchd		*sc = con->con_sc;
 	struct ofp_header	*oh;
-	ssize_t			 len;
-	struct ibuf		*ibuf = NULL;
-
-	event_add(&con->con_ev, NULL);
-	if ((event & EV_TIMEOUT))
-		goto fail;
-
-	if ((len = read(fd, buf, sizeof(buf))) == -1)
-		goto fail;
-	if (len == 0)
-		goto fail;
-
-	if ((ibuf = ibuf_new(buf, len)) == NULL)
-		goto fail;
-
-	sw = con->con_switch;
-	log_debug("%s: connection %d: %ld bytes from switch %u", __func__,
-	    con->con_id, len, sw == NULL ? 0 : sw->sw_id);
 
 	if ((oh = ibuf_seek(ibuf, 0, sizeof(*oh))) == NULL) {
 		log_debug("short header");
-		goto fail;
+		return (-1);
 	}
 
 	switch (oh->oh_version) {
 	case OFP_V_1_0:
 		if (ofp10_input(sc, con, oh, ibuf) != 0)
-			goto fail;
+			return (-1);
 		break;
 	case OFP_V_1_3:
 		if (ofp13_input(sc, con, oh, ibuf) != 0)
-			goto fail;
+			return (-1);
 		break;
 	case OFP_V_1_1:
 	case OFP_V_1_2:
@@ -234,147 +191,52 @@ ofp_read(int fd, short event, void *arg)
 		(void)ofp10_validate(sc,
 		    &con->con_peer, &con->con_local, oh, ibuf);
 		ofp10_hello(sc, con, oh, ibuf);
-		goto fail;
+		return (-1);
 	}
 
-	ibuf_release(ibuf);
-	return;
-
- fail:
-	ibuf_release(ibuf);
-	ofp_close(con);
+	return (0);
 }
 
 int
 ofp_output(struct switch_connection *con, struct ofp_header *oh,
     struct ibuf *obuf)
 {
-	struct iovec		 iov[2];
-	int			 cnt = 0;
-	void			*data;
-	ssize_t			 len;
-
-	if (oh != NULL) {
-		iov[cnt].iov_base = oh;
-		iov[cnt++].iov_len = sizeof(*oh);
-	}
+	struct ibuf	*buf;
 
-	if (ibuf_length(obuf)) {
-		if (oh != NULL && (ibuf_seek(obuf, 0, sizeof(*oh)) == NULL))
-			return (-1);
-		len = ibuf_dataleft(obuf);
-		if (len < 0) {
-			return (-1);
-		} else if (len > 0 &&
-		    (data = ibuf_getdata(obuf, len)) != NULL) {
-			iov[cnt].iov_base = data;
-			iov[cnt++].iov_len = len;
-		}
-	}
-
-	if (cnt == 0)
+	if ((buf = ibuf_static()) == NULL)
 		return (-1);
-
-	/* XXX */
-	if (writev(con->con_fd, iov, cnt) == -1)
+	if ((oh != NULL) &&
+	    (ibuf_add(buf, oh, sizeof(*oh)) == -1)) {
+		ibuf_release(buf);
 		return (-1);
+	}
+	if ((obuf != NULL) &&
+	    (ibuf_cat(buf, obuf) == -1)) {
+		ibuf_release(buf);
+		return (-1);
+	}
+	ibuf_close(&con->con_wbuf, buf);
 
 	return (0);
 }
 
 int
-ofp_add_device(struct switchd *sc, int fd, const char *name)
+ofp_open(struct privsep *ps, struct switch_connection *con)
 {
-	struct switch_connection	*con = NULL;
-	struct sockaddr_un		*sun;
-	struct switch_control		*sw;
-
-	if ((con = calloc(1, sizeof(*con))) == NULL) {
-		log_warn("calloc");
-		goto fail;
-	}
-	con->con_fd = fd;
-	con->con_sc = sc;
-	con->con_id = ++id;
-	sun = (struct sockaddr_un *)&con->con_peer;
-	sun->sun_family = AF_LOCAL;
-	strlcpy(sun->sun_path, name, sizeof(sun->sun_path));
+	struct switch_control	*sw;
 
 	/* Get associated switch, if it exists */
 	sw = switch_get(con);
 
-	log_info("%s: new device %u (%s) from switch %u",
-	    __func__, con->con_id, name, sw == NULL ? 0 : sw->sw_id);
-
-	bzero(&con->con_ev, sizeof(con->con_ev));
-	event_set(&con->con_ev, con->con_fd, EV_READ, ofp_read, con);
-	event_add(&con->con_ev, NULL);
-
-	TAILQ_INSERT_TAIL(&conn_head, con, con_next);
+	log_info("%s: new connection %u.%u from switch %u",
+	    __func__, con->con_id, con->con_instance,
+	    sw == NULL ? 0 : sw->sw_id);
 
 	return (0);
-fail:
-	if (fd != -1)
-		close(fd);
-	free(con);
-
-	return (-1);
 }
 
 void
-ofp_accept(int fd, short event, void *arg)
+ofp_close(struct switch_connection *con)
 {
-	struct switch_server	*server = arg;
-	struct switch_connection *con = NULL;
-	struct switchd		*sc = server->srv_sc;
-	struct switch_control	*sw;
-	struct sockaddr_storage	 ss;
-	socklen_t		 slen;
-	int			 s;
-
-	event_add(&server->srv_ev, NULL);
-	if ((event & EV_TIMEOUT))
-		return;
-
-	/* XXX accept_reserve() */
-	slen = sizeof(ss);
-	if ((s = accept(fd, (struct sockaddr *)&ss, &slen)) == -1) {
-		log_warn("accept");
-		goto fail;
-	}
-
-	if ((con = calloc(1, sizeof(*con))) == NULL) {
-		log_warn("calloc");
-		goto fail;
-	}
-
-	slen = sizeof(con->con_local);
-	if (getsockname(s, (struct sockaddr *)&con->con_local, &slen) == -1) {
-		log_warn("getsockname");
-		goto fail;
-	}
-
-	con->con_fd = s;
-	con->con_sc = sc;
-	con->con_id = ++id;
-	con->con_port = htons(socket_getport(&ss));
-	memcpy(&con->con_peer, &ss, sizeof(ss));
-
-	/* Get associated switch, if it exists */
-	sw = switch_get(con);
-
-	log_info("%s: new connection %u from switch %u",
-	    __func__, con->con_id, sw == NULL ? 0 : sw->sw_id);
-
-	bzero(&con->con_ev, sizeof(con->con_ev));
-	event_set(&con->con_ev, con->con_fd, EV_READ, ofp_read, con);
-	event_add(&con->con_ev, NULL);
-
-	TAILQ_INSERT_TAIL(&conn_head, con, con_next);
-
-	return;
- fail:
-	if (s != -1)
-		close(s);
-	free(con);
+	ofrelay_close(con);
 }
diff --git a/usr.sbin/switchd/ofp10.c b/usr.sbin/switchd/ofp10.c
index 42e7fd15e8a..45668debf8f 100644
--- a/usr.sbin/switchd/ofp10.c
+++ b/usr.sbin/switchd/ofp10.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ofp10.c,v 1.9 2016/09/29 18:25:54 reyk Exp $	*/
+/*	$OpenBSD: ofp10.c,v 1.10 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -258,7 +258,6 @@ ofp10_hello(struct switchd *sc, struct switch_connection *con,
 	if (oh->oh_version == OFP_V_1_0 &&
 	    switch_add(con) == NULL) {
 		log_debug("%s: failed to add switch", __func__);
-		ofp_close(con);
 		return (-1);
 	}
 
diff --git a/usr.sbin/switchd/ofp13.c b/usr.sbin/switchd/ofp13.c
index f4d7688ddc4..10abbe4ead1 100644
--- a/usr.sbin/switchd/ofp13.c
+++ b/usr.sbin/switchd/ofp13.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ofp13.c,v 1.15 2016/09/29 18:25:54 reyk Exp $	*/
+/*	$OpenBSD: ofp13.c,v 1.16 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -595,12 +595,15 @@ ofp13_input(struct switchd *sc, struct switch_connection *con,
 		return (-1);
 
 	if (ofp13_callbacks[oh->oh_type].cb == NULL) {
-		log_debug("message not supported: %s",
+		log_debug("%s: message not supported: %s", __func__,
 		    print_map(oh->oh_type, ofp_t_map));
 		return (-1);
 	}
-	if (ofp13_callbacks[oh->oh_type].cb(sc, con, oh, ibuf) != 0)
+	if (ofp13_callbacks[oh->oh_type].cb(sc, con, oh, ibuf) != 0) {
+		log_debug("%s: message parsing failed: %s", __func__,
+		    print_map(oh->oh_type, ofp_t_map));
 		return (-1);
+	}
 
 	return (0);
 }
@@ -611,7 +614,6 @@ ofp13_hello(struct switchd *sc, struct switch_connection *con,
 {
 	if (switch_add(con) == NULL) {
 		log_debug("%s: failed to add switch", __func__);
-		ofp_close(con);
 		return (-1);
 	}
 
diff --git a/usr.sbin/switchd/ofrelay.c b/usr.sbin/switchd/ofrelay.c
new file mode 100644
index 00000000000..f2b04b23d7e
--- /dev/null
+++ b/usr.sbin/switchd/ofrelay.c
@@ -0,0 +1,505 @@
+/*	$OpenBSD: ofrelay.c,v 1.1 2016/09/30 11:57:57 reyk Exp $	*/
+
+/*
+ * Copyright (c) 2016 Reyk Floeter <reyk@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <sys/queue.h>
+
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <fcntl.h>
+#include <pwd.h>
+#include <errno.h>
+#include <event.h>
+
+#include "switchd.h"
+
+void	 ofrelay_close(struct switch_connection *);
+void	 ofrelay_event(int, short, void *);
+int	 ofrelay_input(int, short, void *);
+int	 ofrelay_output(int, short, void *);
+void	 ofrelay_accept(int, short, void *);
+ssize_t	 ofrelay_read(struct ibuf *, void *, size_t);
+void	 ofrelay_inflight_dec(struct switch_connection *, const char *);
+
+void	*ofrelay_input_open(struct switch_connection *,
+	    struct ibuf *, ssize_t *);
+ssize_t	 ofrelay_input_close(struct switch_connection *,
+	    struct ibuf *, ssize_t);
+int	 ofrelay_input_done(struct switch_connection *, struct ibuf *);
+int	 ofrelay_bufget(struct switch_connection *, struct ibuf *);
+void	 ofrelay_bufput(struct switch_connection *, struct ibuf *);
+
+volatile int	 ofrelay_sessions;
+volatile int	 ofrelay_inflight;
+static uint32_t	 ofrelay_conid;
+
+void
+ofrelay(struct privsep *ps, struct privsep_proc *p)
+{
+	struct switchd		*sc = ps->ps_env;
+	struct switch_server	*srv = &sc->sc_server;
+
+	log_info("listen on %s", print_host(&srv->srv_addr, NULL, 0));
+
+	if ((srv->srv_fd = switchd_listen((struct sockaddr *)
+	    &srv->srv_addr)) == -1)
+		fatal("listen");
+}
+
+void
+ofrelay_run(struct privsep *ps, struct privsep_proc *p, void *arg)
+{
+	struct switchd		*sc = ps->ps_env;
+	struct switch_server	*srv = &sc->sc_server;
+
+	TAILQ_INIT(&sc->sc_conns);
+	TAILQ_INIT(&sc->sc_devs);
+
+	srv->srv_sc = sc;
+	event_set(&srv->srv_ev, srv->srv_fd, EV_READ, ofrelay_accept, srv);
+	event_add(&srv->srv_ev, NULL);
+	evtimer_set(&srv->srv_evt, ofrelay_accept, srv);
+}
+
+void
+ofrelay_close(struct switch_connection *con)
+{
+	struct switchd		*sc = con->con_sc;
+	struct switch_server	*srv = con->con_srv;
+
+	log_info("%s: connection %u.%u closed", __func__,
+	    con->con_id, con->con_instance);
+
+	if (event_initialized(&con->con_ev))
+		event_del(&con->con_ev);
+
+	TAILQ_REMOVE(&sc->sc_conns, con, con_entry);
+	ofrelay_sessions--;
+
+	switch_remove(con->con_sc, con->con_switch);
+	msgbuf_clear(&con->con_wbuf);
+	ibuf_release(con->con_ibuf);
+	ibuf_release(con->con_rbuf);
+	close(con->con_fd);
+
+	ofrelay_inflight_dec(con, __func__);
+
+	/* Some file descriptors are available again. */
+	if (evtimer_pending(&srv->srv_evt, NULL)) {
+		DPRINTF("%s: accepting again", __func__);
+		evtimer_del(&srv->srv_evt);
+		event_add(&srv->srv_ev, NULL);
+	}
+
+	free(con);
+}
+
+void
+ofrelay_event(int fd, short event, void *arg)
+{
+	struct switch_connection	*con = arg;
+	struct ibuf			*rbuf = con->con_rbuf;
+	void				*buf;
+	ssize_t				 len, rlen;
+	const char			*error = NULL;
+
+	event_add(&con->con_ev, NULL);
+	if (event & EV_TIMEOUT) {
+		error = "timeout";
+		goto fail;
+	}
+	if (event & EV_WRITE) {
+		if (ofrelay_output(fd, event, arg) == -1) {
+			error = "write";
+			goto fail;
+		}
+	}
+	if (event & EV_READ) {
+		/*
+		 * This read was originally done in ofrelay_input() without
+		 * the extra buffer but switch(4) does not support partial
+		 * reads so we have to read the whole input at once.
+		 */
+		if ((buf = ofrelay_input_open(con, rbuf, &len)) == NULL) {
+			error = "open input";
+			goto fail;
+		}
+
+		if ((rlen = read(fd, buf, len)) == -1) {
+			if (errno == EINTR || errno == EAGAIN)
+				return;
+			error = "read";
+			goto fail;
+		}
+
+		if ((len = ofrelay_input_close(con, rbuf, rlen)) == -1) {
+			error = "close input";
+			goto fail;
+		} else if (len == 0) {
+			ofrelay_close(con);
+			return;
+		}
+
+		do {
+			rlen = ofrelay_input(fd, event, arg);
+			if (rlen == -1) {
+				error = "input";
+				goto fail;
+			}
+		} while (rlen);
+	}
+
+ fail:
+	if (error != NULL) {
+		DPRINTF("%s: %s error", __func__, error);
+		ofrelay_close(con);
+	}
+}
+
+int
+ofrelay_input(int fd, short event, void *arg)
+{
+	struct switch_connection	*con = arg;
+	struct ibuf		*ibuf = con->con_ibuf;
+	ssize_t			 len, rlen;
+	size_t			 hlen;
+	void			*buf;
+	struct ofp_header	*oh = NULL;
+
+	if ((buf = ofrelay_input_open(con, ibuf, &len)) == NULL) {
+		log_warn("%s: fd %d, failed to get buffer", __func__, fd);
+		return (-1);
+	}
+
+	/* If we got a new buffer, read the complete openflow header first */
+	if ((hlen = ibuf_length(ibuf)) < sizeof(struct ofp_header)) {
+		oh = (struct ofp_header *)ibuf_data(ibuf);
+		len = sizeof(*oh) - hlen;
+	}
+
+	if ((rlen = ofrelay_read(con->con_rbuf, buf, len)) == -1) {
+		log_warn("%s: fd %d, failed to read", __func__, fd);
+		return (-1);
+	}
+
+	print_hex(buf, 0, rlen);
+
+	DPRINTF("%s: connection %u.%u read %zd bytes", __func__,
+	    con->con_id, con->con_instance, rlen);
+
+	if ((len = ofrelay_input_close(con, ibuf, rlen)) == -1)
+		return (-1);
+	else if (len == 0)
+		goto done;
+
+	/* After we verified the openflow header, set the size accordingly */
+	if (oh != NULL && (hlen + rlen) == sizeof(*oh)) {
+		switch (oh->oh_version) {
+		case OFP_V_1_0:
+		case OFP_V_1_1:
+		case OFP_V_1_2:
+		case OFP_V_1_3:
+			break;
+		default:
+			DPRINTF("%s: fd %d, openflow version 0x%02x length %d"
+			    " not supported", __func__, fd, oh->oh_version,
+			    ntohs(oh->oh_length));
+			return (-1);
+		}
+
+		len = ntohs(oh->oh_length);
+		if (len == sizeof(*oh)) {
+			len = 0;
+		} else if (len > (ssize_t)ibuf->size) {
+			log_debug("%s: buffer too big: %zu > %zd", __func__,
+			    len, ibuf->size);
+			return (-1);
+		} else
+			ibuf_setmax(ibuf, len);
+	}
+
+	if (len > 0)
+		return (len);
+ done:
+	return (ofrelay_input_done(con, ibuf));
+}
+
+ssize_t
+ofrelay_read(struct ibuf *rbuf, void *buf, size_t size)
+{
+	void	*p;
+	size_t	 len;
+
+	if ((len = ibuf_dataleft(rbuf)) > size)
+		len = size;
+
+	if ((p = ibuf_getdata(rbuf, len)) == NULL)
+		return (-1);
+
+	/* This could be avoided */
+	memcpy(buf, p, len);
+
+	return ((ssize_t)len);
+}
+
+void *
+ofrelay_input_open(struct switch_connection *con,
+    struct ibuf *buf, ssize_t *len)
+{
+	ssize_t	 left;
+
+	left = buf->max - buf->wpos;
+	if (left == 0) {
+		ofrelay_bufget(con, buf);
+		left = buf->max;
+	}
+	if (len)
+		*len = left;
+
+	return (buf->buf + buf->wpos);
+}
+
+ssize_t
+ofrelay_input_close(struct switch_connection *con,
+    struct ibuf *buf, ssize_t len)
+{
+	if (len <= 0)
+		return (0);
+	if (buf->wpos + len > buf->max)
+		return (-1);
+	buf->wpos += len;
+
+	return (buf->max - buf->wpos);
+}
+
+int
+ofrelay_input_done(struct switch_connection *con, struct ibuf *buf)
+{
+	struct switch_control	*sw;
+
+	if (buf->wpos == 0) {
+		ofrelay_bufput(con, buf);
+		return (0);
+	}
+
+	sw = con->con_switch;
+	log_debug("%s: connection %u.%u: %ld bytes from switch %u", __func__,
+	    con->con_id, con->con_instance, buf->wpos,
+	    sw == NULL ? 0 : sw->sw_id);
+
+	print_hex(buf->buf, 0, buf->wpos);
+
+	if (ofp_input(con, buf) == -1)
+		return (-1);
+
+	/* Update read buffer */
+	if (ofrelay_bufget(con, buf) == -1)
+		return (-1);
+
+	return (0);
+}
+
+int
+ofrelay_bufget(struct switch_connection *con, struct ibuf *buf)
+{
+	ibuf_reset(buf);
+
+	/* Should be a static buffer with maximum size */
+	if (ibuf_setmax(buf, SWITCHD_MSGBUF_MAX) == -1)
+		fatalx("%s: invalid buffer", __func__);
+
+	return (0);
+}
+
+void
+ofrelay_bufput(struct switch_connection *con, struct ibuf *buf)
+{
+	/* Just reset the buffer */
+	ofrelay_bufget(con, buf);	
+}
+
+int
+ofrelay_output(int fd, short event, void *arg)
+{
+	struct switch_connection	*con = arg;
+	struct msgbuf			*wbuf = &con->con_wbuf;
+	ssize_t				 n;
+	struct ibuf			*buf;
+	size_t				 len;
+	void				*base;
+
+	if (!wbuf->queued)
+		return (0);
+
+	/*
+	 * Only write one packet at a time, this is currently needed
+	 * for switch(4) and other OpenFlow implementations that do
+	 * not handle multiple openflow packets in a single buffer.
+	 */
+	if ((buf = TAILQ_FIRST(&wbuf->bufs)) == NULL)
+		return (0);
+
+	base = buf->buf + buf->rpos;
+	len = buf->wpos - buf->rpos;
+
+ again:
+	if ((n = write(wbuf->fd, base, len)) == -1) {
+		if (errno == EINTR)
+			goto again;
+		if (errno == ENOBUFS || errno == EAGAIN)
+			return (0);
+		return (-1);
+	}
+
+	/* connection closed */
+	if (n == 0)
+		return (-1);
+
+	msgbuf_drain(wbuf, n);
+
+	return (0);
+}
+
+void
+ofrelay_accept(int fd, short event, void *arg)
+{
+	struct switch_server	*srv = arg;
+	struct sockaddr_storage	 ss;
+	int			 s;
+	socklen_t		 slen;
+
+	event_add(&srv->srv_ev, NULL);
+	if (event & EV_TIMEOUT)
+		return;
+
+	slen = sizeof(ss);
+	if ((s = accept4_reserve(fd, (struct sockaddr *)&ss, &slen,
+	    SOCK_NONBLOCK|SOCK_CLOEXEC, SWITCHD_FD_RESERVE,
+	    &ofrelay_inflight)) == -1) {
+		/*
+		 * Pause accept if we are out of file descriptors, or
+		 * libevent will haunt us here too.
+		 */
+		if (errno == ENFILE || errno == EMFILE) {
+			struct timeval evtpause = { 1, 0 };
+
+			event_del(&srv->srv_ev);
+			evtimer_add(&srv->srv_evt, &evtpause);
+			log_warn("%s: deferring connections", __func__);
+		} else
+			log_warn("%s accept", __func__);
+		return;
+	}
+	ss.ss_len = slen;
+
+	(void)ofrelay_attach(srv, s, (struct sockaddr *)&ss);
+}
+
+void
+ofrelay_inflight_dec(struct switch_connection *con, const char *why)
+{
+	if (con != NULL) {
+		/* the flight already left inflight mode. */
+		if (con->con_inflight == 0)
+			return;
+		con->con_inflight = 0;
+	}
+
+	/* the file was never opened, thus this was an inflight client. */
+	ofrelay_inflight--;
+	DPRINTF("%s: inflight decremented, now %d, %s",
+	    __func__, ofrelay_inflight, why);
+}
+
+int
+ofrelay_attach(struct switch_server *srv, int s, struct sockaddr *sa)
+{
+	struct switchd			*sc = srv->srv_sc;
+	struct privsep			*ps = &sc->sc_ps;
+	struct switch_connection	*con = NULL;
+	socklen_t			 slen;
+	int				 ret = -1;
+
+	if (ofrelay_sessions >= SWITCHD_MAX_SESSIONS) {
+		log_warn("too many sessions");
+		goto done;
+	}
+
+	if ((con = calloc(1, sizeof(*con))) == NULL) {
+		log_warn("calloc");
+		goto done;
+	}
+
+	con->con_fd = s;
+	con->con_inflight = 1;
+	con->con_sc = sc;
+	con->con_id = ++ofrelay_conid;
+	con->con_instance = ps->ps_instance + 1;
+	con->con_srv = srv;
+
+	memcpy(&con->con_peer, sa, sa->sa_len);
+	con->con_port = htons(socket_getport(&con->con_peer));
+
+	if (getsockname(s, (struct sockaddr *)&con->con_local, &slen) == -1) {
+		/* Set local sockaddr to AF_UNSPEC */
+		memset(&con->con_local, 0, sizeof(con->con_local));
+	}
+
+	log_info("%s: new connection %u.%u",
+	    __func__, con->con_id, con->con_instance);
+
+	ofrelay_sessions++;
+	TAILQ_INSERT_TAIL(&sc->sc_conns, con, con_entry);
+
+	if ((con->con_rbuf = ibuf_static()) == NULL ||
+	    ofrelay_bufget(con, con->con_rbuf) == -1) {
+		log_warn("rbuf");
+		goto done;
+	}
+	if ((con->con_ibuf = ibuf_static()) == NULL ||
+	    ofrelay_bufget(con, con->con_ibuf) == -1) {
+		log_warn("ibuf");
+		goto done;
+	}
+	msgbuf_init(&con->con_wbuf);
+	con->con_wbuf.fd = s;
+
+	memset(&con->con_ev, 0, sizeof(con->con_ev));
+	event_set(&con->con_ev, con->con_fd, EV_READ|EV_WRITE,
+	    ofrelay_event, con);
+	event_add(&con->con_ev, NULL);
+
+	ret = ofp_open(ps, con);
+
+ done:
+	if (con == NULL)
+		close(s);
+	ofrelay_inflight_dec(con, __func__);
+	if (ret != 0)
+		ofrelay_close(con);
+
+	return (ret);
+}
diff --git a/usr.sbin/switchd/parse.y b/usr.sbin/switchd/parse.y
index 7193ceb9d95..a1c338f2c1c 100644
--- a/usr.sbin/switchd/parse.y
+++ b/usr.sbin/switchd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.1 2016/07/19 16:54:26 reyk Exp $	*/
+/*	$OpenBSD: parse.y,v 1.2 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007-2016 Reyk Floeter <reyk@openbsd.org>
@@ -159,7 +159,7 @@ opttls		: /* empty */	{ $$ = 0; }
 device		: DEVICE STRING optofcconn {
 			struct switch_device		*c;
 
-			TAILQ_FOREACH(c, &conf->sc_conns, sdv_next) {
+			TAILQ_FOREACH(c, &conf->sc_devs, sdv_next) {
 				if (strcmp(c->sdv_device, $2) == 0)
 					break;
 			}
@@ -173,7 +173,7 @@ device		: DEVICE STRING optofcconn {
 				YYERROR;
 			}
 			free($2);
-			TAILQ_INSERT_TAIL(&conf->sc_conns, $3, sdv_next);
+			TAILQ_INSERT_TAIL(&conf->sc_devs, $3, sdv_next);
 		}
 		;
 
diff --git a/usr.sbin/switchd/switchd.c b/usr.sbin/switchd/switchd.c
index 59819858342..8d228f73a59 100644
--- a/usr.sbin/switchd/switchd.c
+++ b/usr.sbin/switchd/switchd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: switchd.c,v 1.11 2016/09/25 23:05:29 jsg Exp $	*/
+/*	$OpenBSD: switchd.c,v 1.12 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -51,9 +51,9 @@ int	 switch_device_cmp(struct switch_device *, struct switch_device *);
 __dead void	usage(void);
 
 static struct privsep_proc procs[] = {
-	{ "ofp",	PROC_OFP, NULL, ofp },
-	{ "control",	PROC_CONTROL, parent_dispatch_control, control },
-	{ "ofcconn",	PROC_OFCCONN, NULL, ofcconn }
+	{ "ofp",	PROC_OFP,	NULL, ofp },
+	{ "control",	PROC_CONTROL,	parent_dispatch_control, control },
+	{ "ofcconn",	PROC_OFCCONN,	NULL, ofcconn }
 };
 
 __dead void
@@ -152,7 +152,7 @@ main(int argc, char *argv[])
 	ps = &sc->sc_ps;
 	ps->ps_env = sc;
 	TAILQ_INIT(&ps->ps_rcsocks);
-	TAILQ_INIT(&sc->sc_conns);
+	TAILQ_INIT(&sc->sc_devs);
 
 	if (parse_config(sc->sc_conffile, sc) == -1) {
 		proc_kill(&sc->sc_ps);
@@ -296,6 +296,33 @@ switchd_tap(void)
 }
 
 
+struct switch_connection *
+switchd_connbyid(struct switchd *sc, unsigned int id, unsigned int instance)
+{
+	struct switch_connection	*con;
+
+	TAILQ_FOREACH(con, &sc->sc_conns, con_entry) {
+		if (con->con_id == id && con->con_instance == instance)
+			return (con);
+	}
+
+	return (NULL);
+}
+
+struct switch_connection *
+switchd_connbyaddr(struct switchd *sc, struct sockaddr *sa)
+{
+	struct switch_connection	*con;
+
+	TAILQ_FOREACH(con, &sc->sc_conns, con_entry) {
+		if (sockaddr_cmp((struct sockaddr *)
+		    &con->con_peer, sa, -1) == 0)
+			return (con);
+	}
+
+	return (NULL);
+}
+
 void
 parent_sig_handler(int sig, short event, void *arg)
 {
@@ -330,8 +357,14 @@ int
 parent_configure(struct switchd *sc)
 {
 	struct switch_device	*c;
+	int			 fd;
+
+	if ((fd = switchd_tap()) == -1)
+		fatal("%s: tap", __func__);
+	proc_compose_imsg(&sc->sc_ps, PROC_OFP, -1,
+	    IMSG_TAPFD, -1, fd, NULL, 0);
 
-	TAILQ_FOREACH(c, &sc->sc_conns, sdv_next) {
+	TAILQ_FOREACH(c, &sc->sc_devs, sdv_next) {
 		parent_device_connect(&sc->sc_ps, c);
 	}
 
@@ -346,20 +379,21 @@ parent_reload(struct switchd *sc)
 	enum privsep_procid		 procid;
 
 	memset(&newconf, 0, sizeof(newconf));
+	TAILQ_INIT(&newconf.sc_devs);
 	TAILQ_INIT(&newconf.sc_conns);
 
 	if (parse_config(sc->sc_conffile, &newconf) != -1) {
-		TAILQ_FOREACH_SAFE(sdv, &sc->sc_conns, sdv_next, sdvn) {
-			TAILQ_FOREACH(osdv, &newconf.sc_conns, sdv_next) {
+		TAILQ_FOREACH_SAFE(sdv, &sc->sc_devs, sdv_next, sdvn) {
+			TAILQ_FOREACH(osdv, &newconf.sc_devs, sdv_next) {
 				if (switch_device_cmp(osdv, sdv) == 0) {
-					TAILQ_REMOVE(&newconf.sc_conns,
+					TAILQ_REMOVE(&newconf.sc_devs,
 					    osdv, sdv_next);
 					break;
 				}
 			}
 			if (osdv == NULL) {
 				/* Removed */
-				TAILQ_REMOVE(&sc->sc_conns, sdv, sdv_next);
+				TAILQ_REMOVE(&sc->sc_devs, sdv, sdv_next);
 				procid = (sdv->sdv_swc.swc_type ==
 				    SWITCH_CONN_LOCAL)
 				    ? PROC_OFP : PROC_OFCCONN;
@@ -368,15 +402,15 @@ parent_reload(struct switchd *sc)
 				    -1, -1, sdv, sizeof(*sdv));
 			} else {
 				/* Keep the existing one */
-				TAILQ_REMOVE(&newconf.sc_conns, osdv, sdv_next);
+				TAILQ_REMOVE(&newconf.sc_devs, osdv, sdv_next);
 				free(osdv);
 			}
 		}
-		TAILQ_FOREACH(sdv, &newconf.sc_conns, sdv_next) {
+		TAILQ_FOREACH(sdv, &newconf.sc_devs, sdv_next) {
 			procid =
 			    (sdv->sdv_swc.swc_type == SWITCH_CONN_LOCAL)
 			    ? PROC_OFP : PROC_OFCCONN;
-			TAILQ_INSERT_TAIL(&sc->sc_conns, sdv, sdv_next);
+			TAILQ_INSERT_TAIL(&sc->sc_devs, sdv, sdv_next);
 			parent_device_connect(&sc->sc_ps, sdv);
 		}
 	}
diff --git a/usr.sbin/switchd/switchd.h b/usr.sbin/switchd/switchd.h
index ae03d05e401..9035d7925ae 100644
--- a/usr.sbin/switchd/switchd.h
+++ b/usr.sbin/switchd/switchd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: switchd.h,v 1.10 2016/09/29 20:46:06 reyk Exp $	*/
+/*	$OpenBSD: switchd.h,v 1.11 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -68,23 +68,36 @@ RB_HEAD(switch_head, switch_control);
 
 struct switch_connection {
 	unsigned int		 con_id;
+	unsigned int		 con_instance;
+
 	int			 con_fd;
+	int			 con_inflight;
+
 	struct sockaddr_storage	 con_peer;
 	struct sockaddr_storage	 con_local;
-	struct switch_control	*con_switch;
 	in_port_t		 con_port;
+	uint32_t		 con_xidnxt;
+
 	struct event		 con_ev;
+	struct ibuf		*con_rbuf;
+	struct ibuf		*con_ibuf;
+	struct msgbuf		 con_wbuf;
+
+	struct switch_control	*con_switch;
 	struct switchd		*con_sc;
-	uint32_t		 con_xidnxt;
+	struct switch_server	*con_srv;
+
 	TAILQ_ENTRY(switch_connection)
-				 con_next;
+				 con_entry;
 };
+TAILQ_HEAD(switch_connections, switch_connection);
 
 struct switch_server {
 	int			 srv_fd;
 	int			 srv_tls;
 	struct sockaddr_storage	 srv_addr;
 	struct event		 srv_ev;
+	struct event		 srv_evt;
 	struct switchd		*srv_sc;
 };
 
@@ -99,6 +112,7 @@ struct switch_device {
 	TAILQ_ENTRY(switch_device)
 				 sdv_next;
 };
+TAILQ_HEAD(switch_devices, switch_device);
 
 struct switchd {
 	struct privsep		 sc_ps;
@@ -110,7 +124,8 @@ struct switchd {
 	unsigned int		 sc_cache_timeout;
 	char			 sc_conffile[PATH_MAX];
 	uint8_t			 sc_opts;
-	TAILQ_HEAD(, switch_device)
+	struct switch_devices	 sc_devs;
+	struct switch_connections
 				 sc_conns;
 };
 
@@ -132,6 +147,10 @@ int		 switchd_listen(struct sockaddr *);
 int		 switchd_sockaddr(const char *, in_port_t, struct sockaddr_storage *);
 int		 switchd_tap(void);
 int		 switchd_open_device(struct privsep *, const char *, size_t);
+struct switch_connection *
+		 switchd_connbyid(struct switchd *, unsigned int, unsigned int);
+struct switch_connection *
+		 switchd_connbyaddr(struct switchd *, struct sockaddr *);
 
 /* packet.c */
 int		 packet_input(struct switchd *, struct switch_control *,
@@ -161,6 +180,8 @@ void		 timer_del(struct switchd *, struct timer *);
 
 /* util.c */
 void		 socket_set_blockmode(int, enum blockmodes);
+int		 accept4_reserve(int, struct sockaddr *, socklen_t *,
+		    int, int, volatile int *);
 in_port_t	 socket_getport(struct sockaddr_storage *);
 int		 sockaddr_cmp(struct sockaddr *, struct sockaddr *, int);
 struct in6_addr *prefixlen2mask6(uint8_t, uint32_t *);
@@ -177,16 +198,24 @@ void		 print_hex(uint8_t *, off_t, size_t);
 void		 getmonotime(struct timeval *);
 int		 parsehostport(const char *, struct sockaddr *, socklen_t);
 
+/* ofrelay.c */
+void		 ofrelay(struct privsep *, struct privsep_proc *);
+void		 ofrelay_run(struct privsep *, struct privsep_proc *, void *);
+int		 ofrelay_attach(struct switch_server *, int,
+		    struct sockaddr *);
+void		 ofrelay_close(struct switch_connection *);
+
 /* ofp.c */
 void		 ofp(struct privsep *, struct privsep_proc *);
 void		 ofp_close(struct switch_connection *);
-void		 ofp_read(int, short, void *);
+int		 ofp_open(struct privsep *, struct switch_connection *);
 int		 ofp_output(struct switch_connection *, struct ofp_header *,
 		    struct ibuf *);
 void		 ofp_accept(int, short, void *);
 int		 ofp_validate_header(struct switchd *,
 		    struct sockaddr_storage *, struct sockaddr_storage *,
 		    struct ofp_header *, uint8_t);
+int		 ofp_input(struct switch_connection *, struct ibuf *);
 
 /* ofp10.c */
 int		 ofp10_hello(struct switchd *, struct switch_connection *,
@@ -212,6 +241,7 @@ int		 ibuf_cat(struct ibuf *, struct ibuf *);
 void		 ibuf_release(struct ibuf *);
 size_t		 ibuf_length(struct ibuf *);
 int		 ibuf_setsize(struct ibuf *, size_t);
+int		 ibuf_setmax(struct ibuf *, size_t);
 uint8_t		*ibuf_data(struct ibuf *);
 void		*ibuf_getdata(struct ibuf *, size_t);
 ssize_t		 ibuf_dataleft(struct ibuf *);
diff --git a/usr.sbin/switchd/types.h b/usr.sbin/switchd/types.h
index e09101f69b8..6bb1598dd2e 100644
--- a/usr.sbin/switchd/types.h
+++ b/usr.sbin/switchd/types.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: types.h,v 1.3 2016/09/14 13:46:51 rzalamena Exp $	*/
+/*	$OpenBSD: types.h,v 1.4 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -30,6 +30,8 @@
 #endif
 #define SWITCHD_SOCKET	"/var/run/" SWITCHD_NAME "d.sock"
 
+#define SWITCHD_MAX_SESSIONS	0xffff
+#define	SWITCHD_FD_RESERVE	5
 #define SWITCHD_CYCLE_BUFFERS	8	/* # of static buffers for mapping */
 #define SWITCHD_READ_BUFFER	0xffff
 #define SWITCHD_MSGBUF_MAX	0xffff
@@ -42,6 +44,7 @@
 
 #define SWITCHD_OFCCONN_TIMEOUT	20	/* connect timeout for OpenFlow ch. */
 
+
 #ifndef ETHER_ADDR_LEN
 #define ETHER_ADDR_LEN		6
 #endif
@@ -66,7 +69,8 @@ enum imsg_type {
 	IMSG_CTL_MAC,
 	IMSG_CTL_SHOW_SUM,
 	IMSG_CTL_DEVICE_CONNECT,
-	IMSG_CTL_DEVICE_DISCONNECT
+	IMSG_CTL_DEVICE_DISCONNECT,
+	IMSG_TAPFD
 };
 
 enum privsep_procid {
diff --git a/usr.sbin/switchd/util.c b/usr.sbin/switchd/util.c
index 716423c7460..c0066d081be 100644
--- a/usr.sbin/switchd/util.c
+++ b/usr.sbin/switchd/util.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: util.c,v 1.3 2016/09/29 20:46:06 reyk Exp $	*/
+/*	$OpenBSD: util.c,v 1.4 2016/09/30 11:57:57 reyk Exp $	*/
 
 /*
  * Copyright (c) 2013-2016 Reyk Floeter <reyk@openbsd.org>
@@ -26,11 +26,13 @@
 #include <netinet/if_ether.h>
 #include <netinet/tcp.h>
 
+#include <unistd.h>
 #include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <string.h>
+#include <errno.h>
 #include <event.h>
 
 #include "switchd.h"
@@ -55,6 +57,25 @@ socket_set_blockmode(int fd, enum blockmodes bm)
 		fatal("fcntl F_SETFL");
 }
 
+int
+accept4_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    int flags, int reserve, volatile int *counter)
+{
+	int	 fd;
+
+	if (getdtablecount() + reserve + *counter >= getdtablesize()) {
+		errno = EMFILE;
+		return (-1);
+	}
+
+	if ((fd = accept4(sockfd, addr, addrlen, flags)) != -1) {
+		(*counter)++;
+		DPRINTF("%s: inflight incremented, now %d",__func__, *counter);
+	}
+
+	return (fd);
+}
+
 in_port_t
 socket_getport(struct sockaddr_storage *ss)
 {