Removes the KERNEL_LOCK() from bridge(4)'s output fast-path.

This redefines the ifp <-> bridge relationship.  No lock can be
currently used across the multiples contexts where the bridge has
tentacles to protect a pointer, use an interface index.

Tested by various, ok dlg@, visa@

8 files changed, 198 insertions, 163 deletions

diff --git a/sys/net/bridgectl.c b/sys/net/bridgectl.c
index df22a977d15..4d195aa4af3 100644
--- a/sys/net/bridgectl.c
+++ b/sys/net/bridgectl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bridgectl.c,v 1.17 2019/03/08 17:48:35 mpi Exp $	*/
+/*	$OpenBSD: bridgectl.c,v 1.18 2019/04/28 22:15:57 mpi Exp $	*/
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -84,8 +84,7 @@ bridgectl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
@@ -126,8 +125,7 @@ bridgectl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
@@ -137,6 +135,7 @@ bridgectl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = EINVAL;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		if (brlreq->ifbr_flags & BRL_FLAG_IN) {
 			error = bridge_addrule(bif, brlreq, 0);
 			if (error)
@@ -154,11 +153,11 @@ bridgectl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		bridge_flushrule(bif);
 		break;
 	case SIOCBRDGGRL:
@@ -167,11 +166,11 @@ bridgectl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		error = bridge_brlconf(bif, bc);
 		break;
 	default:
@@ -206,7 +205,7 @@ bridge_rtupdate(struct bridge_softc *sc, struct ether_addr *ea,
 			goto done;
 
 		bcopy(ea, &p->brt_addr, sizeof(p->brt_addr));
-		p->brt_if = ifp;
+		p->brt_ifidx = ifp->if_index;
 		p->brt_age = 1;
 		bridge_copytag(brtag, &p->brt_tunnel);
 
@@ -227,16 +226,14 @@ bridge_rtupdate(struct bridge_softc *sc, struct ether_addr *ea,
 		dir = memcmp(ea, &q->brt_addr, sizeof(q->brt_addr));
 		if (dir == 0) {
 			if (setflags) {
-				q->brt_if = ifp;
+				q->brt_ifidx = ifp->if_index;
 				q->brt_flags = flags;
 			} else if (!(q->brt_flags & IFBAF_STATIC))
-				q->brt_if = ifp;
+				q->brt_ifidx = ifp->if_index;
 
-			if (q->brt_if == ifp)
+			if (q->brt_ifidx == ifp->if_index)
 				q->brt_age = 1;
-			ifp = q->brt_if;
 			bridge_copytag(brtag, &q->brt_tunnel);
-
 			goto want;
 		}
 
@@ -248,7 +245,7 @@ bridge_rtupdate(struct bridge_softc *sc, struct ether_addr *ea,
 				goto done;
 
 			bcopy(ea, &p->brt_addr, sizeof(p->brt_addr));
-			p->brt_if = ifp;
+			p->brt_ifidx = ifp->if_index;
 			p->brt_age = 1;
 			bridge_copytag(brtag, &p->brt_tunnel);
 
@@ -270,7 +267,7 @@ bridge_rtupdate(struct bridge_softc *sc, struct ether_addr *ea,
 				goto done;
 
 			bcopy(ea, &p->brt_addr, sizeof(p->brt_addr));
-			p->brt_if = ifp;
+			p->brt_ifidx = ifp->if_index;
 			p->brt_age = 1;
 			bridge_copytag(brtag, &p->brt_tunnel);
 
@@ -291,11 +288,12 @@ want:
 	return (error);
 }
 
-struct ifnet *
-bridge_rtlookup(struct bridge_softc *sc, struct ether_addr *ea, struct mbuf *m)
+unsigned int
+bridge_rtlookup(struct ifnet *brifp, struct ether_addr *ea, struct mbuf *m)
 {
+	struct bridge_softc *sc = brifp->if_softc;
 	struct bridge_rtnode *p = NULL;
-	struct ifnet *ifp = NULL;
+	unsigned int ifidx = 0;
 	u_int32_t h;
 	int dir;
 
@@ -311,7 +309,7 @@ bridge_rtlookup(struct bridge_softc *sc, struct ether_addr *ea, struct mbuf *m)
 		}
 	}
 	if (p != NULL) {
-		ifp = p->brt_if;
+		ifidx = p->brt_ifidx;
 
 		if (p->brt_family != AF_UNSPEC && m != NULL) {
 			struct bridge_tunneltag *brtag;
@@ -323,7 +321,7 @@ bridge_rtlookup(struct bridge_softc *sc, struct ether_addr *ea, struct mbuf *m)
 	}
 	mtx_leave(&sc->sc_mtx);
 
-	return (ifp);
+	return (ifidx);
 }
 
 u_int32_t
@@ -378,16 +376,14 @@ void
 bridge_rtagenode(struct ifnet *ifp, int age)
 {
 	struct bridge_softc *sc;
-	struct bridge_iflist *bif;
 	struct bridge_rtnode *n;
+	struct ifnet *bifp;
 	int i;
 
-	bif = (struct bridge_iflist *)ifp->if_bridgeport;
-	if (bif == NULL)
-		return;
-	sc = bif->bridge_sc;
-	if (sc == NULL)
+	bifp = if_get(ifp->if_bridgeidx);
+	if (bifp == NULL)
 		return;
+	sc = bifp->if_softc;
 
 	/*
 	 * If the age is zero then flush, otherwise set all the expiry times to
@@ -400,7 +396,7 @@ bridge_rtagenode(struct ifnet *ifp, int age)
 		for (i = 0; i < BRIDGE_RTABLE_SIZE; i++) {
 			LIST_FOREACH(n, &sc->sc_rts[i], brt_next) {
 				/* Cap the expiry time to 'age' */
-				if (n->brt_if == ifp &&
+				if (n->brt_ifidx == ifp->if_index &&
 				    n->brt_age > time_uptime + age &&
 				    (n->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC)
 					n->brt_age = time_uptime + age;
@@ -408,6 +404,8 @@ bridge_rtagenode(struct ifnet *ifp, int age)
 		}
 		mtx_leave(&sc->sc_mtx);
 	}
+
+	if_put(bifp);
 }
 
 /*
@@ -479,7 +477,7 @@ bridge_rtdelete(struct bridge_softc *sc, struct ifnet *ifp, int dynonly)
 	for (i = 0; i < BRIDGE_RTABLE_SIZE; i++) {
 		n = LIST_FIRST(&sc->sc_rts[i]);
 		while (n != NULL) {
-			if (n->brt_if != ifp) {
+			if (n->brt_ifidx != ifp->if_index) {
 				/* Not ours */
 				n = LIST_NEXT(n, brt_next);
 				continue;
@@ -531,13 +529,21 @@ bridge_rtfind(struct bridge_softc *sc, struct ifbaconf *baconf)
 	mtx_enter(&sc->sc_mtx);
 	for (k = 0; k < BRIDGE_RTABLE_SIZE; k++) {
 		LIST_FOREACH(n, &sc->sc_rts[k], brt_next) {
+			struct ifnet *ifp;
+
 			if (i >= total)
 				goto done;
 			bareq = &bareqs[i];
+
+			ifp = if_get(n->brt_ifidx);
+			if (ifp == NULL)
+				continue;
+			bcopy(ifp->if_xname, bareq->ifba_ifsname,
+			    sizeof(bareq->ifba_ifsname));
+			if_put(ifp);
+
 			bcopy(sc->sc_if.if_xname, bareq->ifba_name,
 			    sizeof(bareq->ifba_name));
-			bcopy(n->brt_if->if_xname, bareq->ifba_ifsname,
-			    sizeof(bareq->ifba_ifsname));
 			bcopy(&n->brt_addr, &bareq->ifba_dst,
 			    sizeof(bareq->ifba_dst));
 			bridge_copyaddr(&n->brt_tunnel.brtag_peer.sa,
@@ -565,7 +571,7 @@ bridge_update(struct ifnet *ifp, struct ether_addr *ea, int delete)
 
 	addr = (u_int8_t *)ea;
 
-	bif = (struct bridge_iflist *)ifp->if_bridgeport;
+	bif = bridge_getbif(ifp);
 	if (bif == NULL)
 		return;
 	sc = bif->bridge_sc;
diff --git a/sys/net/bridgestp.c b/sys/net/bridgestp.c
index 2fb0370e55f..3c8081cbdec 100644
--- a/sys/net/bridgestp.c
+++ b/sys/net/bridgestp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bridgestp.c,v 1.68 2019/03/31 13:56:25 mpi Exp $	*/
+/*	$OpenBSD: bridgestp.c,v 1.69 2019/04/28 22:15:57 mpi Exp $	*/
 
 /*
  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
@@ -1616,7 +1616,7 @@ bstp_ifstate(void *arg)
 		return;
 
 	s = splnet();
-	if ((bif = (struct bridge_iflist *)ifp->if_bridgeport) == NULL)
+	if ((bif = bridge_getbif(ifp)) == NULL)
 		goto done;
 	if ((bif->bif_flags & IFBIF_STP) == 0)
 		goto done;
@@ -2092,8 +2092,7 @@ bstp_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			err = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			err = ESRCH;
 			break;
 		}
diff --git a/sys/net/if.c b/sys/net/if.c
index 9285372af00..8b7d7075cdf 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.580 2019/04/22 03:26:16 dlg Exp $	*/
+/*	$OpenBSD: if.c,v 1.581 2019/04/28 22:15:57 mpi Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -695,12 +695,10 @@ if_enqueue(struct ifnet *ifp, struct mbuf *m)
 #endif
 
 #if NBRIDGE > 0
-	if (ifp->if_bridgeport && (m->m_flags & M_PROTO1) == 0) {
+	if (ifp->if_bridgeidx && (m->m_flags & M_PROTO1) == 0) {
 		int error;
 
-		KERNEL_LOCK();
-		error = bridge_output(ifp, m, NULL, NULL);
-		KERNEL_UNLOCK();
+		error = bridge_enqueue(ifp, m);
 		return (error);
 	}
 #endif
@@ -1194,7 +1192,7 @@ if_isconnected(const struct ifnet *ifp0, unsigned int ifidx)
 		connected = 1;
 
 #if NBRIDGE > 0
-	if (SAME_BRIDGE(ifp0->if_bridgeport, ifp->if_bridgeport))
+	if (ifp0->if_bridgeidx == ifp->if_bridgeidx)
 		connected = 1;
 #endif
 #if NCARP > 0
diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index a1ba46f8581..0d9c21af678 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bridge.c,v 1.327 2019/04/15 03:26:55 visa Exp $	*/
+/*	$OpenBSD: if_bridge.c,v 1.328 2019/04/28 22:15:57 mpi Exp $	*/
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -111,14 +111,14 @@ int	bridge_ifremove(struct bridge_iflist *);
 void	bridge_spanremove(struct bridge_iflist *);
 int	bridge_input(struct ifnet *, struct mbuf *, void *);
 void	bridge_process(struct ifnet *, struct mbuf *);
-void	bridgeintr_frame(struct bridge_softc *, struct ifnet *, struct mbuf *);
+void	bridgeintr_frame(struct ifnet *, struct ifnet *, struct mbuf *);
 void	bridge_bifgetstp(struct bridge_softc *, struct bridge_iflist *,
 	    struct ifbreq *);
 void	bridge_broadcast(struct bridge_softc *, struct ifnet *,
     struct ether_header *, struct mbuf *);
 int	bridge_localbroadcast(struct ifnet *, struct ether_header *,
     struct mbuf *);
-void	bridge_span(struct bridge_softc *, struct mbuf *);
+void	bridge_span(struct ifnet *, struct mbuf *);
 void	bridge_stop(struct bridge_softc *);
 void	bridge_init(struct bridge_softc *);
 int	bridge_bifconf(struct bridge_softc *, struct ifbifconf *);
@@ -248,7 +248,7 @@ bridge_delete(struct bridge_softc *sc, struct bridge_iflist *bif)
 	if (bif->bif_flags & IFBIF_STP)
 		bstp_delete(bif->bif_stp);
 
-	bif->ifp->if_bridgeport = NULL;
+	bif->ifp->if_bridgeidx = 0;
 	error = ifpromisc(bif->ifp, 0);
 	hook_disestablish(bif->ifp->if_detachhooks, bif->bif_dhcookie);
 
@@ -290,9 +290,8 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			break;
 		}
 
-		if (ifs->if_bridgeport != NULL) {
-			bif = (struct bridge_iflist *)ifs->if_bridgeport;
-			if (bif->bridge_sc == sc)
+		if (ifs->if_bridgeidx != 0) {
+			if (ifs->if_bridgeidx == ifp->if_index)
 				error = EEXIST;
 			else
 				error = EBUSY;
@@ -331,7 +330,7 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		bif->bif_flags = IFBIF_LEARNING | IFBIF_DISCOVER;
 		SIMPLEQ_INIT(&bif->bif_brlin);
 		SIMPLEQ_INIT(&bif->bif_brlout);
-		ifs->if_bridgeport = (caddr_t)bif;
+		ifs->if_bridgeidx = ifp->if_index;
 		bif->bif_dhcookie = hook_establish(ifs->if_detachhooks, 0,
 		    bridge_ifdetach, bif);
 		if_ih_insert(bif->ifp, bridge_input, NULL);
@@ -345,11 +344,11 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		error = bridge_ifremove(bif);
 		break;
 	case SIOCBRDGIFS:
@@ -367,7 +366,7 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = EINVAL;
 			break;
 		}
-		if (ifs->if_bridgeport != NULL) {
+		if (ifs->if_bridgeidx != 0) {
 			error = EBUSY;
 			break;
 		}
@@ -417,11 +416,11 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		req->ifbr_ifsflags = bif->bif_flags;
 		req->ifbr_portno = bif->ifp->if_index & 0xfff;
 		req->ifbr_protected = bif->bif_protected;
@@ -436,8 +435,7 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
@@ -445,6 +443,7 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = EINVAL;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		if (req->ifbr_ifsflags & IFBIF_STP) {
 			if ((bif->bif_flags & IFBIF_STP) == 0) {
 				/* Enable STP */
@@ -495,11 +494,11 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = ENOENT;
 			break;
 		}
-		bif = (struct bridge_iflist *)ifs->if_bridgeport;
-		if (bif == NULL || bif->bridge_sc != sc) {
+		if (ifs->if_bridgeidx != ifp->if_index) {
 			error = ESRCH;
 			break;
 		}
+		bif = bridge_getbif(ifs);
 		bif->bif_protected = req->ifbr_protected;
 		break;
 	case SIOCBRDGRTS:
@@ -663,6 +662,28 @@ done:
 	return (error);
 }
 
+struct bridge_iflist *
+bridge_getbif(struct ifnet *ifp)
+{
+	struct bridge_iflist *bif;
+	struct bridge_softc *sc;
+	struct ifnet *bifp;
+
+	bifp = if_get(ifp->if_bridgeidx);
+	if (bifp == NULL)
+		return (NULL);
+
+	sc = bifp->if_softc;
+	SLIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
+		if (bif->ifp == ifp)
+			break;
+	}
+
+	if_put(bifp);
+
+	return (bif);
+}
+
 void
 bridge_init(struct bridge_softc *sc)
 {
@@ -702,19 +723,16 @@ bridge_stop(struct bridge_softc *sc)
  * already attached.  We must enqueue or free the mbuf before exiting.
  */
 int
-bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
-    struct rtentry *rt)
+bridge_enqueue(struct ifnet *ifp, struct mbuf *m)
 {
-	struct bridge_iflist *bif;
+	struct ifnet *brifp;
 	struct ether_header *eh;
 	struct ifnet *dst_if = NULL;
-	struct bridge_softc *sc;
+	unsigned int dst_ifidx = 0;
 #if NBPFILTER > 0
 	caddr_t if_bpf;
 #endif
-	int error;
-
-	KERNEL_ASSERT_LOCKED();
+	int error = 0;
 
 	if (m->m_len < sizeof(*eh)) {
 		m = m_pullup(m, sizeof(*eh));
@@ -723,8 +741,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 	}
 
 	/* ifp must be a member interface of the bridge. */
-	bif = (struct bridge_iflist *)ifp->if_bridgeport;
-	if (bif == NULL) {
+	brifp = if_get(ifp->if_bridgeidx);
+	if (brifp == NULL) {
 		m_freem(m);
 		return (EINVAL);
 	}
@@ -734,40 +752,43 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 	 * go ahead and send out that interface.  Otherwise the packet
 	 * is dropped below.
 	 */
-	sc = bif->bridge_sc;
-	if ((sc->sc_if.if_flags & IFF_RUNNING) == 0) {
+	if (!ISSET(brifp->if_flags, IFF_RUNNING)) {
 		/* Loop prevention. */
 		m->m_flags |= M_PROTO1;
 		error = if_enqueue(ifp, m);
+		if_put(brifp);
 		return (error);
 	}
 
 #if NBPFILTER > 0
-	if_bpf = sc->sc_if.if_bpf;
+	if_bpf = brifp->if_bpf;
 	if (if_bpf)
 		bpf_mtap(if_bpf, m, BPF_DIRECTION_OUT);
 #endif
 	ifp->if_opackets++;
 	ifp->if_obytes += m->m_pkthdr.len;
 
-	bridge_span(sc, m);
+	bridge_span(brifp, m);
 
 	eh = mtod(m, struct ether_header *);
 	if (!ETHER_IS_MULTICAST(eh->ether_dhost)) {
 		struct ether_addr *dst;
 
 		dst = (struct ether_addr *)&eh->ether_dhost[0];
-		dst_if = bridge_rtlookup(sc, dst, m);
+		dst_ifidx = bridge_rtlookup(brifp, dst, m);
 	}
 
 	/*
 	 * If the packet is a broadcast or we don't know a better way to
 	 * get there, send to all interfaces.
 	 */
-	if (dst_if == NULL) {
+	if (dst_ifidx == 0) {
+		struct bridge_softc *sc = brifp->if_softc;
+		struct bridge_iflist *bif;
 		struct mbuf *mc;
 		int used = 0;
 
+		KERNEL_LOCK();
 		SLIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
 			dst_if = bif->ifp;
 			if ((dst_if->if_flags & IFF_RUNNING) == 0)
@@ -793,7 +814,7 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 			} else {
 				mc = m_dup_pkt(m, ETHER_ALIGN, M_NOWAIT);
 				if (mc == NULL) {
-					sc->sc_if.if_oerrors++;
+					brifp->if_oerrors++;
 					continue;
 				}
 			}
@@ -802,21 +823,29 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 			    BRL_ACTION_BLOCK)
 				continue;
 
-			error = bridge_ifenqueue(&sc->sc_if, dst_if, mc);
+			error = bridge_ifenqueue(brifp, dst_if, mc);
 			if (error)
 				continue;
 		}
+		KERNEL_UNLOCK();
 		if (!used)
 			m_freem(m);
-		return (0);
+		goto out;
 	}
 
-	if ((dst_if->if_flags & IFF_RUNNING) == 0) {
+	dst_if = if_get(dst_ifidx);
+	if ((dst_if == NULL) || !ISSET(dst_if->if_flags, IFF_RUNNING)) {
 		m_freem(m);
-		return (ENETDOWN);
+		if_put(dst_if);
+		error = ENETDOWN;
+		goto out;
 	}
-	bridge_ifenqueue(&sc->sc_if, dst_if, m);
-	return (0);
+
+	bridge_ifenqueue(brifp, dst_if, m);
+	if_put(dst_if);
+out:
+	if_put(brifp);
+	return (error);
 }
 
 /*
@@ -853,12 +882,14 @@ bridgeintr(void)
  * Process a single frame.  Frame must be freed or queued before returning.
  */
 void
-bridgeintr_frame(struct bridge_softc *sc, struct ifnet *src_if, struct mbuf *m)
+bridgeintr_frame(struct ifnet *brifp, struct ifnet *src_if, struct mbuf *m)
 {
+	struct bridge_softc *sc = brifp->if_softc;
 	struct ifnet *dst_if = NULL;
 	struct bridge_iflist *bif;
 	struct ether_addr *dst, *src;
 	struct ether_header eh;
+	unsigned int dst_ifidx;
 	u_int32_t protected;
 	int len;
 
@@ -866,7 +897,7 @@ bridgeintr_frame(struct bridge_softc *sc, struct ifnet *src_if, struct mbuf *m)
 	sc->sc_if.if_ipackets++;
 	sc->sc_if.if_ibytes += m->m_pkthdr.len;
 
-	bif = (struct bridge_iflist *)src_if->if_bridgeport;
+	bif = bridge_getbif(src_if);
 	KASSERT(bif != NULL);
 
 	if (m->m_pkthdr.len < sizeof(eh)) {
@@ -904,8 +935,8 @@ bridgeintr_frame(struct bridge_softc *sc, struct ifnet *src_if, struct mbuf *m)
 	 * side of the bridge, drop it.
 	 */
 	if (!ETHER_IS_MULTICAST(eh.ether_dhost)) {
-		dst_if = bridge_rtlookup(sc, dst, NULL);
-		if (dst_if == src_if) {
+		dst_ifidx = bridge_rtlookup(brifp, dst, NULL);
+		if (dst_ifidx == src_if->if_index) {
 			m_freem(m);
 			return;
 		}
@@ -963,42 +994,38 @@ bridgeintr_frame(struct bridge_softc *sc, struct ifnet *src_if, struct mbuf *m)
 	 * If the packet is a multicast or broadcast OR if we don't
 	 * know any better, forward it to all interfaces.
 	 */
-	if ((m->m_flags & (M_BCAST | M_MCAST)) || dst_if == NULL) {
+	if ((m->m_flags & (M_BCAST | M_MCAST)) || dst_ifidx == 0) {
 		sc->sc_if.if_imcasts++;
 		bridge_broadcast(sc, src_if, &eh, m);
 		return;
 	}
 	protected = bif->bif_protected;
 
+	dst_if = if_get(dst_ifidx);
+	if (dst_if == NULL)
+		goto bad;
+
 	/*
 	 * At this point, we're dealing with a unicast frame going to a
 	 * different interface
 	 */
-	if ((dst_if->if_flags & IFF_RUNNING) == 0) {
-		m_freem(m);
-		return;
-	}
-	bif = (struct bridge_iflist *)dst_if->if_bridgeport;
+	if (!ISSET(dst_if->if_flags, IFF_RUNNING))
+		goto bad;
+	bif = bridge_getbif(dst_if);
 	if ((bif->bif_flags & IFBIF_STP) &&
-	    (bif->bif_state == BSTP_IFSTATE_DISCARDING)) {
-		m_freem(m);
-		return;
-	}
+	    (bif->bif_state == BSTP_IFSTATE_DISCARDING))
+		goto bad;
 	/*
 	 * Do not transmit if both ports are part of the same protected
 	 * domain.
 	 */
-	if (protected != 0 && (protected & bif->bif_protected)) {
-		m_freem(m);
-		return;
-	}
-	if (bridge_filterrule(&bif->bif_brlout, &eh, m) == BRL_ACTION_BLOCK) {
-		m_freem(m);
-		return;
-	}
+	if (protected != 0 && (protected & bif->bif_protected))
+		goto bad;
+	if (bridge_filterrule(&bif->bif_brlout, &eh, m) == BRL_ACTION_BLOCK)
+		goto bad;
 	m = bridge_ip(&sc->sc_if, BRIDGE_OUT, dst_if, &eh, m);
 	if (m == NULL)
-		return;
+		goto bad;
 
 	len = m->m_pkthdr.len;
 #if NVLAN > 0
@@ -1011,6 +1038,10 @@ bridgeintr_frame(struct bridge_softc *sc, struct ifnet *src_if, struct mbuf *m)
 	else {
 		bridge_ifenqueue(&sc->sc_if, dst_if, m);
 	}
+	m = NULL;
+bad:
+	if_put(dst_if);
+	m_freem(m);
 }
 
 /*
@@ -1054,6 +1085,7 @@ bridge_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 void
 bridge_process(struct ifnet *ifp, struct mbuf *m)
 {
+	struct ifnet *brifp;
 	struct bridge_softc *sc;
 	struct bridge_iflist *bif, *bif0;
 	struct ether_header *eh;
@@ -1064,12 +1096,8 @@ bridge_process(struct ifnet *ifp, struct mbuf *m)
 
 	KERNEL_ASSERT_LOCKED();
 
-	bif = (struct bridge_iflist *)ifp->if_bridgeport;
-	if (bif == NULL)
-		goto reenqueue;
-
-	sc = bif->bridge_sc;
-	if ((sc->sc_if.if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING))
+	brifp = if_get(ifp->if_bridgeidx);
+	if ((brifp == NULL) || !ISSET(brifp->if_flags, IFF_RUNNING))
 		goto reenqueue;
 
 #if NVLAN > 0
@@ -1080,17 +1108,25 @@ bridge_process(struct ifnet *ifp, struct mbuf *m)
 	if (ISSET(m->m_flags, M_VLANTAG)) {
 		m = vlan_inject(m, ETHERTYPE_VLAN, m->m_pkthdr.ether_vtag);
 		if (m == NULL)
-			return;
+			goto bad;
 	}
 #endif
 
 #if NBPFILTER > 0
-	if_bpf = sc->sc_if.if_bpf;
+	if_bpf = brifp->if_bpf;
 	if (if_bpf)
 		bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_IN);
 #endif
 
-	bridge_span(sc, m);
+	sc = brifp->if_softc;
+	SLIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
+		if (bif->ifp == ifp)
+			break;
+	}
+	if (bif == NULL)
+		goto reenqueue;
+
+	bridge_span(brifp, m);
 
 	eh = mtod(m, struct ether_header *);
 	if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
@@ -1105,13 +1141,11 @@ bridge_process(struct ifnet *ifp, struct mbuf *m)
 		    ETHER_ADDR_LEN - 1) == 0) {
 			if (eh->ether_dhost[ETHER_ADDR_LEN - 1] == 0) {
 				/* STP traffic */
-				if ((m = bstp_input(sc->sc_stp, bif->bif_stp,
-				    eh, m)) == NULL)
-					return;
-			} else if (eh->ether_dhost[ETHER_ADDR_LEN - 1] <= 0xf) {
-				m_freem(m);
-				return;
-			}
+				m = bstp_input(sc->sc_stp, bif->bif_stp, eh, m);
+				if (m == NULL)
+					goto bad;
+			} else if (eh->ether_dhost[ETHER_ADDR_LEN - 1] <= 0xf)
+				goto bad;
 		}
 
 		/*
@@ -1127,18 +1161,11 @@ bridge_process(struct ifnet *ifp, struct mbuf *m)
 
 		bridge_ifinput(ifp, mc);
 
-		bridgeintr_frame(sc, ifp, m);
+		bridgeintr_frame(brifp, ifp, m);
 		return;
 	}
 
 	/*
-	 * No need to queue frames for ifs in the discarding state
-	 */
-	if ((bif->bif_flags & IFBIF_STP) &&
-	    (bif->bif_state == BSTP_IFSTATE_DISCARDING))
-		goto reenqueue;
-
-	/*
 	 * Unicast, make sure it's not for us.
 	 */
 	bif0 = bif;
@@ -1152,28 +1179,30 @@ bridge_process(struct ifnet *ifp, struct mbuf *m)
 				    ifp, 0, IFBAF_DYNAMIC, m);
 			if (bridge_filterrule(&bif0->bif_brlin, eh, m) ==
 			    BRL_ACTION_BLOCK) {
-				m_freem(m);
-				return;
+			    	goto bad;
 			}
 
 			/* Count for the bridge */
-			sc->sc_if.if_ipackets++;
-			sc->sc_if.if_ibytes += m->m_pkthdr.len;
+			brifp->if_ipackets++;
+			brifp->if_ibytes += m->m_pkthdr.len;
 
-			bridge_ifinput(bif->ifp, m);
-			return;
-		}
-		if (bridge_ourether(bif->ifp, eh->ether_shost)) {
-			m_freem(m);
-			return;
+			ifp = bif->ifp;
+			goto reenqueue;
 		}
+		if (bridge_ourether(bif->ifp, eh->ether_shost))
+			goto bad;
 	}
 
-	bridgeintr_frame(sc, ifp, m);
+	bridgeintr_frame(brifp, ifp, m);
+	if_put(brifp);
 	return;
 
 reenqueue:
 	bridge_ifinput(ifp, m);
+	m = NULL;
+bad:
+	m_freem(m);
+	if_put(brifp);
 }
 
 /*
@@ -1190,7 +1219,7 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *ifp,
 	int len, used = 0;
 	u_int32_t protected;
 
-	bif = (struct bridge_iflist *)ifp->if_bridgeport;
+	bif = bridge_getbif(ifp);
 	protected = bif->bif_protected;
 
 	SLIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
@@ -1301,13 +1330,18 @@ bridge_localbroadcast(struct ifnet *ifp, struct ether_header *eh,
 }
 
 void
-bridge_span(struct bridge_softc *sc, struct mbuf *m)
+bridge_span(struct ifnet *brifp, struct mbuf *m)
 {
+	struct bridge_softc *sc = brifp->if_softc;
 	struct bridge_iflist *bif;
 	struct ifnet *ifp;
 	struct mbuf *mc;
 	int error;
 
+	if (SLIST_EMPTY(&sc->sc_spanlist))
+		return;
+
+	KERNEL_LOCK();
 	SLIST_FOREACH(bif, &sc->sc_spanlist, bif_next) {
 		ifp = bif->ifp;
 
@@ -1316,14 +1350,15 @@ bridge_span(struct bridge_softc *sc, struct mbuf *m)
 
 		mc = m_copym(m, 0, M_COPYALL, M_DONTWAIT);
 		if (mc == NULL) {
-			sc->sc_if.if_oerrors++;
+			brifp->if_oerrors++;
 			continue;
 		}
 
-		error = bridge_ifenqueue(&sc->sc_if, ifp, mc);
+		error = bridge_ifenqueue(brifp, ifp, mc);
 		if (error)
 			continue;
 	}
+	KERNEL_UNLOCK();
 }
 
 /*
@@ -1936,7 +1971,7 @@ bridge_send_icmp_err(struct ifnet *ifp,
 		goto dropit;
 	bcopy(eh, mtod(m, caddr_t), sizeof(*eh));
 
-	bridge_output(ifp, m, NULL, NULL);
+	bridge_enqueue(ifp, m);
 	m_freem(n);
 	return;
 
diff --git a/sys/net/if_bridge.h b/sys/net/if_bridge.h
index c3a40c8799b..1495f83846f 100644
--- a/sys/net/if_bridge.h
+++ b/sys/net/if_bridge.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bridge.h,v 1.63 2019/03/08 17:48:35 mpi Exp $	*/
+/*	$OpenBSD: if_bridge.h,v 1.64 2019/04/28 22:15:57 mpi Exp $	*/
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -424,10 +424,6 @@ struct bridge_iflist {
 };
 #define bif_state			bif_stp->bp_state
 
-#define SAME_BRIDGE(_bp1, _bp2)						\
-	(_bp1 && _bp2 && ((struct bridge_iflist *)_bp1)->bridge_sc ==	\
-	    ((struct bridge_iflist *)_bp2)->bridge_sc)
-
 /*
  * XXX ip_ipsp.h's sockaddr_union should be converted to sockaddr *
  * passing with correct sa_len, then a good approach for cleaning this
@@ -453,7 +449,7 @@ struct bridge_tunneltag {
  */
 struct bridge_rtnode {
 	LIST_ENTRY(bridge_rtnode)	brt_next;	/* next in list */
-	struct ifnet			*brt_if;	/* destination ifs */
+	unsigned int			brt_ifidx;	/* destination ifs */
 	u_int8_t			brt_flags;	/* address flags */
 	u_int8_t			brt_age;	/* age counter */
 	struct ether_addr		brt_addr;	/* dst addr */
@@ -470,6 +466,7 @@ struct bridge_rtnode {
  *  Locks used to protect struct members in this file:
  *	I	immutable after creation
  *	m	per-softc mutex
+ *	k	kernel lock
  */
 /*
  * Software state for each bridge
@@ -482,8 +479,8 @@ struct bridge_softc {
 	uint64_t			sc_hashkey[2];	/* [I] siphash key */
 	struct timeout			sc_brtimeout;	/* timeout state */
 	struct bstp_state		*sc_stp;	/* stp state */
-	SLIST_HEAD(, bridge_iflist)	sc_iflist;	/* interface list */
-	SLIST_HEAD(, bridge_iflist)	sc_spanlist;	/* span ports */
+	SLIST_HEAD(, bridge_iflist)	sc_iflist;	/* [k] interface list */
+	SLIST_HEAD(, bridge_iflist)	sc_spanlist;	/* [k] span ports */
 	struct mutex			sc_mtx;		/* mutex */
 	LIST_HEAD(, bridge_rtnode)	sc_rts[BRIDGE_RTABLE_SIZE];	/* [m] hash table */
 };
@@ -491,8 +488,7 @@ struct bridge_softc {
 extern const u_int8_t bstp_etheraddr[];
 struct llc;
 
-int	bridge_output(struct ifnet *, struct mbuf *, struct sockaddr *,
-    struct rtentry *);
+int	bridge_enqueue(struct ifnet *, struct mbuf *);
 void	bridge_update(struct ifnet *, struct ether_addr *, int);
 void	bridge_rtdelete(struct bridge_softc *, struct ifnet *, int);
 void	bridge_rtagenode(struct ifnet *, int);
@@ -517,8 +513,8 @@ void	bstp_ifsflags(struct bstp_port *, u_int);
 
 int	bridgectl_ioctl(struct ifnet *, u_long, caddr_t);
 int	bridge_rtupdate(struct bridge_softc *,
-    struct ether_addr *, struct ifnet *ifp, int, u_int8_t, struct mbuf *);
-struct ifnet *bridge_rtlookup(struct bridge_softc *,
+    struct ether_addr *, struct ifnet *, int, u_int8_t, struct mbuf *);
+unsigned int bridge_rtlookup(struct ifnet *,
     struct ether_addr *, struct mbuf *);
 void	bridge_rtflush(struct bridge_softc *, int);
 void	bridge_rtage(void *);
@@ -529,6 +525,7 @@ void	bridge_flushrule(struct bridge_iflist *);
 
 void	bridge_fragment(struct ifnet *, struct ifnet *, struct ether_header *,
     struct mbuf *);
+struct bridge_iflist *bridge_getbif(struct ifnet *);
 
 #endif /* _KERNEL */
 #endif /* _NET_IF_BRIDGE_H_ */
diff --git a/sys/net/if_switch.c b/sys/net/if_switch.c
index 04df95a1843..d61941c47f7 100644
--- a/sys/net/if_switch.c
+++ b/sys/net/if_switch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_switch.c,v 1.25 2018/12/28 14:32:47 bluhm Exp $	*/
+/*	$OpenBSD: if_switch.c,v 1.26 2019/04/28 22:15:57 mpi Exp $	*/
 
 /*
  * Copyright (c) 2016 Kazuya GODA <goda@openbsd.org>
@@ -492,7 +492,7 @@ switch_port_add(struct switch_softc *sc, struct ifbreq *req)
 	if ((ifs = ifunit(req->ifbr_ifsname)) == NULL)
 		return (ENOENT);
 
-	if (ifs->if_bridgeport != NULL)
+	if (ifs->if_bridgeidx != 0)
 		return (EBUSY);
 
 	if (ifs->if_switchport != NULL) {
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index fa61488f71d..bba9bb51ab5 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.98 2019/04/22 03:26:16 dlg Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.99 2019/04/28 22:15:57 mpi Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -131,8 +131,8 @@ struct ifnet {				/* and the entries */
 	void	(*if_rtrequest)(struct ifnet *, int, struct rtentry *);
 	char	if_xname[IFNAMSIZ];	/* [I] external name (name + unit) */
 	int	if_pcount;		/* [k] # of promiscuous listeners */
+	unsigned int if_bridgeidx;	/* [k] used by bridge ports */
 	caddr_t	if_bpf;			/* packet filter structure */
-	caddr_t if_bridgeport;		/* used by bridge ports */
 	caddr_t if_switchport;		/* used by switch ports */
 	caddr_t if_mcast;		/* used by multicast code */
 	caddr_t if_mcast6;		/* used by IPv6 multicast code */
diff --git a/sys/net/if_vxlan.c b/sys/net/if_vxlan.c
index 73535393979..ca42e7f8a21 100644
--- a/sys/net/if_vxlan.c
+++ b/sys/net/if_vxlan.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_vxlan.c,v 1.71 2019/04/23 10:53:45 dlg Exp $	*/
+/*	$OpenBSD: if_vxlan.c,v 1.72 2019/04/28 22:15:58 mpi Exp $	*/
 
 /*
  * Copyright (c) 2013 Reyk Floeter <reyk@openbsd.org>
@@ -711,7 +711,7 @@ vxlan_lookup(struct mbuf *m, struct udphdr *uh, int iphlen,
 
 #if NBRIDGE > 0
 	/* Store the tunnel src/dst IP and vni for the bridge or switch */
-	if ((ifp->if_bridgeport != NULL || ifp->if_switchport != NULL) &&
+	if ((ifp->if_bridgeidx != 0 || ifp->if_switchport != NULL) &&
 	    srcsa->sa_family != AF_UNSPEC &&
 	    ((brtag = bridge_tunneltag(m)) != NULL)) {
 		memcpy(&brtag->brtag_peer.sa, srcsa, srcsa->sa_len);