Automatically create a default lo(4) interface per rdomain.

In order to stop abusing lo0 for all rdomains, a new loopback interface
will be created every time a rdomain is created.  The unit number will
be the same as the rdomain, i.e. lo1 will be attached to rdomain 1.

If this loopback interface is already in use it wont be possible to create
the corresponding rdomain.

In order to know which lo(4) interface is attached to a rdomain, its index
is stored in the rtable/rdomain map.

This is a long overdue since the introduction of rtable/rdomain.  It also
fixes a recent regression due to resetting the rdomain of an incoming
packet reported by semarie@, Andreas Bartelt and Nils Frohberg.

ok claudio@

11 files changed, 94 insertions, 46 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 47408b45081..f90fc05373c 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: init_main.c,v 1.262 2016/11/07 00:26:32 guenther Exp $	*/
+/*	$OpenBSD: init_main.c,v 1.263 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: init_main.c,v 1.84.4.1 1996/06/02 09:08:06 mrg Exp $	*/
 
 /*
@@ -389,6 +389,9 @@ main(void *framep)
 	msginit();
 #endif
 
+	/* Create default routing table before attaching lo0. */
+	rtable_init();
+
 	/* Attach pseudo-devices. */
 	for (pdev = pdevinit; pdev->pdev_attach != NULL; pdev++)
 		if (pdev->pdev_count > 0)
@@ -399,8 +402,6 @@ main(void *framep)
 	swcr_init();
 #endif /* CRYPTO */
 
-	rtable_init();
-
 	/*
 	 * Initialize protocols.  Block reception of incoming packets
 	 * until everything is ready.
diff --git a/sys/net/if.c b/sys/net/if.c
index adcd24bdefe..54f2cf7e1df 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.458 2016/11/08 10:47:10 mpi Exp $	*/
+/*	$OpenBSD: if.c,v 1.459 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -258,7 +258,6 @@ struct srp_gc if_ifp_gc = SRP_GC_INITIALIZER(if_ifp_dtor, NULL);
 struct srp_gc if_map_gc = SRP_GC_INITIALIZER(if_map_dtor, NULL);
 
 struct ifnet_head ifnet = TAILQ_HEAD_INITIALIZER(ifnet);
-unsigned int lo0ifidx;
 
 void
 if_idxmap_init(unsigned int limit)
@@ -1350,12 +1349,7 @@ p2p_rtrequest(struct ifnet *ifp, int req, struct rtentry *rt)
 
 		KASSERT(ifa == rt->rt_ifa);
 
-		/*
-		 * XXX Since lo0 is in the default rdomain we should not
-		 * (ab)use it for any route related to an interface of a
-		 * different rdomain.
-		 */
-		lo0ifp = if_get(lo0ifidx);
+		lo0ifp = if_get(rtable_loindex(ifp->if_rdomain));
 		KASSERT(lo0ifp != NULL);
 		TAILQ_FOREACH(lo0ifa, &lo0ifp->if_addrlist, ifa_list) {
 			if (lo0ifa->ifa_addr->sa_family ==
@@ -1438,7 +1432,7 @@ if_up(struct ifnet *ifp)
 
 #ifdef INET6
 	/* Userland expects the kernel to set ::1 on lo0. */
-	if (ifp->if_index == lo0ifidx)
+	if (ifp->if_index == rtable_loindex(0))
 		in6_ifattach(ifp);
 #endif
 
@@ -1605,14 +1599,32 @@ if_setrdomain(struct ifnet *ifp, int rdomain)
 	if (rdomain < 0 || rdomain > RT_TABLEID_MAX)
 		return (EINVAL);
 
-	/* make sure that the routing table exists */
+	/*
+	 * Create the routing table if it does not exist, including its
+	 * loopback interface with unit == rdomain.
+	 */
 	if (!rtable_exists(rdomain)) {
+		struct ifnet *loifp;
+		char loifname[IFNAMSIZ];
+		unsigned int unit = rdomain;
+
+		snprintf(loifname, sizeof(loifname), "lo%u", unit);
+		if ((error = if_clone_create(loifname, 0)))
+			return (error);
+
+		if ((loifp = ifunit(loifname)) == NULL)
+			return (ENXIO);
+
 		s = splsoftnet();
 		if ((error = rtable_add(rdomain)) == 0)
-			rtable_l2set(rdomain, rdomain);
+			rtable_l2set(rdomain, rdomain, loifp->if_index);
 		splx(s);
-		if (error)
+		if (error) {
+			if_clone_destroy(loifname);
 			return (error);
+		}
+
+		loifp->if_rdomain = rdomain;
 	}
 
 	/* make sure that the routing table is a real rdomain */
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index 7d03b3eab81..2f9a5a55ba3 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_loop.c,v 1.76 2016/04/13 11:41:15 mpi Exp $	*/
+/*	$OpenBSD: if_loop.c,v 1.77 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: if_loop.c,v 1.15 1996/05/07 02:40:33 thorpej Exp $	*/
 
 /*
@@ -121,6 +121,7 @@
 #include <net/if_var.h>
 #include <net/if_types.h>
 #include <net/netisr.h>
+#include <net/rtable.h>
 #include <net/route.h>
 
 #include <netinet/in.h>
@@ -182,7 +183,7 @@ loop_clone_create(struct if_clone *ifc, int unit)
 	if (unit == 0) {
 		if_attachhead(ifp);
 		if_addgroup(ifp, ifc->ifc_name);
-		lo0ifidx = ifp->if_index;
+		rtable_l2set(0, 0, ifp->if_index);
 	} else
 		if_attach(ifp);
 	if_alloc_sadl(ifp);
@@ -195,7 +196,7 @@ loop_clone_create(struct if_clone *ifc, int unit)
 int
 loop_clone_destroy(struct ifnet *ifp)
 {
-	if (ifp->if_index == lo0ifidx)
+	if (ifp->if_index == rtable_loindex(ifp->if_rdomain))
 		return (EPERM);
 
 	if_detach(ifp);
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 31915b410e6..83ce441ab01 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.76 2016/11/08 10:46:05 mpi Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.77 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -291,7 +291,6 @@ int		niq_enlist(struct niqueue *, struct mbuf_list *);
     sysctl_mq((_n), (_l), (_op), (_olp), (_np), (_nl), &(_niq)->ni_q)
 
 extern struct ifnet_head ifnet;
-extern unsigned int lo0ifidx;
 extern struct taskq *softnettq;
 
 void	if_start(struct ifnet *);
diff --git a/sys/net/route.c b/sys/net/route.c
index 5ea2ab94e2c..3146867ab3a 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.c,v 1.335 2016/11/09 09:04:48 mpi Exp $	*/
+/*	$OpenBSD: route.c,v 1.336 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: route.c,v 1.14 1996/02/13 22:00:46 christos Exp $	*/
 
 /*
@@ -197,8 +197,6 @@ route_init(void)
 	while (rt_hashjitter == 0)
 		rt_hashjitter = arc4random();
 
-	if (rtable_add(0) != 0)
-		panic("route_init rtable_add");
 #ifdef BFD
 	bfdinit();
 #endif
diff --git a/sys/net/rtable.c b/sys/net/rtable.c
index 2f70de81936..a82ee8452ca 100644
--- a/sys/net/rtable.c
+++ b/sys/net/rtable.c
@@ -1,7 +1,7 @@
-/*	$OpenBSD: rtable.c,v 1.53 2016/11/14 08:54:19 mpi Exp $ */
+/*	$OpenBSD: rtable.c,v 1.54 2016/11/14 10:32:46 mpi Exp $ */
 
 /*
- * Copyright (c) 2014-2015 Martin Pieuchot
+ * Copyright (c) 2014-2016 Martin Pieuchot
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -41,7 +41,7 @@
  *	afmap		    rtmap/dommp
  *   -----------          ---------     -----
  *   |   0     |--------> | 0 | 0 | ... | 0 |	Array mapping rtableid (=index)
- *   -----------          ---------     -----   to rdomain (=value).
+ *   -----------          ---------     -----   to rdomain/loopback (=value).
  *   | AF_INET |.
  *   ----------- `.       .---------.     .---------.
  *       ...	   `----> | rtable0 | ... | rtableN |	Array of pointers for
@@ -59,10 +59,20 @@ struct rtmap {
 	void		 **tbl;
 };
 
-/* Array of rtableid -> rdomain mapping. */
+/*
+ * Array of rtableid -> rdomain mapping.
+ *
+ * Only used for the first index as describbed above.
+ */
 struct dommp {
 	unsigned int	   limit;
-	unsigned int	  *dom;
+	/*
+	 * Array to get the routing domain and loopback interface related to
+	 * a routing table. Format:
+	 *
+	 * 8 unused bits | 16 bits for loopback index | 8 bits for rdomain
+	 */
+	unsigned int	  *value;
 };
 
 unsigned int	   rtmap_limit = 0;
@@ -146,6 +156,8 @@ rtable_init(void)
 	unsigned int	 keylen = 0;
 	int		 i;
 
+	KASSERT(sizeof(struct rtmap) == sizeof(struct dommp));
+
 	/* We use index 0 for the rtable/rdomain map. */
 	af2idx_max = 1;
 	memset(af2idx, 0, sizeof(af2idx));
@@ -173,6 +185,9 @@ rtable_init(void)
 	    M_WAITOK|M_ZERO);
 
 	rtmap_init();
+
+	if (rtable_add(0) != 0)
+		panic("unable to create default routing table");
 }
 
 int
@@ -221,7 +236,7 @@ rtable_add(unsigned int id)
 
 	/* Use main rtable/rdomain by default. */
 	dmm = srp_get_locked(&afmap[0]);
-	dmm->dom[id] = 0;
+	dmm->value[id] = 0;
 
 	return (0);
 }
@@ -272,24 +287,42 @@ rtable_l2(unsigned int rtableid)
 
 	dmm = srp_enter(&sr, &afmap[0]);
 	if (rtableid < dmm->limit)
-		rdomain = dmm->dom[rtableid];
+		rdomain = (dmm->value[rtableid] & RT_TABLEID_MASK);
 	srp_leave(&sr);
 
 	return (rdomain);
 }
 
+unsigned int
+rtable_loindex(unsigned int rtableid)
+{
+	struct dommp	*dmm;
+	unsigned int	 loifidx = 0;
+	struct srp_ref	 sr;
+
+	dmm = srp_enter(&sr, &afmap[0]);
+	if (rtableid < dmm->limit)
+		loifidx = (dmm->value[rtableid] >> RT_TABLEID_BITS);
+	srp_leave(&sr);
+
+	return (loifidx);
+}
+
 void
-rtable_l2set(unsigned int rtableid, unsigned int rdomain)
+rtable_l2set(unsigned int rtableid, unsigned int rdomain, unsigned int loifidx)
 {
 	struct dommp	*dmm;
+	unsigned int	 value;
 
 	KERNEL_ASSERT_LOCKED();
 
 	if (!rtable_exists(rtableid) || !rtable_exists(rdomain))
 		return;
 
+	value = (rdomain & RT_TABLEID_MASK) | (loifidx << RT_TABLEID_BITS);
+
 	dmm = srp_get_locked(&afmap[0]);
-	dmm->dom[rtableid] = rdomain;
+	dmm->value[rtableid] = value;
 }
 
 #ifndef ART
diff --git a/sys/net/rtable.h b/sys/net/rtable.h
index dfd955710c8..0653d348f61 100644
--- a/sys/net/rtable.h
+++ b/sys/net/rtable.h
@@ -1,7 +1,7 @@
-/*	$OpenBSD: rtable.h,v 1.17 2016/11/14 08:54:19 mpi Exp $ */
+/*	$OpenBSD: rtable.h,v 1.18 2016/11/14 10:32:46 mpi Exp $ */
 
 /*
- * Copyright (c) 2014-2015 Martin Pieuchot
+ * Copyright (c) 2014-2016 Martin Pieuchot
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -51,7 +51,8 @@ void		 rtable_init(void);
 int		 rtable_exists(unsigned int);
 int		 rtable_add(unsigned int);
 unsigned int	 rtable_l2(unsigned int);
-void		 rtable_l2set(unsigned int, unsigned int);
+unsigned int	 rtable_loindex(unsigned int);
+void		 rtable_l2set(unsigned int, unsigned int, unsigned int);
 
 struct rtentry	*rtable_lookup(unsigned int, struct sockaddr *,
 		     struct sockaddr *, struct sockaddr *, uint8_t);
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index 7ce0239c95f..86ec40e56c7 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_output.c,v 1.328 2016/11/14 03:51:53 dlg Exp $	*/
+/*	$OpenBSD: ip_output.c,v 1.329 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: ip_output.c,v 1.28 1996/02/13 23:43:07 christos Exp $	*/
 
 /*
@@ -211,7 +211,7 @@ reroute:
 
 		ia = ifatoia(ro->ro_rt->rt_ifa);
 		if (ISSET(ro->ro_rt->rt_flags, RTF_LOCAL))
-			ifp = if_get(lo0ifidx);
+			ifp = if_get(rtable_loindex(m->m_pkthdr.ph_rtableid));
 		else
 			ifp = if_get(ro->ro_rt->rt_ifidx);
 		if (ifp == NULL) {
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index 9ac2555e632..4688d738f8d 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_input.c,v 1.168 2016/08/24 09:41:12 mpi Exp $	*/
+/*	$OpenBSD: ip6_input.c,v 1.169 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $	*/
 
 /*
@@ -210,10 +210,11 @@ ip6_input(struct mbuf *m)
 			ip6stat.ip6s_mext1++;
 	} else {
 		if (m->m_next) {
-			if (m->m_flags & M_LOOP) {
-				ip6stat.ip6s_m2m[lo0ifidx]++;	/*XXX*/
-			} else if (ifp->if_index < nitems(ip6stat.ip6s_m2m))
-				ip6stat.ip6s_m2m[ifp->if_index]++;
+			int ifidx = m->m_pkthdr.ph_ifidx;
+			if (m->m_flags & M_LOOP)
+				ifidx = rtable_loindex(m->m_pkthdr.ph_rtableid);
+			if (ifidx < nitems(ip6stat.ip6s_m2m))
+				ip6stat.ip6s_m2m[ifidx]++;
 			else
 				ip6stat.ip6s_m2m[0]++;
 		} else
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 1383aa93691..f579a2d88b6 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_output.c,v 1.216 2016/09/19 18:09:09 tedu Exp $	*/
+/*	$OpenBSD: ip6_output.c,v 1.217 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$KAME: ip6_output.c,v 1.172 2001/03/25 09:55:56 itojun Exp $	*/
 
 /*
@@ -460,7 +460,7 @@ reroute:
 			goto bad;
 		}
 		if (ISSET(rt->rt_flags, RTF_LOCAL))
-			ifp = if_get(lo0ifidx);
+			ifp = if_get(rtable_loindex(m->m_pkthdr.ph_rtableid));
 		else
 			ifp = if_get(rt->rt_ifidx);
 	} else {
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
index f9feaf415da..54acd64d357 100644
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: socket.h,v 1.92 2016/09/28 18:50:20 jca Exp $	*/
+/*	$OpenBSD: socket.h,v 1.93 2016/11/14 10:32:46 mpi Exp $	*/
 /*	$NetBSD: socket.h,v 1.14 1996/02/09 18:25:36 christos Exp $	*/
 
 /*
@@ -143,7 +143,9 @@ struct	splice {
 /*
  * Maximum number of alternate routing tables
  */
-#define	RT_TABLEID_MAX	255
+#define	RT_TABLEID_MAX		255
+#define	RT_TABLEID_BITS		8
+#define	RT_TABLEID_MASK		0xff
 
 #endif /* __BSD_VISIBLE */