Move ARP processing back to the KERNEL_LOCK()ed task until the race

triggered by updating a cached, but removed from the table, entry is
properly fixed.

Diff from dlg@, prodding deraadt@

3 files changed, 34 insertions, 5 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index a035c440cdd..198251439c5 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.435 2016/07/12 09:33:13 mpi Exp $	*/
+/*	$OpenBSD: if.c,v 1.436 2016/07/13 16:45:19 mpi Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -833,6 +833,10 @@ if_netisr(void *unused)
 
 		atomic_clearbits_int(&netisr, n);
 
+#if NETHER > 0
+		if (n & (1 << NETISR_ARP))
+			arpintr();
+#endif
 		if (n & (1 << NETISR_IP))
 			ipintr();
 #ifdef INET6
diff --git a/sys/net/netisr.h b/sys/net/netisr.h
index 5c709f5b76d..c295f86afbd 100644
--- a/sys/net/netisr.h
+++ b/sys/net/netisr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: netisr.h,v 1.45 2016/05/03 14:52:39 mpi Exp $	*/
+/*	$OpenBSD: netisr.h,v 1.46 2016/07/13 16:45:19 mpi Exp $	*/
 /*	$NetBSD: netisr.h,v 1.12 1995/08/12 23:59:24 mycroft Exp $	*/
 
 /*
@@ -53,6 +53,7 @@
 #define	NETISR_IP	2		/* same as AF_INET */
 #define	NETISR_TX	3		/* for if_snd processing */
 #define	NETISR_PFSYNC	5		/* for pfsync "immediate" tx */
+#define	NETISR_ARP	18		/* same as AF_LINK */
 #define	NETISR_IPV6	24		/* same as AF_INET6 */
 #define	NETISR_ISDN	26		/* same as AF_E164 */
 #define	NETISR_PPP	28		/* for PPP processing */
@@ -68,6 +69,7 @@
 extern int	netisr;			/* scheduling bits for network */
 extern struct task if_input_task_locked;
 
+void	arpintr(void);
 void	ipintr(void);
 void	ip6intr(void);
 void	pppintr(void);
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index da733631a42..857ceb47c6d 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.c,v 1.218 2016/07/13 08:40:46 mpi Exp $	*/
+/*	$OpenBSD: if_ether.c,v 1.219 2016/07/13 16:45:19 mpi Exp $	*/
 /*	$NetBSD: if_ether.c,v 1.31 1996/05/11 12:59:58 mycroft Exp $	*/
 
 /*
@@ -86,6 +86,8 @@ void in_revarpinput(struct ifnet *, struct mbuf *);
 int arpcache(struct ifnet *, struct ether_arp *, struct rtentry *);
 void arpreply(struct ifnet *, struct mbuf *, struct in_addr *, uint8_t *);
 
+struct niqueue arpinq = NIQUEUE_INITIALIZER(50, NETISR_ARP);
+
 LIST_HEAD(, llinfo_arp) arp_list;
 struct	pool arp_pool;		/* pool for llinfo_arp structures */
 int	arp_maxtries = 5;
@@ -438,7 +440,28 @@ arpinput(struct ifnet *ifp, struct mbuf *m)
 	if (m->m_len < len && (m = m_pullup(m, len)) == NULL)
 		return;
 
-	in_arpinput(ifp, m);
+	niq_enqueue(&arpinq, m);
+}
+
+void
+arpintr(void)
+{
+	struct mbuf_list ml;
+	struct mbuf *m;
+	struct ifnet *ifp;
+
+	niq_delist(&arpinq, &ml);
+
+	while ((m = ml_dequeue(&ml)) != NULL) {
+		ifp = if_get(m->m_pkthdr.ph_ifidx);
+
+		if (ifp != NULL)
+			in_arpinput(ifp, m);
+		else
+			m_freem(m);
+
+		if_put(ifp);
+	}
 }
 
 /*
@@ -788,7 +811,7 @@ in_revarpinput(struct ifnet *ifp, struct mbuf *m)
 	switch (op) {
 	case ARPOP_REQUEST:
 	case ARPOP_REPLY:	/* per RFC */
-		in_arpinput(ifp, m);
+		niq_enqueue(&arpinq, m);
 		return;
 	case ARPOP_REVREPLY:
 		break;