have another go at tx mitigation

the idea is to call the hardware transmit routine less since in a
lot of cases posting a producer ring update to the chip is (very)
expensive. it's better to do it for several packets instead of each
packet, hence calling this tx mitigation.

this diff defers the call to the transmit routine to a network
taskq, or until a backlog of packets has built up. dragonflybsd
uses 16 as the size of it's backlog, so i'm copying them for now.

i've tried this before, but previous versions caused deadlocks. i
discovered that the deadlocks in the previous version was from
ifq_barrier calling taskq_barrier against the nettq. interfaces
generally hold NET_LOCK while calling ifq_barrier, but the tq might
already be waiting for the lock we hold.

this version just doesnt have ifq_barrier call taskq_barrier. it
instead relies on the IFF_RUNNING flag and normal ifq serialiser
barrier to guarantee the start routine wont be called when an
interface is going down. the taskq_barrier is only used during
interface destruction to make sure the task struct wont get used
in the future, which is already done without the NET_LOCK being
held.

tx mitigation provides a nice performanace bump in some setups. up
to 25% in some cases.

tested by tb@ and hrvoje popovski (who's running this in production).
ok visa@

4 files changed, 46 insertions, 10 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 3289400486e..6534b5e9159 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.575 2019/04/14 06:57:00 dlg Exp $	*/
+/*	$OpenBSD: if.c,v 1.576 2019/04/16 04:04:19 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -615,6 +615,8 @@ if_attach_common(struct ifnet *ifp)
 	ifp->if_snd.ifq_ifqs[0] = &ifp->if_snd;
 	ifp->if_ifqs = ifp->if_snd.ifq_ifqs;
 	ifp->if_nifqs = 1;
+	if (ifp->if_txmit == 0)
+		ifp->if_txmit = IF_TXMIT_DEFAULT;
 
 	ifiq_init(&ifp->if_rcv, ifp, 0);
 
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 4cad0830309..634c6dff9e6 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.95 2019/03/31 13:58:18 mpi Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.96 2019/04/16 04:04:19 dlg Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -176,6 +176,7 @@ struct ifnet {				/* and the entries */
 	struct	ifqueue **if_ifqs;	/* [I] pointer to an array of sndqs */
 	void	(*if_qstart)(struct ifqueue *);
 	unsigned int if_nifqs;		/* [I] number of output queues */
+	unsigned int if_txmit;		/* [c] txmitigation amount */
 
 	struct	ifiqueue if_rcv;	/* rx/input queue */
 	struct	ifiqueue **if_iqs;	/* [I] pointer to the array of iqs */
@@ -303,6 +304,9 @@ do {									\
 #define	IFQ_IS_EMPTY(ifq)		ifq_empty(ifq)
 #define	IFQ_SET_MAXLEN(ifq, len)	ifq_set_maxlen(ifq, len)
 
+#define IF_TXMIT_MIN			1
+#define IF_TXMIT_DEFAULT		16
+
 /* default interface priorities */
 #define IF_WIRED_DEFAULT_PRIORITY	0
 #define IF_WIRELESS_DEFAULT_PRIORITY	4
diff --git a/sys/net/ifq.c b/sys/net/ifq.c
index 7ba629dab04..c35f33f642e 100644
--- a/sys/net/ifq.c
+++ b/sys/net/ifq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.c,v 1.30 2019/03/29 04:21:55 dlg Exp $ */
+/*	$OpenBSD: ifq.c,v 1.31 2019/04/16 04:04:19 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -70,6 +70,13 @@ struct priq {
 void	ifq_start_task(void *);
 void	ifq_restart_task(void *);
 void	ifq_barrier_task(void *);
+void	ifq_bundle_task(void *);
+
+static inline void
+ifq_run_start(struct ifqueue *ifq)
+{
+	ifq_serialize(ifq, &ifq->ifq_start);
+}
 
 void
 ifq_serialize(struct ifqueue *ifq, struct task *t)
@@ -112,6 +119,16 @@ ifq_is_serialized(struct ifqueue *ifq)
 }
 
 void
+ifq_start(struct ifqueue *ifq)
+{
+	if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) {
+		task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
+		ifq_run_start(ifq);
+	} else
+		task_add(ifq->ifq_softnet, &ifq->ifq_bundle);
+}
+
+void
 ifq_start_task(void *p)
 {
 	struct ifqueue *ifq = p;
@@ -135,11 +152,21 @@ ifq_restart_task(void *p)
 }
 
 void
+ifq_bundle_task(void *p)
+{
+	struct ifqueue *ifq = p;
+
+	ifq_run_start(ifq);
+}
+
+void
 ifq_barrier(struct ifqueue *ifq)
 {
 	struct cond c = COND_INITIALIZER();
 	struct task t = TASK_INITIALIZER(ifq_barrier_task, &c);
 
+	task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
+
 	if (ifq->ifq_serializer == NULL)
 		return;
 
@@ -164,6 +191,7 @@ void
 ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 {
 	ifq->ifq_if = ifp;
+	ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */
 	ifq->ifq_softc = NULL;
 
 	mtx_init(&ifq->ifq_mtx, IPL_NET);
@@ -184,6 +212,7 @@ ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
 	TAILQ_INIT(&ifq->ifq_task_list);
 	ifq->ifq_serializer = NULL;
+	task_set(&ifq->ifq_bundle, ifq_bundle_task, ifq);
 
 	task_set(&ifq->ifq_start, ifq_start_task, ifq);
 	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
@@ -235,6 +264,10 @@ ifq_destroy(struct ifqueue *ifq)
 {
 	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 
+	NET_ASSERT_UNLOCKED();
+	if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle))
+		taskq_barrier(ifq->ifq_softnet);
+
 	/* don't need to lock because this is the last use of the ifq */
 
 	ifq->ifq_ops->ifqop_purge(ifq, &ml);
diff --git a/sys/net/ifq.h b/sys/net/ifq.h
index 974617e7f07..cd0d6538709 100644
--- a/sys/net/ifq.h
+++ b/sys/net/ifq.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.h,v 1.25 2019/03/29 04:21:55 dlg Exp $ */
+/*	$OpenBSD: ifq.h,v 1.26 2019/04/16 04:04:19 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -25,6 +25,7 @@ struct ifq_ops;
 
 struct ifqueue {
 	struct ifnet		*ifq_if;
+	struct taskq		*ifq_softnet;
 	union {
 		void			*_ifq_softc;
 		/*
@@ -57,6 +58,7 @@ struct ifqueue {
 	struct mutex		 ifq_task_mtx;
 	struct task_list	 ifq_task_list;
 	void			*ifq_serializer;
+	struct task		 ifq_bundle;
 
 	/* work to be serialised */
 	struct task		 ifq_start;
@@ -397,6 +399,7 @@ void		 ifq_attach(struct ifqueue *, const struct ifq_ops *, void *);
 void		 ifq_destroy(struct ifqueue *);
 void		 ifq_add_data(struct ifqueue *, struct if_data *);
 int		 ifq_enqueue(struct ifqueue *, struct mbuf *);
+void		 ifq_start(struct ifqueue *);
 struct mbuf	*ifq_deq_begin(struct ifqueue *);
 void		 ifq_deq_commit(struct ifqueue *, struct mbuf *);
 void		 ifq_deq_rollback(struct ifqueue *, struct mbuf *);
@@ -439,12 +442,6 @@ ifq_is_oactive(struct ifqueue *ifq)
 }
 
 static inline void
-ifq_start(struct ifqueue *ifq)
-{
-	ifq_serialize(ifq, &ifq->ifq_start);
-}
-
-static inline void
 ifq_restart(struct ifqueue *ifq)
 {
 	ifq_serialize(ifq, &ifq->ifq_restart);