reintroduce tx mitigation, like dfly does and like we used to do.

this replaces ifq_start with code that waits until at least 4 packets
have been queued on the ifq before calling the drivers start routine.
if less than 4 packets get queued, the start routine is called from
a task in a softnet tq.

4 packets was chosen this time based on testing sephe did in dfly
which showed no real improvement when bundling more packets.  hrvoje
popovski tested this on several nics and found an improvement of
10 to 20 percent when forwarding across the board.

because some of the ifq's work could be sitting on a softnet tq,
ifq_barrier now calls taskq_barrier to guarantee any work that was
pending there has finished.

ok mpi@ visa@

2 files changed, 46 insertions, 8 deletions

diff --git a/sys/net/ifq.c b/sys/net/ifq.c
index bace390ca65..d460d764488 100644
--- a/sys/net/ifq.c
+++ b/sys/net/ifq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.c,v 1.12 2017/06/02 00:07:12 dlg Exp $ */
+/*	$OpenBSD: ifq.c,v 1.13 2017/11/14 00:00:35 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -64,9 +64,16 @@ struct priq {
 void	ifq_start_task(void *);
 void	ifq_restart_task(void *);
 void	ifq_barrier_task(void *);
+void	ifq_bundle_task(void *);
 
 #define TASK_ONQUEUE 0x1
 
+static inline void
+ifq_run_start(struct ifqueue *ifq)
+{
+	ifq_serialize(ifq, &ifq->ifq_start);
+}
+
 void
 ifq_serialize(struct ifqueue *ifq, struct task *t)
 {
@@ -108,6 +115,16 @@ ifq_is_serialized(struct ifqueue *ifq)
 }
 
 void
+ifq_start(struct ifqueue *ifq)
+{
+	if (ifq_len(ifq) >= min(4, ifq->ifq_maxlen)) {
+		task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
+		ifq_run_start(ifq);
+	} else
+		task_add(ifq->ifq_softnet, &ifq->ifq_bundle);
+}
+
+void
 ifq_start_task(void *p)
 {
 	struct ifqueue *ifq = p;
@@ -131,6 +148,14 @@ ifq_restart_task(void *p)
 }
 
 void
+ifq_bundle_task(void *p)
+{
+	struct ifqueue *ifq = p;
+
+	ifq_run_start(ifq);
+}
+
+void
 ifq_barrier(struct ifqueue *ifq)
 {
 	struct sleep_state sls;
@@ -140,6 +165,18 @@ ifq_barrier(struct ifqueue *ifq)
 	/* this should only be called from converted drivers */
 	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
 
+	if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle)) {
+		int netlocked = (rw_status(&netlock) == RW_WRITE);
+
+		if (netlocked) /* XXXSMP breaks atomicity */
+			NET_UNLOCK();
+
+		taskq_barrier(ifq->ifq_softnet);
+
+		if (netlocked)
+			NET_LOCK();
+	}
+
 	if (ifq->ifq_serializer == NULL)
 		return;
 
@@ -168,6 +205,7 @@ void
 ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 {
 	ifq->ifq_if = ifp;
+	ifq->ifq_softnet = net_tq(ifp->if_index);
 	ifq->ifq_softc = NULL;
 
 	mtx_init(&ifq->ifq_mtx, IPL_NET);
@@ -189,6 +227,7 @@ ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
 	TAILQ_INIT(&ifq->ifq_task_list);
 	ifq->ifq_serializer = NULL;
+	task_set(&ifq->ifq_bundle, ifq_bundle_task, ifq);
 
 	task_set(&ifq->ifq_start, ifq_start_task, ifq);
 	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
@@ -240,6 +279,8 @@ ifq_destroy(struct ifqueue *ifq)
 {
 	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 
+	ifq_barrier(ifq); /* ensure nothing is running with the ifq */
+
 	/* don't need to lock because this is the last use of the ifq */
 
 	ifq->ifq_ops->ifqop_purge(ifq, &ml);
diff --git a/sys/net/ifq.h b/sys/net/ifq.h
index 29b89c38b23..b76a6d82b77 100644
--- a/sys/net/ifq.h
+++ b/sys/net/ifq.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.h,v 1.13 2017/05/03 20:55:29 mikeb Exp $ */
+/*	$OpenBSD: ifq.h,v 1.14 2017/11/14 00:00:35 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -25,6 +25,7 @@ struct ifq_ops;
 
 struct ifqueue {
 	struct ifnet		*ifq_if;
+	struct taskq		*ifq_softnet;
 	union {
 		void			*_ifq_softc;
 		/*
@@ -57,6 +58,7 @@ struct ifqueue {
 	struct mutex		 ifq_task_mtx;
 	struct task_list	 ifq_task_list;
 	void			*ifq_serializer;
+	struct task		 ifq_bundle;
 
 	/* work to be serialised */
 	struct task		 ifq_start;
@@ -378,6 +380,7 @@ void		 ifq_init(struct ifqueue *, struct ifnet *, unsigned int);
 void		 ifq_attach(struct ifqueue *, const struct ifq_ops *, void *);
 void		 ifq_destroy(struct ifqueue *);
 int		 ifq_enqueue(struct ifqueue *, struct mbuf *);
+void		 ifq_start(struct ifqueue *);
 struct mbuf	*ifq_deq_begin(struct ifqueue *);
 void		 ifq_deq_commit(struct ifqueue *, struct mbuf *);
 void		 ifq_deq_rollback(struct ifqueue *, struct mbuf *);
@@ -414,12 +417,6 @@ ifq_is_oactive(struct ifqueue *ifq)
 }
 
 static inline void
-ifq_start(struct ifqueue *ifq)
-{
-	ifq_serialize(ifq, &ifq->ifq_start);
-}
-
-static inline void
 ifq_restart(struct ifqueue *ifq)
 {
 	ifq_serialize(ifq, &ifq->ifq_restart);