aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/include/net/sch_generic.h
diff options
context:
space:
mode:
authorPaolo Abeni <pabeni@redhat.com>2018-05-15 10:50:31 +0200
committerDavid S. Miller <davem@davemloft.net>2018-05-16 12:26:25 -0400
commit32f7b44d0f5661044fcfa84e9ad402ed9d759107 (patch)
treed2453e21a4187e32c9da74a25bbfee4446e725b5 /include/net/sch_generic.h
parentMerge branch 'bonding-performance-and-reliability' (diff)
downloadwireguard-linux-32f7b44d0f5661044fcfa84e9ad402ed9d759107.tar.xz
wireguard-linux-32f7b44d0f5661044fcfa84e9ad402ed9d759107.zip
sched: manipulate __QDISC_STATE_RUNNING in qdisc_run_* helpers
Currently NOLOCK qdiscs pay a measurable overhead to atomically manipulate the __QDISC_STATE_RUNNING. Such bit is flipped twice per packet in the uncontended scenario with packet rate below the line rate: on packed dequeue and on the next, failing dequeue attempt. This changeset moves the bit manipulation into the qdisc_run_{begin,end} helpers, so that the bit is now flipped only once per packet, with measurable performance improvement in the uncontended scenario. This also allows simplifying the qdisc teardown code path - since qdisc_is_running() is now effective for each qdisc type - and avoid a possible race between qdisc_run() and dev_deactivate_many(), as now the some_qdisc_is_busy() can properly detect NOLOCK qdiscs being busy dequeuing packets. Signed-off-by: Paolo Abeni <pabeni@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'include/net/sch_generic.h')
-rw-r--r--include/net/sch_generic.h10
1 files changed, 9 insertions, 1 deletions
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 5154c8300262..4d2b37226e75 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -113,13 +113,19 @@ static inline void qdisc_refcount_inc(struct Qdisc *qdisc)
static inline bool qdisc_is_running(const struct Qdisc *qdisc)
{
+ if (qdisc->flags & TCQ_F_NOLOCK)
+ return test_bit(__QDISC_STATE_RUNNING, &qdisc->state);
return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
}
static inline bool qdisc_run_begin(struct Qdisc *qdisc)
{
- if (qdisc_is_running(qdisc))
+ if (qdisc->flags & TCQ_F_NOLOCK) {
+ if (test_and_set_bit(__QDISC_STATE_RUNNING, &qdisc->state))
+ return false;
+ } else if (qdisc_is_running(qdisc)) {
return false;
+ }
/* Variant of write_seqcount_begin() telling lockdep a trylock
* was attempted.
*/
@@ -131,6 +137,8 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
static inline void qdisc_run_end(struct Qdisc *qdisc)
{
write_seqcount_end(&qdisc->running);
+ if (qdisc->flags & TCQ_F_NOLOCK)
+ clear_bit(__QDISC_STATE_RUNNING, &qdisc->state);
}
static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)