diff options
| author |   Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2015-06-29 17:06:39 -0700 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2015-07-17 14:59:00 -0700 |
| commit | 2cd6ffafec066118365f6d7eb7a42ea16c1f032c (patch) | |
| tree | 39656499f5a78c4b61528904e3464c2403a0b83b /kernel/rcu/tree.h | |
| parent | rcu: Consolidate last open-coded expedited memory barrier (diff) | |
| download | linux-dev-2cd6ffafec066118365f6d7eb7a42ea16c1f032c.tar.xz linux-dev-2cd6ffafec066118365f6d7eb7a42ea16c1f032c.zip | |
rcu: Extend expedited funnel locking to rcu_data structure
The strictly rcu_node based funnel-locking scheme works well in many
cases, but systems with CONFIG_RCU_FANOUT_LEAF=64 won't necessarily get
all that much concurrency. This commit therefore extends the funnel
locking into the per-CPU rcu_data structure, providing concurrency equal
to the number of CPUs.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Diffstat (limited to 'kernel/rcu/tree.h')
| -rw-r--r-- | kernel/rcu/tree.h | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 5c1042d9c310..efee84ce1e08 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -364,11 +364,12 @@ struct rcu_data { unsigned long n_rp_nocb_defer_wakeup; unsigned long n_rp_need_nothing; - /* 6) _rcu_barrier() and OOM callbacks. */ + /* 6) _rcu_barrier(), OOM callbacks, and expediting. */ struct rcu_head barrier_head; #ifdef CONFIG_RCU_FAST_NO_HZ struct rcu_head oom_head; #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ + struct mutex exp_funnel_mutex; /* 7) Callback offloading. */ #ifdef CONFIG_RCU_NOCB_CPU @@ -494,6 +495,7 @@ struct rcu_state { atomic_long_t expedited_tryfail; /* # acquisition failures. */ atomic_long_t expedited_workdone1; /* # done by others #1. */ atomic_long_t expedited_workdone2; /* # done by others #2. */ + atomic_long_t expedited_workdone3; /* # done by others #3. */ atomic_long_t expedited_normal; /* # fallbacks to normal. */ atomic_t expedited_need_qs; /* # CPUs left to check in. */ wait_queue_head_t expedited_wq; /* Wait for check-ins. */ |