From 279758f8001f0014b15656a4ef130a20852f6df6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 28 May 2019 15:02:31 +0800 Subject: rhashtable: Add rht_ptr_rcu and improve rht_ptr This patch moves common code between rht_ptr and rht_ptr_exclusive into __rht_ptr. It also adds a new helper rht_ptr_rcu exclusively for the RCU case. This way rht_ptr becomes a lock-only construct so we can use the lighter rcu_dereference_protected primitive. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- include/linux/rhashtable.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 9f8bc06d4136..beb9a9da1699 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -352,37 +352,38 @@ static inline void rht_unlock(struct bucket_table *tbl, static inline struct rhash_head __rcu *__rht_ptr( struct rhash_lock_head *const *bkt) { - return (struct rhash_head __rcu *)((unsigned long)*bkt & ~BIT(0)); + return (struct rhash_head __rcu *) + ((unsigned long)*bkt & ~BIT(0) ?: + (unsigned long)RHT_NULLS_MARKER(bkt)); } /* * Where 'bkt' is a bucket and might be locked: - * rht_ptr() dereferences that pointer and clears the lock bit. + * rht_ptr_rcu() dereferences that pointer and clears the lock bit. + * rht_ptr() dereferences in a context where the bucket is locked. * rht_ptr_exclusive() dereferences in a context where exclusive * access is guaranteed, such as when destroying the table. */ +static inline struct rhash_head *rht_ptr_rcu( + struct rhash_lock_head *const *bkt) +{ + struct rhash_head __rcu *p = __rht_ptr(bkt); + + return rcu_dereference(p); +} + static inline struct rhash_head *rht_ptr( struct rhash_lock_head *const *bkt, struct bucket_table *tbl, unsigned int hash) { - struct rhash_head __rcu *p = __rht_ptr(bkt); - - if (!p) - return RHT_NULLS_MARKER(bkt); - - return rht_dereference_bucket_rcu(p, tbl, hash); + return rht_dereference_bucket(__rht_ptr(bkt), tbl, hash); } static inline struct rhash_head *rht_ptr_exclusive( struct rhash_lock_head *const *bkt) { - struct rhash_head __rcu *p = __rht_ptr(bkt); - - if (!p) - return RHT_NULLS_MARKER(bkt); - - return rcu_dereference_protected(p, 1); + return rcu_dereference_protected(__rht_ptr(bkt), 1); } static inline void rht_assign_locked(struct rhash_lock_head **bkt, @@ -509,7 +510,7 @@ static inline void rht_assign_unlock(struct bucket_table *tbl, */ #define rht_for_each_rcu(pos, tbl, hash) \ for (({barrier(); }), \ - pos = rht_ptr(rht_bucket(tbl, hash), tbl, hash); \ + pos = rht_ptr_rcu(rht_bucket(tbl, hash)); \ !rht_is_a_nulls(pos); \ pos = rcu_dereference_raw(pos->next)) @@ -546,8 +547,7 @@ static inline void rht_assign_unlock(struct bucket_table *tbl, */ #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \ rht_for_each_entry_rcu_from(tpos, pos, \ - rht_ptr(rht_bucket(tbl, hash), \ - tbl, hash), \ + rht_ptr_rcu(rht_bucket(tbl, hash)), \ tbl, hash, member) /** @@ -603,7 +603,7 @@ restart: hash = rht_key_hashfn(ht, tbl, key, params); bkt = rht_bucket(tbl, hash); do { - rht_for_each_rcu_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) { + rht_for_each_rcu_from(he, rht_ptr_rcu(bkt), tbl, hash) { if (params.obj_cmpfn ? params.obj_cmpfn(&arg, rht_obj(ht, he)) : rhashtable_compare(&arg, rht_obj(ht, he))) -- cgit v1.2.3-59-g8ed1b