diff options
| author |   Nicholas Piggin <npiggin@gmail.com> | 2020-07-24 23:14:22 +1000 |
|---|---|---|
| committer |   Michael Ellerman <mpe@ellerman.id.au> | 2020-07-27 00:01:29 +1000 |
| commit | 2f6560e652dfdbdb59df28b45a3458bf36d3c580 (patch) | |
| tree | eb923dd64c883cf1cbd977111baf6a146ad55e23 /arch/powerpc/include/asm/atomic.h | |
| parent | powerpc/pseries: Implement paravirt qspinlocks for SPLPAR (diff) | |
| download | linux-dev-2f6560e652dfdbdb59df28b45a3458bf36d3c580.tar.xz linux-dev-2f6560e652dfdbdb59df28b45a3458bf36d3c580.zip | |
powerpc/qspinlock: Optimised atomic_try_cmpxchg_lock() that adds the lock hint
This brings the behaviour of the uncontended fast path back to roughly
equivalent to simple spinlocks -- a single atomic op with lock hint.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Acked-by: Waiman Long <longman@redhat.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200724131423.1362108-6-npiggin@gmail.com
Diffstat (limited to 'arch/powerpc/include/asm/atomic.h')
| -rw-r--r-- | arch/powerpc/include/asm/atomic.h | 28 |
1 files changed, 28 insertions, 0 deletions
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 498785ffc25f..f6a3d145ffb7 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -193,6 +193,34 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) +/* + * Don't want to override the generic atomic_try_cmpxchg_acquire, because + * we add a lock hint to the lwarx, which may not be wanted for the + * _acquire case (and is not used by the other _acquire variants so it + * would be a surprise). + */ +static __always_inline bool +atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) +{ + int r, o = *old; + + __asm__ __volatile__ ( +"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" +" cmpw 0,%0,%3 \n" +" bne- 2f \n" +" stwcx. %4,0,%2 \n" +" bne- 1b \n" +"\t" PPC_ACQUIRE_BARRIER " \n" +"2: \n" + : "=&r" (r), "+m" (v->counter) + : "r" (&v->counter), "r" (o), "r" (new) + : "cr0", "memory"); + + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} + /** * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t |