aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64/include/asm/cmpxchg.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64/include/asm/cmpxchg.h')
-rw-r--r--arch/arm64/include/asm/cmpxchg.h192
1 files changed, 74 insertions, 118 deletions
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index d8c25b7b18fb..899e9f1d19e4 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -21,7 +21,9 @@
#include <linux/bug.h>
#include <linux/mmdebug.h>
+#include <asm/atomic.h>
#include <asm/barrier.h>
+#include <asm/lse.h>
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
{
@@ -29,37 +31,73 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
switch (size) {
case 1:
- asm volatile("// __xchg1\n"
+ asm volatile(ARM64_LSE_ATOMIC_INSN(
+ /* LL/SC */
+ " prfm pstl1strm, %2\n"
"1: ldxrb %w0, %2\n"
" stlxrb %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
+ " dmb ish",
+ /* LSE atomics */
+ " nop\n"
+ " nop\n"
+ " swpalb %w3, %w0, %2\n"
+ " nop\n"
+ " nop")
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
: "r" (x)
: "memory");
break;
case 2:
- asm volatile("// __xchg2\n"
+ asm volatile(ARM64_LSE_ATOMIC_INSN(
+ /* LL/SC */
+ " prfm pstl1strm, %2\n"
"1: ldxrh %w0, %2\n"
" stlxrh %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
+ " dmb ish",
+ /* LSE atomics */
+ " nop\n"
+ " nop\n"
+ " swpalh %w3, %w0, %2\n"
+ " nop\n"
+ " nop")
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
: "r" (x)
: "memory");
break;
case 4:
- asm volatile("// __xchg4\n"
+ asm volatile(ARM64_LSE_ATOMIC_INSN(
+ /* LL/SC */
+ " prfm pstl1strm, %2\n"
"1: ldxr %w0, %2\n"
" stlxr %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
+ " dmb ish",
+ /* LSE atomics */
+ " nop\n"
+ " nop\n"
+ " swpal %w3, %w0, %2\n"
+ " nop\n"
+ " nop")
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
: "r" (x)
: "memory");
break;
case 8:
- asm volatile("// __xchg8\n"
+ asm volatile(ARM64_LSE_ATOMIC_INSN(
+ /* LL/SC */
+ " prfm pstl1strm, %2\n"
"1: ldxr %0, %2\n"
" stlxr %w1, %3, %2\n"
" cbnz %w1, 1b\n"
+ " dmb ish",
+ /* LSE atomics */
+ " nop\n"
+ " nop\n"
+ " swpal %3, %0, %2\n"
+ " nop\n"
+ " nop")
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
: "r" (x)
: "memory");
@@ -68,7 +106,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
BUILD_BUG();
}
- smp_mb();
return ret;
}
@@ -83,131 +120,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
unsigned long new, int size)
{
- unsigned long oldval = 0, res;
-
switch (size) {
case 1:
- do {
- asm volatile("// __cmpxchg1\n"
- " ldxrb %w1, %2\n"
- " mov %w0, #0\n"
- " cmp %w1, %w3\n"
- " b.ne 1f\n"
- " stxrb %w0, %w4, %2\n"
- "1:\n"
- : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
- : "Ir" (old), "r" (new)
- : "cc");
- } while (res);
- break;
-
+ return __cmpxchg_case_1(ptr, (u8)old, new);
case 2:
- do {
- asm volatile("// __cmpxchg2\n"
- " ldxrh %w1, %2\n"
- " mov %w0, #0\n"
- " cmp %w1, %w3\n"
- " b.ne 1f\n"
- " stxrh %w0, %w4, %2\n"
- "1:\n"
- : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
- : "Ir" (old), "r" (new)
- : "cc");
- } while (res);
- break;
-
+ return __cmpxchg_case_2(ptr, (u16)old, new);
case 4:
- do {
- asm volatile("// __cmpxchg4\n"
- " ldxr %w1, %2\n"
- " mov %w0, #0\n"
- " cmp %w1, %w3\n"
- " b.ne 1f\n"
- " stxr %w0, %w4, %2\n"
- "1:\n"
- : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
- : "Ir" (old), "r" (new)
- : "cc");
- } while (res);
- break;
-
+ return __cmpxchg_case_4(ptr, old, new);
case 8:
- do {
- asm volatile("// __cmpxchg8\n"
- " ldxr %1, %2\n"
- " mov %w0, #0\n"
- " cmp %1, %3\n"
- " b.ne 1f\n"
- " stxr %w0, %4, %2\n"
- "1:\n"
- : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
- : "Ir" (old), "r" (new)
- : "cc");
- } while (res);
- break;
-
+ return __cmpxchg_case_8(ptr, old, new);
default:
BUILD_BUG();
}
- return oldval;
+ unreachable();
}
-#define system_has_cmpxchg_double() 1
-
-static inline int __cmpxchg_double(volatile void *ptr1, volatile void *ptr2,
- unsigned long old1, unsigned long old2,
- unsigned long new1, unsigned long new2, int size)
+static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
+ unsigned long new, int size)
{
- unsigned long loop, lost;
-
switch (size) {
+ case 1:
+ return __cmpxchg_case_mb_1(ptr, (u8)old, new);
+ case 2:
+ return __cmpxchg_case_mb_2(ptr, (u16)old, new);
+ case 4:
+ return __cmpxchg_case_mb_4(ptr, old, new);
case 8:
- VM_BUG_ON((unsigned long *)ptr2 - (unsigned long *)ptr1 != 1);
- do {
- asm volatile("// __cmpxchg_double8\n"
- " ldxp %0, %1, %2\n"
- " eor %0, %0, %3\n"
- " eor %1, %1, %4\n"
- " orr %1, %0, %1\n"
- " mov %w0, #0\n"
- " cbnz %1, 1f\n"
- " stxp %w0, %5, %6, %2\n"
- "1:\n"
- : "=&r"(loop), "=&r"(lost), "+Q" (*(u64 *)ptr1)
- : "r" (old1), "r"(old2), "r"(new1), "r"(new2));
- } while (loop);
- break;
+ return __cmpxchg_case_mb_8(ptr, old, new);
default:
BUILD_BUG();
}
- return !lost;
-}
-
-static inline int __cmpxchg_double_mb(volatile void *ptr1, volatile void *ptr2,
- unsigned long old1, unsigned long old2,
- unsigned long new1, unsigned long new2, int size)
-{
- int ret;
-
- smp_mb();
- ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size);
- smp_mb();
-
- return ret;
-}
-
-static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
- unsigned long new, int size)
-{
- unsigned long ret;
-
- smp_mb();
- ret = __cmpxchg(ptr, old, new, size);
- smp_mb();
-
- return ret;
+ unreachable();
}
#define cmpxchg(ptr, o, n) \
@@ -228,21 +173,32 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
+#define system_has_cmpxchg_double() 1
+
+#define __cmpxchg_double_check(ptr1, ptr2) \
+({ \
+ if (sizeof(*(ptr1)) != 8) \
+ BUILD_BUG(); \
+ VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
+})
+
#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
({\
int __ret;\
- __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \
- (unsigned long)(o2), (unsigned long)(n1), \
- (unsigned long)(n2), sizeof(*(ptr1)));\
+ __cmpxchg_double_check(ptr1, ptr2); \
+ __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
+ (unsigned long)(n1), (unsigned long)(n2), \
+ ptr1); \
__ret; \
})
#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
({\
int __ret;\
- __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \
- (unsigned long)(o2), (unsigned long)(n1), \
- (unsigned long)(n2), sizeof(*(ptr1)));\
+ __cmpxchg_double_check(ptr1, ptr2); \
+ __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
+ (unsigned long)(n1), (unsigned long)(n2), \
+ ptr1); \
__ret; \
})
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.