diff options
Diffstat (limited to 'include/asm-generic')
| -rw-r--r-- | include/asm-generic/atomic-instrumented.h | 1659 | ||||
| -rw-r--r-- | include/asm-generic/atomic-long.h | 1174 | ||||
| -rw-r--r-- | include/asm-generic/bug.h | 3 | ||||
| -rw-r--r-- | include/asm-generic/io.h | 20 | ||||
| -rw-r--r-- | include/asm-generic/iomap.h | 22 | ||||
| -rw-r--r-- | include/asm-generic/page.h | 2 | ||||
| -rw-r--r-- | include/asm-generic/pgtable.h | 18 | ||||
| -rw-r--r-- | include/asm-generic/shmparam.h | 7 | ||||
| -rw-r--r-- | include/asm-generic/uaccess.h | 1 | ||||
| -rw-r--r-- | include/asm-generic/vmlinux.lds.h | 2 |
10 files changed, 2499 insertions, 409 deletions
diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h index 0d4b1d3dbc1e..e8730c6b9fe2 100644 --- a/include/asm-generic/atomic-instrumented.h +++ b/include/asm-generic/atomic-instrumented.h @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-atomic-instrumented.sh +// DO NOT MODIFY THIS FILE DIRECTLY + /* * This file provides wrappers with KASAN instrumentation for atomic operations. * To use this functionality an arch's atomic.h file needs to define all @@ -9,459 +14,1775 @@ * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid * double instrumentation. */ - -#ifndef _LINUX_ATOMIC_INSTRUMENTED_H -#define _LINUX_ATOMIC_INSTRUMENTED_H +#ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H +#define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H #include <linux/build_bug.h> #include <linux/kasan-checks.h> -static __always_inline int atomic_read(const atomic_t *v) +static inline int +atomic_read(const atomic_t *v) { kasan_check_read(v, sizeof(*v)); return arch_atomic_read(v); } +#define atomic_read atomic_read -static __always_inline s64 atomic64_read(const atomic64_t *v) +#if defined(arch_atomic_read_acquire) +static inline int +atomic_read_acquire(const atomic_t *v) { kasan_check_read(v, sizeof(*v)); - return arch_atomic64_read(v); + return arch_atomic_read_acquire(v); } +#define atomic_read_acquire atomic_read_acquire +#endif -static __always_inline void atomic_set(atomic_t *v, int i) +static inline void +atomic_set(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); arch_atomic_set(v, i); } +#define atomic_set atomic_set -static __always_inline void atomic64_set(atomic64_t *v, s64 i) +#if defined(arch_atomic_set_release) +static inline void +atomic_set_release(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_set(v, i); + arch_atomic_set_release(v, i); } +#define atomic_set_release atomic_set_release +#endif -static __always_inline int atomic_xchg(atomic_t *v, int i) +static inline void +atomic_add(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_xchg(v, i); + arch_atomic_add(i, v); } +#define atomic_add atomic_add -static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) +#if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return) +static inline int +atomic_add_return(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_xchg(v, i); + return arch_atomic_add_return(i, v); } +#define atomic_add_return atomic_add_return +#endif -static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) +#if defined(arch_atomic_add_return_acquire) +static inline int +atomic_add_return_acquire(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_cmpxchg(v, old, new); + return arch_atomic_add_return_acquire(i, v); } +#define atomic_add_return_acquire atomic_add_return_acquire +#endif -static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +#if defined(arch_atomic_add_return_release) +static inline int +atomic_add_return_release(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg(v, old, new); + return arch_atomic_add_return_release(i, v); } +#define atomic_add_return_release atomic_add_return_release +#endif -#ifdef arch_atomic_try_cmpxchg -#define atomic_try_cmpxchg atomic_try_cmpxchg -static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) +#if defined(arch_atomic_add_return_relaxed) +static inline int +atomic_add_return_relaxed(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - kasan_check_read(old, sizeof(*old)); - return arch_atomic_try_cmpxchg(v, old, new); + return arch_atomic_add_return_relaxed(i, v); } +#define atomic_add_return_relaxed atomic_add_return_relaxed #endif -#ifdef arch_atomic64_try_cmpxchg -#define atomic64_try_cmpxchg atomic64_try_cmpxchg -static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +#if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add) +static inline int +atomic_fetch_add(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - kasan_check_read(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg(v, old, new); + return arch_atomic_fetch_add(i, v); } +#define atomic_fetch_add atomic_fetch_add #endif -#ifdef arch_atomic_fetch_add_unless -#define atomic_fetch_add_unless atomic_fetch_add_unless -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +#if defined(arch_atomic_fetch_add_acquire) +static inline int +atomic_fetch_add_acquire(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_add_unless(v, a, u); + return arch_atomic_fetch_add_acquire(i, v); } +#define atomic_fetch_add_acquire atomic_fetch_add_acquire #endif -#ifdef arch_atomic64_fetch_add_unless -#define atomic64_fetch_add_unless atomic64_fetch_add_unless -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +#if defined(arch_atomic_fetch_add_release) +static inline int +atomic_fetch_add_release(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_unless(v, a, u); + return arch_atomic_fetch_add_release(i, v); } +#define atomic_fetch_add_release atomic_fetch_add_release #endif -#ifdef arch_atomic_inc -#define atomic_inc atomic_inc -static __always_inline void atomic_inc(atomic_t *v) +#if defined(arch_atomic_fetch_add_relaxed) +static inline int +atomic_fetch_add_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_add_relaxed(i, v); +} +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#endif + +static inline void +atomic_sub(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic_sub(i, v); +} +#define atomic_sub atomic_sub + +#if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return) +static inline int +atomic_sub_return(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_sub_return(i, v); +} +#define atomic_sub_return atomic_sub_return +#endif + +#if defined(arch_atomic_sub_return_acquire) +static inline int +atomic_sub_return_acquire(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_sub_return_acquire(i, v); +} +#define atomic_sub_return_acquire atomic_sub_return_acquire +#endif + +#if defined(arch_atomic_sub_return_release) +static inline int +atomic_sub_return_release(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_sub_return_release(i, v); +} +#define atomic_sub_return_release atomic_sub_return_release +#endif + +#if defined(arch_atomic_sub_return_relaxed) +static inline int +atomic_sub_return_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_sub_return_relaxed(i, v); +} +#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#endif + +#if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub) +static inline int +atomic_fetch_sub(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_sub(i, v); +} +#define atomic_fetch_sub atomic_fetch_sub +#endif + +#if defined(arch_atomic_fetch_sub_acquire) +static inline int +atomic_fetch_sub_acquire(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_sub_acquire(i, v); +} +#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire +#endif + +#if defined(arch_atomic_fetch_sub_release) +static inline int +atomic_fetch_sub_release(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_sub_release(i, v); +} +#define atomic_fetch_sub_release atomic_fetch_sub_release +#endif + +#if defined(arch_atomic_fetch_sub_relaxed) +static inline int +atomic_fetch_sub_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_sub_relaxed(i, v); +} +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#endif + +#if defined(arch_atomic_inc) +static inline void +atomic_inc(atomic_t *v) { kasan_check_write(v, sizeof(*v)); arch_atomic_inc(v); } +#define atomic_inc atomic_inc #endif -#ifdef arch_atomic64_inc -#define atomic64_inc atomic64_inc -static __always_inline void atomic64_inc(atomic64_t *v) +#if defined(arch_atomic_inc_return) +static inline int +atomic_inc_return(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_inc(v); + return arch_atomic_inc_return(v); } +#define atomic_inc_return atomic_inc_return #endif -#ifdef arch_atomic_dec -#define atomic_dec atomic_dec -static __always_inline void atomic_dec(atomic_t *v) +#if defined(arch_atomic_inc_return_acquire) +static inline int +atomic_inc_return_acquire(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_inc_return_acquire(v); +} +#define atomic_inc_return_acquire atomic_inc_return_acquire +#endif + +#if defined(arch_atomic_inc_return_release) +static inline int +atomic_inc_return_release(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_inc_return_release(v); +} +#define atomic_inc_return_release atomic_inc_return_release +#endif + +#if defined(arch_atomic_inc_return_relaxed) +static inline int +atomic_inc_return_relaxed(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_inc_return_relaxed(v); +} +#define atomic_inc_return_relaxed atomic_inc_return_relaxed +#endif + +#if defined(arch_atomic_fetch_inc) +static inline int +atomic_fetch_inc(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_inc(v); +} +#define atomic_fetch_inc atomic_fetch_inc +#endif + +#if defined(arch_atomic_fetch_inc_acquire) +static inline int +atomic_fetch_inc_acquire(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_inc_acquire(v); +} +#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire +#endif + +#if defined(arch_atomic_fetch_inc_release) +static inline int +atomic_fetch_inc_release(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_inc_release(v); +} +#define atomic_fetch_inc_release atomic_fetch_inc_release +#endif + +#if defined(arch_atomic_fetch_inc_relaxed) +static inline int +atomic_fetch_inc_relaxed(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_inc_relaxed(v); +} +#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed +#endif + +#if defined(arch_atomic_dec) +static inline void +atomic_dec(atomic_t *v) { kasan_check_write(v, sizeof(*v)); arch_atomic_dec(v); } +#define atomic_dec atomic_dec #endif -#ifdef atch_atomic64_dec -#define atomic64_dec -static __always_inline void atomic64_dec(atomic64_t *v) +#if defined(arch_atomic_dec_return) +static inline int +atomic_dec_return(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_dec(v); + return arch_atomic_dec_return(v); } +#define atomic_dec_return atomic_dec_return #endif -static __always_inline void atomic_add(int i, atomic_t *v) +#if defined(arch_atomic_dec_return_acquire) +static inline int +atomic_dec_return_acquire(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic_add(i, v); + return arch_atomic_dec_return_acquire(v); } +#define atomic_dec_return_acquire atomic_dec_return_acquire +#endif -static __always_inline void atomic64_add(s64 i, atomic64_t *v) +#if defined(arch_atomic_dec_return_release) +static inline int +atomic_dec_return_release(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_add(i, v); + return arch_atomic_dec_return_release(v); } +#define atomic_dec_return_release atomic_dec_return_release +#endif -static __always_inline void atomic_sub(int i, atomic_t *v) +#if defined(arch_atomic_dec_return_relaxed) +static inline int +atomic_dec_return_relaxed(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic_sub(i, v); + return arch_atomic_dec_return_relaxed(v); } +#define atomic_dec_return_relaxed atomic_dec_return_relaxed +#endif -static __always_inline void atomic64_sub(s64 i, atomic64_t *v) +#if defined(arch_atomic_fetch_dec) +static inline int +atomic_fetch_dec(atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_sub(i, v); + return arch_atomic_fetch_dec(v); } +#define atomic_fetch_dec atomic_fetch_dec +#endif -static __always_inline void atomic_and(int i, atomic_t *v) +#if defined(arch_atomic_fetch_dec_acquire) +static inline int +atomic_fetch_dec_acquire(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_dec_acquire(v); +} +#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire +#endif + +#if defined(arch_atomic_fetch_dec_release) +static inline int +atomic_fetch_dec_release(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_dec_release(v); +} +#define atomic_fetch_dec_release atomic_fetch_dec_release +#endif + +#if defined(arch_atomic_fetch_dec_relaxed) +static inline int +atomic_fetch_dec_relaxed(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_dec_relaxed(v); +} +#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed +#endif + +static inline void +atomic_and(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); arch_atomic_and(i, v); } +#define atomic_and atomic_and -static __always_inline void atomic64_and(s64 i, atomic64_t *v) +#if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and) +static inline int +atomic_fetch_and(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_and(i, v); + return arch_atomic_fetch_and(i, v); +} +#define atomic_fetch_and atomic_fetch_and +#endif + +#if defined(arch_atomic_fetch_and_acquire) +static inline int +atomic_fetch_and_acquire(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_and_acquire(i, v); +} +#define atomic_fetch_and_acquire atomic_fetch_and_acquire +#endif + +#if defined(arch_atomic_fetch_and_release) +static inline int +atomic_fetch_and_release(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_and_release(i, v); +} +#define atomic_fetch_and_release atomic_fetch_and_release +#endif + +#if defined(arch_atomic_fetch_and_relaxed) +static inline int +atomic_fetch_and_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_and_relaxed(i, v); } +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#endif + +#if defined(arch_atomic_andnot) +static inline void +atomic_andnot(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic_andnot(i, v); +} +#define atomic_andnot atomic_andnot +#endif -static __always_inline void atomic_or(int i, atomic_t *v) +#if defined(arch_atomic_fetch_andnot) +static inline int +atomic_fetch_andnot(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_andnot(i, v); +} +#define atomic_fetch_andnot atomic_fetch_andnot +#endif + +#if defined(arch_atomic_fetch_andnot_acquire) +static inline int +atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_andnot_acquire(i, v); +} +#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire +#endif + +#if defined(arch_atomic_fetch_andnot_release) +static inline int +atomic_fetch_andnot_release(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_andnot_release(i, v); +} +#define atomic_fetch_andnot_release atomic_fetch_andnot_release +#endif + +#if defined(arch_atomic_fetch_andnot_relaxed) +static inline int +atomic_fetch_andnot_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_andnot_relaxed(i, v); +} +#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed +#endif + +static inline void +atomic_or(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); arch_atomic_or(i, v); } +#define atomic_or atomic_or -static __always_inline void atomic64_or(s64 i, atomic64_t *v) +#if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or) +static inline int +atomic_fetch_or(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_or(i, v); + return arch_atomic_fetch_or(i, v); +} +#define atomic_fetch_or atomic_fetch_or +#endif + +#if defined(arch_atomic_fetch_or_acquire) +static inline int +atomic_fetch_or_acquire(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_or_acquire(i, v); +} +#define atomic_fetch_or_acquire atomic_fetch_or_acquire +#endif + +#if defined(arch_atomic_fetch_or_release) +static inline int +atomic_fetch_or_release(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_or_release(i, v); +} +#define atomic_fetch_or_release atomic_fetch_or_release +#endif + +#if defined(arch_atomic_fetch_or_relaxed) +static inline int +atomic_fetch_or_relaxed(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_fetch_or_relaxed(i, v); } +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#endif -static __always_inline void atomic_xor(int i, atomic_t *v) +static inline void +atomic_xor(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); arch_atomic_xor(i, v); } +#define atomic_xor atomic_xor -static __always_inline void atomic64_xor(s64 i, atomic64_t *v) +#if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor) +static inline int +atomic_fetch_xor(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - arch_atomic64_xor(i, v); + return arch_atomic_fetch_xor(i, v); } +#define atomic_fetch_xor atomic_fetch_xor +#endif -#ifdef arch_atomic_inc_return -#define atomic_inc_return atomic_inc_return -static __always_inline int atomic_inc_return(atomic_t *v) +#if defined(arch_atomic_fetch_xor_acquire) +static inline int +atomic_fetch_xor_acquire(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_inc_return(v); + return arch_atomic_fetch_xor_acquire(i, v); } +#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire #endif -#ifdef arch_atomic64_in_return -#define atomic64_inc_return atomic64_inc_return -static __always_inline s64 atomic64_inc_return(atomic64_t *v) +#if defined(arch_atomic_fetch_xor_release) +static inline int +atomic_fetch_xor_release(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_return(v); + return arch_atomic_fetch_xor_release(i, v); } +#define atomic_fetch_xor_release atomic_fetch_xor_release #endif -#ifdef arch_atomic_dec_return -#define atomic_dec_return atomic_dec_return -static __always_inline int atomic_dec_return(atomic_t *v) +#if defined(arch_atomic_fetch_xor_relaxed) +static inline int +atomic_fetch_xor_relaxed(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_dec_return(v); + return arch_atomic_fetch_xor_relaxed(i, v); } +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed #endif -#ifdef arch_atomic64_dec_return -#define atomic64_dec_return atomic64_dec_return -static __always_inline s64 atomic64_dec_return(atomic64_t *v) +#if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg) +static inline int +atomic_xchg(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_return(v); + return arch_atomic_xchg(v, i); } +#define atomic_xchg atomic_xchg #endif -#ifdef arch_atomic64_inc_not_zero -#define atomic64_inc_not_zero atomic64_inc_not_zero -static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) +#if defined(arch_atomic_xchg_acquire) +static inline int +atomic_xchg_acquire(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_not_zero(v); + return arch_atomic_xchg_acquire(v, i); } +#define atomic_xchg_acquire atomic_xchg_acquire #endif -#ifdef arch_atomic64_dec_if_positive -#define atomic64_dec_if_positive atomic64_dec_if_positive -static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) +#if defined(arch_atomic_xchg_release) +static inline int +atomic_xchg_release(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_if_positive(v); + return arch_atomic_xchg_release(v, i); } +#define atomic_xchg_release atomic_xchg_release #endif -#ifdef arch_atomic_dec_and_test -#define atomic_dec_and_test atomic_dec_and_test -static __always_inline bool atomic_dec_and_test(atomic_t *v) +#if defined(arch_atomic_xchg_relaxed) +static inline int +atomic_xchg_relaxed(atomic_t *v, int i) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_dec_and_test(v); + return arch_atomic_xchg_relaxed(v, i); } +#define atomic_xchg_relaxed atomic_xchg_relaxed #endif -#ifdef arch_atomic64_dec_and_test -#define atomic64_dec_and_test atomic64_dec_and_test -static __always_inline bool atomic64_dec_and_test(atomic64_t *v) +#if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) +static inline int +atomic_cmpxchg(atomic_t *v, int old, int new) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_and_test(v); + return arch_atomic_cmpxchg(v, old, new); } +#define atomic_cmpxchg atomic_cmpxchg #endif -#ifdef arch_atomic_inc_and_test -#define atomic_inc_and_test atomic_inc_and_test -static __always_inline bool atomic_inc_and_test(atomic_t *v) +#if defined(arch_atomic_cmpxchg_acquire) +static inline int +atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_cmpxchg_acquire(v, old, new); +} +#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire +#endif + +#if defined(arch_atomic_cmpxchg_release) +static inline int +atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_cmpxchg_release(v, old, new); +} +#define atomic_cmpxchg_release atomic_cmpxchg_release +#endif + +#if defined(arch_atomic_cmpxchg_relaxed) +static inline int +atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_cmpxchg_relaxed(v, old, new); +} +#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed +#endif + +#if defined(arch_atomic_try_cmpxchg) +static inline bool +atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic_try_cmpxchg(v, old, new); +} +#define atomic_try_cmpxchg atomic_try_cmpxchg +#endif + +#if defined(arch_atomic_try_cmpxchg_acquire) +static inline bool +atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic_try_cmpxchg_acquire(v, old, new); +} +#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire +#endif + +#if defined(arch_atomic_try_cmpxchg_release) +static inline bool +atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic_try_cmpxchg_release(v, old, new); +} +#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release +#endif + +#if defined(arch_atomic_try_cmpxchg_relaxed) +static inline bool +atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic_try_cmpxchg_relaxed(v, old, new); +} +#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed +#endif + +#if defined(arch_atomic_sub_and_test) +static inline bool +atomic_sub_and_test(int i, atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_sub_and_test(i, v); +} +#define atomic_sub_and_test atomic_sub_and_test +#endif + +#if defined(arch_atomic_dec_and_test) +static inline bool +atomic_dec_and_test(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_dec_and_test(v); +} +#define atomic_dec_and_test atomic_dec_and_test +#endif + +#if defined(arch_atomic_inc_and_test) +static inline bool +atomic_inc_and_test(atomic_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic_inc_and_test(v); } +#define atomic_inc_and_test atomic_inc_and_test #endif -#ifdef arch_atomic64_inc_and_test -#define atomic64_inc_and_test atomic64_inc_and_test -static __always_inline bool atomic64_inc_and_test(atomic64_t *v) +#if defined(arch_atomic_add_negative) +static inline bool +atomic_add_negative(int i, atomic_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_and_test(v); + return arch_atomic_add_negative(i, v); } +#define atomic_add_negative atomic_add_negative #endif -static __always_inline int atomic_add_return(int i, atomic_t *v) +#if defined(arch_atomic_fetch_add_unless) +static inline int +atomic_fetch_add_unless(atomic_t *v, int a, int u) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_add_return(i, v); + return arch_atomic_fetch_add_unless(v, a, u); +} +#define atomic_fetch_add_unless atomic_fetch_add_unless +#endif + +#if defined(arch_atomic_add_unless) +static inline bool +atomic_add_unless(atomic_t *v, int a, int u) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_add_unless(v, a, u); +} +#define atomic_add_unless atomic_add_unless +#endif + +#if defined(arch_atomic_inc_not_zero) +static inline bool +atomic_inc_not_zero(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_inc_not_zero(v); +} +#define atomic_inc_not_zero atomic_inc_not_zero +#endif + +#if defined(arch_atomic_inc_unless_negative) +static inline bool +atomic_inc_unless_negative(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_inc_unless_negative(v); +} +#define atomic_inc_unless_negative atomic_inc_unless_negative +#endif + +#if defined(arch_atomic_dec_unless_positive) +static inline bool +atomic_dec_unless_positive(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_dec_unless_positive(v); +} +#define atomic_dec_unless_positive atomic_dec_unless_positive +#endif + +#if defined(arch_atomic_dec_if_positive) +static inline int +atomic_dec_if_positive(atomic_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic_dec_if_positive(v); +} +#define atomic_dec_if_positive atomic_dec_if_positive +#endif + +static inline s64 +atomic64_read(const atomic64_t *v) +{ + kasan_check_read(v, sizeof(*v)); + return arch_atomic64_read(v); +} +#define atomic64_read atomic64_read + +#if defined(arch_atomic64_read_acquire) +static inline s64 +atomic64_read_acquire(const atomic64_t *v) +{ + kasan_check_read(v, sizeof(*v)); + return arch_atomic64_read_acquire(v); +} +#define atomic64_read_acquire atomic64_read_acquire +#endif + +static inline void +atomic64_set(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_set(v, i); +} +#define atomic64_set atomic64_set + +#if defined(arch_atomic64_set_release) +static inline void +atomic64_set_release(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_set_release(v, i); } +#define atomic64_set_release atomic64_set_release +#endif -static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) +static inline void +atomic64_add(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_add(i, v); +} +#define atomic64_add atomic64_add + +#if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return) +static inline s64 +atomic64_add_return(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_add_return(i, v); } +#define atomic64_add_return atomic64_add_return +#endif -static __always_inline int atomic_sub_return(int i, atomic_t *v) +#if defined(arch_atomic64_add_return_acquire) +static inline s64 +atomic64_add_return_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_sub_return(i, v); + return arch_atomic64_add_return_acquire(i, v); } +#define atomic64_add_return_acquire atomic64_add_return_acquire +#endif -static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) +#if defined(arch_atomic64_add_return_release) +static inline s64 +atomic64_add_return_release(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic64_sub_return(i, v); + return arch_atomic64_add_return_release(i, v); } +#define atomic64_add_return_release atomic64_add_return_release +#endif -static __always_inline int atomic_fetch_add(int i, atomic_t *v) +#if defined(arch_atomic64_add_return_relaxed) +static inline s64 +atomic64_add_return_relaxed(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_add(i, v); + return arch_atomic64_add_return_relaxed(i, v); } +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#endif -static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) +#if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add) +static inline s64 +atomic64_fetch_add(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_fetch_add(i, v); } +#define atomic64_fetch_add atomic64_fetch_add +#endif -static __always_inline int atomic_fetch_sub(int i, atomic_t *v) +#if defined(arch_atomic64_fetch_add_acquire) +static inline s64 +atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_sub(i, v); + return arch_atomic64_fetch_add_acquire(i, v); } +#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire +#endif + +#if defined(arch_atomic64_fetch_add_release) +static inline s64 +atomic64_fetch_add_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_add_release(i, v); +} +#define atomic64_fetch_add_release atomic64_fetch_add_release +#endif -static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) +#if defined(arch_atomic64_fetch_add_relaxed) +static inline s64 +atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_add_relaxed(i, v); +} +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#endif + +static inline void +atomic64_sub(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_sub(i, v); +} +#define atomic64_sub atomic64_sub + +#if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return) +static inline s64 +atomic64_sub_return(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_sub_return(i, v); +} +#define atomic64_sub_return atomic64_sub_return +#endif + +#if defined(arch_atomic64_sub_return_acquire) +static inline s64 +atomic64_sub_return_acquire(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_sub_return_acquire(i, v); +} +#define atomic64_sub_return_acquire atomic64_sub_return_acquire +#endif + +#if defined(arch_atomic64_sub_return_release) +static inline s64 +atomic64_sub_return_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_sub_return_release(i, v); +} +#define atomic64_sub_return_release atomic64_sub_return_release +#endif + +#if defined(arch_atomic64_sub_return_relaxed) +static inline s64 +atomic64_sub_return_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_sub_return_relaxed(i, v); +} +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#endif + +#if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub) +static inline s64 +atomic64_fetch_sub(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_fetch_sub(i, v); } +#define atomic64_fetch_sub atomic64_fetch_sub +#endif -static __always_inline int atomic_fetch_and(int i, atomic_t *v) +#if defined(arch_atomic64_fetch_sub_acquire) +static inline s64 +atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_and(i, v); + return arch_atomic64_fetch_sub_acquire(i, v); +} +#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire +#endif + +#if defined(arch_atomic64_fetch_sub_release) +static inline s64 +atomic64_fetch_sub_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_sub_release(i, v); } +#define atomic64_fetch_sub_release atomic64_fetch_sub_release +#endif -static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) +#if defined(arch_atomic64_fetch_sub_relaxed) +static inline s64 +atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_sub_relaxed(i, v); +} +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#endif + +#if defined(arch_atomic64_inc) +static inline void +atomic64_inc(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_inc(v); +} +#define atomic64_inc atomic64_inc +#endif + +#if defined(arch_atomic64_inc_return) +static inline s64 +atomic64_inc_return(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_return(v); +} +#define atomic64_inc_return atomic64_inc_return +#endif + +#if defined(arch_atomic64_inc_return_acquire) +static inline s64 +atomic64_inc_return_acquire(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_return_acquire(v); +} +#define atomic64_inc_return_acquire atomic64_inc_return_acquire +#endif + +#if defined(arch_atomic64_inc_return_release) +static inline s64 +atomic64_inc_return_release(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_return_release(v); +} +#define atomic64_inc_return_release atomic64_inc_return_release +#endif + +#if defined(arch_atomic64_inc_return_relaxed) +static inline s64 +atomic64_inc_return_relaxed(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_return_relaxed(v); +} +#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed +#endif + +#if defined(arch_atomic64_fetch_inc) +static inline s64 +atomic64_fetch_inc(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_inc(v); +} +#define atomic64_fetch_inc atomic64_fetch_inc +#endif + +#if defined(arch_atomic64_fetch_inc_acquire) +static inline s64 +atomic64_fetch_inc_acquire(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_inc_acquire(v); +} +#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire +#endif + +#if defined(arch_atomic64_fetch_inc_release) +static inline s64 +atomic64_fetch_inc_release(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_inc_release(v); +} +#define atomic64_fetch_inc_release atomic64_fetch_inc_release +#endif + +#if defined(arch_atomic64_fetch_inc_relaxed) +static inline s64 +atomic64_fetch_inc_relaxed(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_inc_relaxed(v); +} +#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed +#endif + +#if defined(arch_atomic64_dec) +static inline void +atomic64_dec(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_dec(v); +} +#define atomic64_dec atomic64_dec +#endif + +#if defined(arch_atomic64_dec_return) +static inline s64 +atomic64_dec_return(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_return(v); +} +#define atomic64_dec_return atomic64_dec_return +#endif + +#if defined(arch_atomic64_dec_return_acquire) +static inline s64 +atomic64_dec_return_acquire(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_return_acquire(v); +} +#define atomic64_dec_return_acquire atomic64_dec_return_acquire +#endif + +#if defined(arch_atomic64_dec_return_release) +static inline s64 +atomic64_dec_return_release(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_return_release(v); +} +#define atomic64_dec_return_release atomic64_dec_return_release +#endif + +#if defined(arch_atomic64_dec_return_relaxed) +static inline s64 +atomic64_dec_return_relaxed(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_return_relaxed(v); +} +#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed +#endif + +#if defined(arch_atomic64_fetch_dec) +static inline s64 +atomic64_fetch_dec(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_dec(v); +} +#define atomic64_fetch_dec atomic64_fetch_dec +#endif + +#if defined(arch_atomic64_fetch_dec_acquire) +static inline s64 +atomic64_fetch_dec_acquire(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_dec_acquire(v); +} +#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire +#endif + +#if defined(arch_atomic64_fetch_dec_release) +static inline s64 +atomic64_fetch_dec_release(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_dec_release(v); +} +#define atomic64_fetch_dec_release atomic64_fetch_dec_release +#endif + +#if defined(arch_atomic64_fetch_dec_relaxed) +static inline s64 +atomic64_fetch_dec_relaxed(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_dec_relaxed(v); +} +#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed +#endif + +static inline void +atomic64_and(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_and(i, v); +} +#define atomic64_and atomic64_and + +#if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and) +static inline s64 +atomic64_fetch_and(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_fetch_and(i, v); } +#define atomic64_fetch_and atomic64_fetch_and +#endif -static __always_inline int atomic_fetch_or(int i, atomic_t *v) +#if defined(arch_atomic64_fetch_and_acquire) +static inline s64 +atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_or(i, v); + return arch_atomic64_fetch_and_acquire(i, v); +} +#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire +#endif + +#if defined(arch_atomic64_fetch_and_release) +static inline s64 +atomic64_fetch_and_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_and_release(i, v); +} +#define atomic64_fetch_and_release atomic64_fetch_and_release +#endif + +#if defined(arch_atomic64_fetch_and_relaxed) +static inline s64 +atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_and_relaxed(i, v); +} +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#endif + +#if defined(arch_atomic64_andnot) +static inline void +atomic64_andnot(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_andnot(i, v); +} +#define atomic64_andnot atomic64_andnot +#endif + +#if defined(arch_atomic64_fetch_andnot) +static inline s64 +atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_andnot(i, v); +} +#define atomic64_fetch_andnot atomic64_fetch_andnot +#endif + +#if defined(arch_atomic64_fetch_andnot_acquire) +static inline s64 +atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_andnot_acquire(i, v); +} +#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire +#endif + +#if defined(arch_atomic64_fetch_andnot_release) +static inline s64 +atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_andnot_release(i, v); +} +#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release +#endif + +#if defined(arch_atomic64_fetch_andnot_relaxed) +static inline s64 +atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_andnot_relaxed(i, v); +} +#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed +#endif + +static inline void +atomic64_or(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_or(i, v); } +#define atomic64_or atomic64_or -static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) +#if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or) +static inline s64 +atomic64_fetch_or(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_fetch_or(i, v); } +#define atomic64_fetch_or atomic64_fetch_or +#endif -static __always_inline int atomic_fetch_xor(int i, atomic_t *v) +#if defined(arch_atomic64_fetch_or_acquire) +static inline s64 +atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_xor(i, v); + return arch_atomic64_fetch_or_acquire(i, v); +} +#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire +#endif + +#if defined(arch_atomic64_fetch_or_release) +static inline s64 +atomic64_fetch_or_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_or_release(i, v); +} +#define atomic64_fetch_or_release atomic64_fetch_or_release +#endif + +#if defined(arch_atomic64_fetch_or_relaxed) +static inline s64 +atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_or_relaxed(i, v); +} +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#endif + +static inline void +atomic64_xor(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + arch_atomic64_xor(i, v); } +#define atomic64_xor atomic64_xor -static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) +#if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor) +static inline s64 +atomic64_fetch_xor(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_fetch_xor(i, v); } +#define atomic64_fetch_xor atomic64_fetch_xor +#endif -#ifdef arch_atomic_sub_and_test -#define atomic_sub_and_test atomic_sub_and_test -static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) +#if defined(arch_atomic64_fetch_xor_acquire) +static inline s64 +atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_sub_and_test(i, v); + return arch_atomic64_fetch_xor_acquire(i, v); } +#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire #endif -#ifdef arch_atomic64_sub_and_test -#define atomic64_sub_and_test atomic64_sub_and_test -static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) +#if defined(arch_atomic64_fetch_xor_release) +static inline s64 +atomic64_fetch_xor_release(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_xor_release(i, v); +} +#define atomic64_fetch_xor_release atomic64_fetch_xor_release +#endif + +#if defined(arch_atomic64_fetch_xor_relaxed) +static inline s64 +atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_xor_relaxed(i, v); +} +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#endif + +#if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg) +static inline s64 +atomic64_xchg(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_xchg(v, i); +} +#define atomic64_xchg atomic64_xchg +#endif + +#if defined(arch_atomic64_xchg_acquire) +static inline s64 +atomic64_xchg_acquire(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_xchg_acquire(v, i); +} +#define atomic64_xchg_acquire atomic64_xchg_acquire +#endif + +#if defined(arch_atomic64_xchg_release) +static inline s64 +atomic64_xchg_release(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_xchg_release(v, i); +} +#define atomic64_xchg_release atomic64_xchg_release +#endif + +#if defined(arch_atomic64_xchg_relaxed) +static inline s64 +atomic64_xchg_relaxed(atomic64_t *v, s64 i) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_xchg_relaxed(v, i); +} +#define atomic64_xchg_relaxed atomic64_xchg_relaxed +#endif + +#if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg) +static inline s64 +atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_cmpxchg(v, old, new); +} +#define atomic64_cmpxchg atomic64_cmpxchg +#endif + +#if defined(arch_atomic64_cmpxchg_acquire) +static inline s64 +atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_cmpxchg_acquire(v, old, new); +} +#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire +#endif + +#if defined(arch_atomic64_cmpxchg_release) +static inline s64 +atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_cmpxchg_release(v, old, new); +} +#define atomic64_cmpxchg_release atomic64_cmpxchg_release +#endif + +#if defined(arch_atomic64_cmpxchg_relaxed) +static inline s64 +atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_cmpxchg_relaxed(v, old, new); +} +#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed +#endif + +#if defined(arch_atomic64_try_cmpxchg) +static inline bool +atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic64_try_cmpxchg(v, old, new); +} +#define atomic64_try_cmpxchg atomic64_try_cmpxchg +#endif + +#if defined(arch_atomic64_try_cmpxchg_acquire) +static inline bool +atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic64_try_cmpxchg_acquire(v, old, new); +} +#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire +#endif + +#if defined(arch_atomic64_try_cmpxchg_release) +static inline bool +atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic64_try_cmpxchg_release(v, old, new); +} +#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release +#endif + +#if defined(arch_atomic64_try_cmpxchg_relaxed) +static inline bool +atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +{ + kasan_check_write(v, sizeof(*v)); + kasan_check_write(old, sizeof(*old)); + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); +} +#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed +#endif + +#if defined(arch_atomic64_sub_and_test) +static inline bool +atomic64_sub_and_test(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_sub_and_test(i, v); } +#define atomic64_sub_and_test atomic64_sub_and_test #endif -#ifdef arch_atomic_add_negative -#define atomic_add_negative atomic_add_negative -static __always_inline bool atomic_add_negative(int i, atomic_t *v) +#if defined(arch_atomic64_dec_and_test) +static inline bool +atomic64_dec_and_test(atomic64_t *v) { kasan_check_write(v, sizeof(*v)); - return arch_atomic_add_negative(i, v); + return arch_atomic64_dec_and_test(v); } +#define atomic64_dec_and_test atomic64_dec_and_test #endif -#ifdef arch_atomic64_add_negative -#define atomic64_add_negative atomic64_add_negative -static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) +#if defined(arch_atomic64_inc_and_test) +static inline bool +atomic64_inc_and_test(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_and_test(v); +} +#define atomic64_inc_and_test atomic64_inc_and_test +#endif + +#if defined(arch_atomic64_add_negative) +static inline bool +atomic64_add_negative(s64 i, atomic64_t *v) { kasan_check_write(v, sizeof(*v)); return arch_atomic64_add_negative(i, v); } +#define atomic64_add_negative atomic64_add_negative +#endif + +#if defined(arch_atomic64_fetch_add_unless) +static inline s64 +atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_fetch_add_unless(v, a, u); +} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#endif + +#if defined(arch_atomic64_add_unless) +static inline bool +atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_add_unless(v, a, u); +} +#define atomic64_add_unless atomic64_add_unless +#endif + +#if defined(arch_atomic64_inc_not_zero) +static inline bool +atomic64_inc_not_zero(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_not_zero(v); +} +#define atomic64_inc_not_zero atomic64_inc_not_zero +#endif + +#if defined(arch_atomic64_inc_unless_negative) +static inline bool +atomic64_inc_unless_negative(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_inc_unless_negative(v); +} +#define atomic64_inc_unless_negative atomic64_inc_unless_negative +#endif + +#if defined(arch_atomic64_dec_unless_positive) +static inline bool +atomic64_dec_unless_positive(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_unless_positive(v); +} +#define atomic64_dec_unless_positive atomic64_dec_unless_positive +#endif + +#if defined(arch_atomic64_dec_if_positive) +static inline s64 +atomic64_dec_if_positive(atomic64_t *v) +{ + kasan_check_write(v, sizeof(*v)); + return arch_atomic64_dec_if_positive(v); +} +#define atomic64_dec_if_positive atomic64_dec_if_positive +#endif + +#if !defined(arch_xchg_relaxed) || defined(arch_xchg) +#define xchg(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_acquire) +#define xchg_acquire(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_release) +#define xchg_release(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_relaxed) +#define xchg_relaxed(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) +#define cmpxchg(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_acquire) +#define cmpxchg_acquire(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_release) +#define cmpxchg_release(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ +}) #endif -#define xchg(ptr, new) \ +#if defined(arch_cmpxchg_relaxed) +#define cmpxchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) +#endif -#define cmpxchg(ptr, old, new) \ +#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64) +#define cmpxchg64(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, (old), (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) +#endif -#define sync_cmpxchg(ptr, old, new) \ +#if defined(arch_cmpxchg64_acquire) +#define cmpxchg64_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, (old), (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ }) +#endif -#define cmpxchg_local(ptr, old, new) \ +#if defined(arch_cmpxchg64_release) +#define cmpxchg64_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, (old), (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) +#endif -#define cmpxchg64(ptr, old, new) \ +#if defined(arch_cmpxchg64_relaxed) +#define cmpxchg64_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64(__ai_ptr, (old), (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) +#endif -#define cmpxchg64_local(ptr, old, new) \ +#define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_local(__ai_ptr, (old), (new)); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ }) -#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ +#define cmpxchg64_local(ptr, ...) \ ({ \ - typeof(p1) __ai_p1 = (p1); \ - kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \ - arch_cmpxchg_double(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ }) -#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \ -({ \ - typeof(p1) __ai_p1 = (p1); \ - kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \ - arch_cmpxchg_double_local(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \ +#define sync_cmpxchg(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) + +#define cmpxchg_double(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ +}) + + +#define cmpxchg_double_local(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ }) -#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ +#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ +// b29b625d5de9280f680e42c7be859b55b15e5f6a diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h index 87d14476edc2..881c7e27af28 100644 --- a/include/asm-generic/atomic-long.h +++ b/include/asm-generic/atomic-long.h @@ -1,269 +1,1013 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-atomic-long.sh +// DO NOT MODIFY THIS FILE DIRECTLY + #ifndef _ASM_GENERIC_ATOMIC_LONG_H #define _ASM_GENERIC_ATOMIC_LONG_H -/* - * Copyright (C) 2005 Silicon Graphics, Inc. - * Christoph Lameter - * - * Allows to provide arch independent atomic definitions without the need to - * edit all arch specific atomic.h files. - */ #include <asm/types.h> -/* - * Suppport for atomic_long_t - * - * Casts for parameters are avoided for existing atomic functions in order to - * avoid issues with cast-as-lval under gcc 4.x and other limitations that the - * macros of a platform may have. - */ +#ifdef CONFIG_64BIT +typedef atomic64_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) +#define atomic_long_cond_read_acquire atomic64_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed +#else +typedef atomic_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) +#define atomic_long_cond_read_acquire atomic_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed +#endif -#if BITS_PER_LONG == 64 +#ifdef CONFIG_64BIT -typedef atomic64_t atomic_long_t; +static inline long +atomic_long_read(const atomic_long_t *v) +{ + return atomic64_read(v); +} -#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) -#define ATOMIC_LONG_PFX(x) atomic64 ## x -#define ATOMIC_LONG_TYPE s64 +static inline long +atomic_long_read_acquire(const atomic_long_t *v) +{ + return atomic64_read_acquire(v); +} -#else +static inline void +atomic_long_set(atomic_long_t *v, long i) +{ + atomic64_set(v, i); +} -typedef atomic_t atomic_long_t; +static inline void +atomic_long_set_release(atomic_long_t *v, long i) +{ + atomic64_set_release(v, i); +} -#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) -#define ATOMIC_LONG_PFX(x) atomic ## x -#define ATOMIC_LONG_TYPE int +static inline void +atomic_long_add(long i, atomic_long_t *v) +{ + atomic64_add(i, v); +} -#endif +static inline long +atomic_long_add_return(long i, atomic_long_t *v) +{ + return atomic64_add_return(i, v); +} + +static inline long +atomic_long_add_return_acquire(long i, atomic_long_t *v) +{ + return atomic64_add_return_acquire(i, v); +} + +static inline long +atomic_long_add_return_release(long i, atomic_long_t *v) +{ + return atomic64_add_return_release(i, v); +} + +static inline long +atomic_long_add_return_relaxed(long i, atomic_long_t *v) +{ + return atomic64_add_return_relaxed(i, v); +} + +static inline long +atomic_long_fetch_add(long i, atomic_long_t *v) +{ + return atomic64_fetch_add(i, v); +} + +static inline long +atomic_long_fetch_add_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_add_acquire(i, v); +} + +static inline long +atomic_long_fetch_add_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_add_release(i, v); +} + +static inline long +atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_add_relaxed(i, v); +} + +static inline void +atomic_long_sub(long i, atomic_long_t *v) +{ + atomic64_sub(i, v); +} + +static inline long +atomic_long_sub_return(long i, atomic_long_t *v) +{ + return atomic64_sub_return(i, v); +} + +static inline long +atomic_long_sub_return_acquire(long i, atomic_long_t *v) +{ + return atomic64_sub_return_acquire(i, v); +} + +static inline long +atomic_long_sub_return_release(long i, atomic_long_t *v) +{ + return atomic64_sub_return_release(i, v); +} + +static inline long +atomic_long_sub_return_relaxed(long i, atomic_long_t *v) +{ + return atomic64_sub_return_relaxed(i, v); +} + +static inline long +atomic_long_fetch_sub(long i, atomic_long_t *v) +{ + return atomic64_fetch_sub(i, v); +} + +static inline long +atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_sub_acquire(i, v); +} + +static inline long +atomic_long_fetch_sub_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_sub_release(i, v); +} + +static inline long +atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_sub_relaxed(i, v); +} + +static inline void +atomic_long_inc(atomic_long_t *v) +{ + atomic64_inc(v); +} + +static inline long +atomic_long_inc_return(atomic_long_t *v) +{ + return atomic64_inc_return(v); +} + +static inline long +atomic_long_inc_return_acquire(atomic_long_t *v) +{ + return atomic64_inc_return_acquire(v); +} + +static inline long +atomic_long_inc_return_release(atomic_long_t *v) +{ + return atomic64_inc_return_release(v); +} + +static inline long +atomic_long_inc_return_relaxed(atomic_long_t *v) +{ + return atomic64_inc_return_relaxed(v); +} + +static inline long +atomic_long_fetch_inc(atomic_long_t *v) +{ + return atomic64_fetch_inc(v); +} + +static inline long +atomic_long_fetch_inc_acquire(atomic_long_t *v) +{ + return atomic64_fetch_inc_acquire(v); +} + +static inline long +atomic_long_fetch_inc_release(atomic_long_t *v) +{ + return atomic64_fetch_inc_release(v); +} + +static inline long +atomic_long_fetch_inc_relaxed(atomic_long_t *v) +{ + return atomic64_fetch_inc_relaxed(v); +} + +static inline void +atomic_long_dec(atomic_long_t *v) +{ + atomic64_dec(v); +} + +static inline long +atomic_long_dec_return(atomic_long_t *v) +{ + return atomic64_dec_return(v); +} + +static inline long +atomic_long_dec_return_acquire(atomic_long_t *v) +{ + return atomic64_dec_return_acquire(v); +} + +static inline long +atomic_long_dec_return_release(atomic_long_t *v) +{ + return atomic64_dec_return_release(v); +} + +static inline long +atomic_long_dec_return_relaxed(atomic_long_t *v) +{ + return atomic64_dec_return_relaxed(v); +} + +static inline long +atomic_long_fetch_dec(atomic_long_t *v) +{ + return atomic64_fetch_dec(v); +} + +static inline long +atomic_long_fetch_dec_acquire(atomic_long_t *v) +{ + return atomic64_fetch_dec_acquire(v); +} + +static inline long +atomic_long_fetch_dec_release(atomic_long_t *v) +{ + return atomic64_fetch_dec_release(v); +} + +static inline long +atomic_long_fetch_dec_relaxed(atomic_long_t *v) +{ + return atomic64_fetch_dec_relaxed(v); +} + +static inline void +atomic_long_and(long i, atomic_long_t *v) +{ + atomic64_and(i, v); +} + +static inline long +atomic_long_fetch_and(long i, atomic_long_t *v) +{ + return atomic64_fetch_and(i, v); +} + +static inline long +atomic_long_fetch_and_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_and_acquire(i, v); +} + +static inline long +atomic_long_fetch_and_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_and_release(i, v); +} + +static inline long +atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_and_relaxed(i, v); +} + +static inline void +atomic_long_andnot(long i, atomic_long_t *v) +{ + atomic64_andnot(i, v); +} + +static inline long +atomic_long_fetch_andnot(long i, atomic_long_t *v) +{ + return atomic64_fetch_andnot(i, v); +} + +static inline long +atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_andnot_acquire(i, v); +} + +static inline long +atomic_long_fetch_andnot_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_andnot_release(i, v); +} + +static inline long +atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_andnot_relaxed(i, v); +} + +static inline void +atomic_long_or(long i, atomic_long_t *v) +{ + atomic64_or(i, v); +} + +static inline long +atomic_long_fetch_or(long i, atomic_long_t *v) +{ + return atomic64_fetch_or(i, v); +} + +static inline long +atomic_long_fetch_or_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_or_acquire(i, v); +} + +static inline long +atomic_long_fetch_or_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_or_release(i, v); +} + +static inline long +atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_or_relaxed(i, v); +} + +static inline void +atomic_long_xor(long i, atomic_long_t *v) +{ + atomic64_xor(i, v); +} + +static inline long +atomic_long_fetch_xor(long i, atomic_long_t *v) +{ + return atomic64_fetch_xor(i, v); +} + +static inline long +atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) +{ + return atomic64_fetch_xor_acquire(i, v); +} + +static inline long +atomic_long_fetch_xor_release(long i, atomic_long_t *v) +{ + return atomic64_fetch_xor_release(i, v); +} + +static inline long +atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) +{ + return atomic64_fetch_xor_relaxed(i, v); +} + +static inline long +atomic_long_xchg(atomic_long_t *v, long i) +{ + return atomic64_xchg(v, i); +} + +static inline long +atomic_long_xchg_acquire(atomic_long_t *v, long i) +{ + return atomic64_xchg_acquire(v, i); +} + +static inline long +atomic_long_xchg_release(atomic_long_t *v, long i) +{ + return atomic64_xchg_release(v, i); +} + +static inline long +atomic_long_xchg_relaxed(atomic_long_t *v, long i) +{ + return atomic64_xchg_relaxed(v, i); +} + +static inline long +atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +{ + return atomic64_cmpxchg(v, old, new); +} + +static inline long +atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) +{ + return atomic64_cmpxchg_acquire(v, old, new); +} + +static inline long +atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) +{ + return atomic64_cmpxchg_release(v, old, new); +} + +static inline long +atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) +{ + return atomic64_cmpxchg_relaxed(v, old, new); +} + +static inline bool +atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) +{ + return atomic64_try_cmpxchg(v, (s64 *)old, new); +} + +static inline bool +atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) +{ + return atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); +} + +static inline bool +atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) +{ + return atomic64_try_cmpxchg_release(v, (s64 *)old, new); +} + +static inline bool +atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) +{ + return atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); +} + +static inline bool +atomic_long_sub_and_test(long i, atomic_long_t *v) +{ + return atomic64_sub_and_test(i, v); +} + +static inline bool +atomic_long_dec_and_test(atomic_long_t *v) +{ + return atomic64_dec_and_test(v); +} + +static inline bool +atomic_long_inc_and_test(atomic_long_t *v) +{ + return atomic64_inc_and_test(v); +} + +static inline bool +atomic_long_add_negative(long i, atomic_long_t *v) +{ + return atomic64_add_negative(i, v); +} + +static inline long +atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +{ + return atomic64_fetch_add_unless(v, a, u); +} + +static inline bool +atomic_long_add_unless(atomic_long_t *v, long a, long u) +{ + return atomic64_add_unless(v, a, u); +} + +static inline bool +atomic_long_inc_not_zero(atomic_long_t *v) +{ + return atomic64_inc_not_zero(v); +} + +static inline bool +atomic_long_inc_unless_negative(atomic_long_t *v) +{ + return atomic64_inc_unless_negative(v); +} + +static inline bool +atomic_long_dec_unless_positive(atomic_long_t *v) +{ + return atomic64_dec_unless_positive(v); +} + +static inline long +atomic_long_dec_if_positive(atomic_long_t *v) +{ + return atomic64_dec_if_positive(v); +} + +#else /* CONFIG_64BIT */ + +static inline long +atomic_long_read(const atomic_long_t *v) +{ + return atomic_read(v); +} + +static inline long +atomic_long_read_acquire(const atomic_long_t *v) +{ + return atomic_read_acquire(v); +} + +static inline void +atomic_long_set(atomic_long_t *v, long i) +{ + atomic_set(v, i); +} + +static inline void +atomic_long_set_release(atomic_long_t *v, long i) +{ + atomic_set_release(v, i); +} + +static inline void +atomic_long_add(long i, atomic_long_t *v) +{ + atomic_add(i, v); +} + +static inline long +atomic_long_add_return(long i, atomic_long_t *v) +{ + return atomic_add_return(i, v); +} + +static inline long +atomic_long_add_return_acquire(long i, atomic_long_t *v) +{ + return atomic_add_return_acquire(i, v); +} + +static inline long +atomic_long_add_return_release(long i, atomic_long_t *v) +{ + return atomic_add_return_release(i, v); +} + +static inline long +atomic_long_add_return_relaxed(long i, atomic_long_t *v) +{ + return atomic_add_return_relaxed(i, v); +} + +static inline long +atomic_long_fetch_add(long i, atomic_long_t *v) +{ + return atomic_fetch_add(i, v); +} + +static inline long +atomic_long_fetch_add_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_add_acquire(i, v); +} + +static inline long +atomic_long_fetch_add_release(long i, atomic_long_t *v) +{ + return atomic_fetch_add_release(i, v); +} + +static inline long +atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_add_relaxed(i, v); +} + +static inline void +atomic_long_sub(long i, atomic_long_t *v) +{ + atomic_sub(i, v); +} + +static inline long +atomic_long_sub_return(long i, atomic_long_t *v) +{ + return atomic_sub_return(i, v); +} + +static inline long +atomic_long_sub_return_acquire(long i, atomic_long_t *v) +{ + return atomic_sub_return_acquire(i, v); +} + +static inline long +atomic_long_sub_return_release(long i, atomic_long_t *v) +{ + return atomic_sub_return_release(i, v); +} + +static inline long +atomic_long_sub_return_relaxed(long i, atomic_long_t *v) +{ + return atomic_sub_return_relaxed(i, v); +} + +static inline long +atomic_long_fetch_sub(long i, atomic_long_t *v) +{ + return atomic_fetch_sub(i, v); +} + +static inline long +atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_sub_acquire(i, v); +} + +static inline long +atomic_long_fetch_sub_release(long i, atomic_long_t *v) +{ + return atomic_fetch_sub_release(i, v); +} + +static inline long +atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_sub_relaxed(i, v); +} + +static inline void +atomic_long_inc(atomic_long_t *v) +{ + atomic_inc(v); +} + +static inline long +atomic_long_inc_return(atomic_long_t *v) +{ + return atomic_inc_return(v); +} + +static inline long +atomic_long_inc_return_acquire(atomic_long_t *v) +{ + return atomic_inc_return_acquire(v); +} + +static inline long +atomic_long_inc_return_release(atomic_long_t *v) +{ + return atomic_inc_return_release(v); +} + +static inline long +atomic_long_inc_return_relaxed(atomic_long_t *v) +{ + return atomic_inc_return_relaxed(v); +} + +static inline long +atomic_long_fetch_inc(atomic_long_t *v) +{ + return atomic_fetch_inc(v); +} + +static inline long +atomic_long_fetch_inc_acquire(atomic_long_t *v) +{ + return atomic_fetch_inc_acquire(v); +} + +static inline long +atomic_long_fetch_inc_release(atomic_long_t *v) +{ + return atomic_fetch_inc_release(v); +} + +static inline long +atomic_long_fetch_inc_relaxed(atomic_long_t *v) +{ + return atomic_fetch_inc_relaxed(v); +} + +static inline void +atomic_long_dec(atomic_long_t *v) +{ + atomic_dec(v); +} + +static inline long +atomic_long_dec_return(atomic_long_t *v) +{ + return atomic_dec_return(v); +} + +static inline long +atomic_long_dec_return_acquire(atomic_long_t *v) +{ + return atomic_dec_return_acquire(v); +} + +static inline long +atomic_long_dec_return_release(atomic_long_t *v) +{ + return atomic_dec_return_release(v); +} + +static inline long +atomic_long_dec_return_relaxed(atomic_long_t *v) +{ + return atomic_dec_return_relaxed(v); +} + +static inline long +atomic_long_fetch_dec(atomic_long_t *v) +{ + return atomic_fetch_dec(v); +} + +static inline long +atomic_long_fetch_dec_acquire(atomic_long_t *v) +{ + return atomic_fetch_dec_acquire(v); +} + +static inline long +atomic_long_fetch_dec_release(atomic_long_t *v) +{ + return atomic_fetch_dec_release(v); +} + +static inline long +atomic_long_fetch_dec_relaxed(atomic_long_t *v) +{ + return atomic_fetch_dec_relaxed(v); +} + +static inline void +atomic_long_and(long i, atomic_long_t *v) +{ + atomic_and(i, v); +} + +static inline long +atomic_long_fetch_and(long i, atomic_long_t *v) +{ + return atomic_fetch_and(i, v); +} + +static inline long +atomic_long_fetch_and_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_and_acquire(i, v); +} + +static inline long +atomic_long_fetch_and_release(long i, atomic_long_t *v) +{ + return atomic_fetch_and_release(i, v); +} + +static inline long +atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_and_relaxed(i, v); +} + +static inline void +atomic_long_andnot(long i, atomic_long_t *v) +{ + atomic_andnot(i, v); +} + +static inline long +atomic_long_fetch_andnot(long i, atomic_long_t *v) +{ + return atomic_fetch_andnot(i, v); +} + +static inline long +atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_andnot_acquire(i, v); +} + +static inline long +atomic_long_fetch_andnot_release(long i, atomic_long_t *v) +{ + return atomic_fetch_andnot_release(i, v); +} + +static inline long +atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_andnot_relaxed(i, v); +} -#define ATOMIC_LONG_READ_OP(mo) \ -static inline long atomic_long_read##mo(const atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_read##mo)(v); \ -} -ATOMIC_LONG_READ_OP() -ATOMIC_LONG_READ_OP(_acquire) - -#undef ATOMIC_LONG_READ_OP - -#define ATOMIC_LONG_SET_OP(mo) \ -static inline void atomic_long_set##mo(atomic_long_t *l, long i) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - ATOMIC_LONG_PFX(_set##mo)(v, i); \ -} -ATOMIC_LONG_SET_OP() -ATOMIC_LONG_SET_OP(_release) - -#undef ATOMIC_LONG_SET_OP - -#define ATOMIC_LONG_ADD_SUB_OP(op, mo) \ -static inline long \ -atomic_long_##op##_return##mo(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(i, v); \ -} -ATOMIC_LONG_ADD_SUB_OP(add,) -ATOMIC_LONG_ADD_SUB_OP(add, _relaxed) -ATOMIC_LONG_ADD_SUB_OP(add, _acquire) -ATOMIC_LONG_ADD_SUB_OP(add, _release) -ATOMIC_LONG_ADD_SUB_OP(sub,) -ATOMIC_LONG_ADD_SUB_OP(sub, _relaxed) -ATOMIC_LONG_ADD_SUB_OP(sub, _acquire) -ATOMIC_LONG_ADD_SUB_OP(sub, _release) - -#undef ATOMIC_LONG_ADD_SUB_OP - -#define atomic_long_cmpxchg_relaxed(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg_acquire(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg_release(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), (old), (new))) - - -#define atomic_long_try_cmpxchg_relaxed(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg_acquire(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg_release(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) - - -#define atomic_long_xchg_relaxed(v, new) \ - (ATOMIC_LONG_PFX(_xchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg_acquire(v, new) \ - (ATOMIC_LONG_PFX(_xchg_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg_release(v, new) \ - (ATOMIC_LONG_PFX(_xchg_release)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg(v, new) \ - (ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new))) - -static __always_inline void atomic_long_inc(atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - ATOMIC_LONG_PFX(_inc)(v); -} - -static __always_inline void atomic_long_dec(atomic_long_t *l) +static inline void +atomic_long_or(long i, atomic_long_t *v) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + atomic_or(i, v); +} - ATOMIC_LONG_PFX(_dec)(v); +static inline long +atomic_long_fetch_or(long i, atomic_long_t *v) +{ + return atomic_fetch_or(i, v); } -#define ATOMIC_LONG_FETCH_OP(op, mo) \ -static inline long \ -atomic_long_fetch_##op##mo(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(i, v); \ +static inline long +atomic_long_fetch_or_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_or_acquire(i, v); } -ATOMIC_LONG_FETCH_OP(add, ) -ATOMIC_LONG_FETCH_OP(add, _relaxed) -ATOMIC_LONG_FETCH_OP(add, _acquire) -ATOMIC_LONG_FETCH_OP(add, _release) -ATOMIC_LONG_FETCH_OP(sub, ) -ATOMIC_LONG_FETCH_OP(sub, _relaxed) -ATOMIC_LONG_FETCH_OP(sub, _acquire) -ATOMIC_LONG_FETCH_OP(sub, _release) -ATOMIC_LONG_FETCH_OP(and, ) -ATOMIC_LONG_FETCH_OP(and, _relaxed) -ATOMIC_LONG_FETCH_OP(and, _acquire) -ATOMIC_LONG_FETCH_OP(and, _release) -ATOMIC_LONG_FETCH_OP(andnot, ) -ATOMIC_LONG_FETCH_OP(andnot, _relaxed) -ATOMIC_LONG_FETCH_OP(andnot, _acquire) -ATOMIC_LONG_FETCH_OP(andnot, _release) -ATOMIC_LONG_FETCH_OP(or, ) -ATOMIC_LONG_FETCH_OP(or, _relaxed) -ATOMIC_LONG_FETCH_OP(or, _acquire) -ATOMIC_LONG_FETCH_OP(or, _release) -ATOMIC_LONG_FETCH_OP(xor, ) -ATOMIC_LONG_FETCH_OP(xor, _relaxed) -ATOMIC_LONG_FETCH_OP(xor, _acquire) -ATOMIC_LONG_FETCH_OP(xor, _release) +static inline long +atomic_long_fetch_or_release(long i, atomic_long_t *v) +{ + return atomic_fetch_or_release(i, v); +} -#undef ATOMIC_LONG_FETCH_OP +static inline long +atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_or_relaxed(i, v); +} -#define ATOMIC_LONG_FETCH_INC_DEC_OP(op, mo) \ -static inline long \ -atomic_long_fetch_##op##mo(atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(v); \ +static inline void +atomic_long_xor(long i, atomic_long_t *v) +{ + atomic_xor(i, v); } -ATOMIC_LONG_FETCH_INC_DEC_OP(inc,) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _relaxed) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _acquire) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _release) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec,) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _relaxed) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _acquire) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _release) +static inline long +atomic_long_fetch_xor(long i, atomic_long_t *v) +{ + return atomic_fetch_xor(i, v); +} -#undef ATOMIC_LONG_FETCH_INC_DEC_OP +static inline long +atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) +{ + return atomic_fetch_xor_acquire(i, v); +} -#define ATOMIC_LONG_OP(op) \ -static __always_inline void \ -atomic_long_##op(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - ATOMIC_LONG_PFX(_##op)(i, v); \ +static inline long +atomic_long_fetch_xor_release(long i, atomic_long_t *v) +{ + return atomic_fetch_xor_release(i, v); } -ATOMIC_LONG_OP(add) -ATOMIC_LONG_OP(sub) -ATOMIC_LONG_OP(and) -ATOMIC_LONG_OP(andnot) -ATOMIC_LONG_OP(or) -ATOMIC_LONG_OP(xor) +static inline long +atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) +{ + return atomic_fetch_xor_relaxed(i, v); +} -#undef ATOMIC_LONG_OP +static inline long +atomic_long_xchg(atomic_long_t *v, long i) +{ + return atomic_xchg(v, i); +} -static inline int atomic_long_sub_and_test(long i, atomic_long_t *l) +static inline long +atomic_long_xchg_acquire(atomic_long_t *v, long i) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + return atomic_xchg_acquire(v, i); +} - return ATOMIC_LONG_PFX(_sub_and_test)(i, v); +static inline long +atomic_long_xchg_release(atomic_long_t *v, long i) +{ + return atomic_xchg_release(v, i); } -static inline int atomic_long_dec_and_test(atomic_long_t *l) +static inline long +atomic_long_xchg_relaxed(atomic_long_t *v, long i) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + return atomic_xchg_relaxed(v, i); +} - return ATOMIC_LONG_PFX(_dec_and_test)(v); +static inline long +atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +{ + return atomic_cmpxchg(v, old, new); } -static inline int atomic_long_inc_and_test(atomic_long_t *l) +static inline long +atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + return atomic_cmpxchg_acquire(v, old, new); +} - return ATOMIC_LONG_PFX(_inc_and_test)(v); +static inline long +atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) +{ + return atomic_cmpxchg_release(v, old, new); } -static inline int atomic_long_add_negative(long i, atomic_long_t *l) +static inline long +atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + return atomic_cmpxchg_relaxed(v, old, new); +} - return ATOMIC_LONG_PFX(_add_negative)(i, v); +static inline bool +atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) +{ + return atomic_try_cmpxchg(v, (int *)old, new); } -#define ATOMIC_LONG_INC_DEC_OP(op, mo) \ -static inline long \ -atomic_long_##op##_return##mo(atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(v); \ +static inline bool +atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) +{ + return atomic_try_cmpxchg_acquire(v, (int *)old, new); } -ATOMIC_LONG_INC_DEC_OP(inc,) -ATOMIC_LONG_INC_DEC_OP(inc, _relaxed) -ATOMIC_LONG_INC_DEC_OP(inc, _acquire) -ATOMIC_LONG_INC_DEC_OP(inc, _release) -ATOMIC_LONG_INC_DEC_OP(dec,) -ATOMIC_LONG_INC_DEC_OP(dec, _relaxed) -ATOMIC_LONG_INC_DEC_OP(dec, _acquire) -ATOMIC_LONG_INC_DEC_OP(dec, _release) -#undef ATOMIC_LONG_INC_DEC_OP +static inline bool +atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) +{ + return atomic_try_cmpxchg_release(v, (int *)old, new); +} + +static inline bool +atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) +{ + return atomic_try_cmpxchg_relaxed(v, (int *)old, new); +} + +static inline bool +atomic_long_sub_and_test(long i, atomic_long_t *v) +{ + return atomic_sub_and_test(i, v); +} + +static inline bool +atomic_long_dec_and_test(atomic_long_t *v) +{ + return atomic_dec_and_test(v); +} + +static inline bool +atomic_long_inc_and_test(atomic_long_t *v) +{ + return atomic_inc_and_test(v); +} + +static inline bool +atomic_long_add_negative(long i, atomic_long_t *v) +{ + return atomic_add_negative(i, v); +} + +static inline long +atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +{ + return atomic_fetch_add_unless(v, a, u); +} + +static inline bool +atomic_long_add_unless(atomic_long_t *v, long a, long u) +{ + return atomic_add_unless(v, a, u); +} -static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) +static inline bool +atomic_long_inc_not_zero(atomic_long_t *v) { - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; + return atomic_inc_not_zero(v); +} - return (long)ATOMIC_LONG_PFX(_add_unless)(v, a, u); +static inline bool +atomic_long_inc_unless_negative(atomic_long_t *v) +{ + return atomic_inc_unless_negative(v); } -#define atomic_long_inc_not_zero(l) \ - ATOMIC_LONG_PFX(_inc_not_zero)((ATOMIC_LONG_PFX(_t) *)(l)) +static inline bool +atomic_long_dec_unless_positive(atomic_long_t *v) +{ + return atomic_dec_unless_positive(v); +} -#define atomic_long_cond_read_relaxed(v, c) \ - ATOMIC_LONG_PFX(_cond_read_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (c)) -#define atomic_long_cond_read_acquire(v, c) \ - ATOMIC_LONG_PFX(_cond_read_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (c)) +static inline long +atomic_long_dec_if_positive(atomic_long_t *v) +{ + return atomic_dec_if_positive(v); +} -#endif /* _ASM_GENERIC_ATOMIC_LONG_H */ +#endif /* CONFIG_64BIT */ +#endif /* _ASM_GENERIC_ATOMIC_LONG_H */ +// 77558968132ce4f911ad53f6f52ce423006f6268 diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 20561a60db9c..0e9bd9c83870 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -211,9 +211,6 @@ void __warn(const char *file, int line, void *caller, unsigned taint, /* * WARN_ON_SMP() is for cases that the warning is either * meaningless for !SMP or may even cause failures. - * This is usually used for cases that we have - * WARN_ON(!spin_is_locked(&lock)) checks, as spin_is_locked() - * returns 0 for uniprocessor settings. * It can also be used with values that are only defined * on SMP: * diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index d356f802945a..303871651f8a 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -32,9 +32,9 @@ /* prevent prefetching of coherent DMA data ahead of a dma-complete */ #ifndef __io_ar #ifdef rmb -#define __io_ar() rmb() +#define __io_ar(v) rmb() #else -#define __io_ar() barrier() +#define __io_ar(v) barrier() #endif #endif @@ -65,7 +65,7 @@ #endif #ifndef __io_par -#define __io_par() __io_ar() +#define __io_par(v) __io_ar(v) #endif @@ -158,7 +158,7 @@ static inline u8 readb(const volatile void __iomem *addr) __io_br(); val = __raw_readb(addr); - __io_ar(); + __io_ar(val); return val; } #endif @@ -171,7 +171,7 @@ static inline u16 readw(const volatile void __iomem *addr) __io_br(); val = __le16_to_cpu(__raw_readw(addr)); - __io_ar(); + __io_ar(val); return val; } #endif @@ -184,7 +184,7 @@ static inline u32 readl(const volatile void __iomem *addr) __io_br(); val = __le32_to_cpu(__raw_readl(addr)); - __io_ar(); + __io_ar(val); return val; } #endif @@ -198,7 +198,7 @@ static inline u64 readq(const volatile void __iomem *addr) __io_br(); val = __le64_to_cpu(__raw_readq(addr)); - __io_ar(); + __io_ar(val); return val; } #endif @@ -471,7 +471,7 @@ static inline u8 inb(unsigned long addr) __io_pbr(); val = __raw_readb(PCI_IOBASE + addr); - __io_par(); + __io_par(val); return val; } #endif @@ -484,7 +484,7 @@ static inline u16 inw(unsigned long addr) __io_pbr(); val = __le16_to_cpu(__raw_readw(PCI_IOBASE + addr)); - __io_par(); + __io_par(val); return val; } #endif @@ -497,7 +497,7 @@ static inline u32 inl(unsigned long addr) __io_pbr(); val = __le32_to_cpu(__raw_readl(PCI_IOBASE + addr)); - __io_par(); + __io_par(val); return val; } #endif diff --git a/include/asm-generic/iomap.h b/include/asm-generic/iomap.h index 5b63b94ef6b5..a008f504a2d0 100644 --- a/include/asm-generic/iomap.h +++ b/include/asm-generic/iomap.h @@ -36,6 +36,17 @@ extern u64 ioread64(void __iomem *); extern u64 ioread64be(void __iomem *); #endif +#ifdef readq +#define ioread64_lo_hi ioread64_lo_hi +#define ioread64_hi_lo ioread64_hi_lo +#define ioread64be_lo_hi ioread64be_lo_hi +#define ioread64be_hi_lo ioread64be_hi_lo +extern u64 ioread64_lo_hi(void __iomem *addr); +extern u64 ioread64_hi_lo(void __iomem *addr); +extern u64 ioread64be_lo_hi(void __iomem *addr); +extern u64 ioread64be_hi_lo(void __iomem *addr); +#endif + extern void iowrite8(u8, void __iomem *); extern void iowrite16(u16, void __iomem *); extern void iowrite16be(u16, void __iomem *); @@ -46,6 +57,17 @@ extern void iowrite64(u64, void __iomem *); extern void iowrite64be(u64, void __iomem *); #endif +#ifdef writeq +#define iowrite64_lo_hi iowrite64_lo_hi +#define iowrite64_hi_lo iowrite64_hi_lo +#define iowrite64be_lo_hi iowrite64be_lo_hi +#define iowrite64be_hi_lo iowrite64be_hi_lo +extern void iowrite64_lo_hi(u64 val, void __iomem *addr); +extern void iowrite64_hi_lo(u64 val, void __iomem *addr); +extern void iowrite64be_lo_hi(u64 val, void __iomem *addr); +extern void iowrite64be_hi_lo(u64 val, void __iomem *addr); +#endif + /* * "string" versions of the above. Note that they * use native byte ordering for the accesses (on diff --git a/include/asm-generic/page.h b/include/asm-generic/page.h index 27bf3377b0cb..fe801f01625e 100644 --- a/include/asm-generic/page.h +++ b/include/asm-generic/page.h @@ -7,7 +7,7 @@ */ #ifdef CONFIG_MMU -#error need to prove a real asm/page.h +#error need to provide a real asm/page.h #endif diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 05e61e6c843f..fa782fba51ee 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -606,7 +606,7 @@ static inline int pmd_none_or_clear_bad(pmd_t *pmd) return 0; } -static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm, +static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -615,10 +615,10 @@ static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm, * non-present, preventing the hardware from asynchronously * updating it. */ - return ptep_get_and_clear(mm, addr, ptep); + return ptep_get_and_clear(vma->vm_mm, addr, ptep); } -static inline void __ptep_modify_prot_commit(struct mm_struct *mm, +static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte) { @@ -626,7 +626,7 @@ static inline void __ptep_modify_prot_commit(struct mm_struct *mm, * The pte is non-present, so there's no hardware state to * preserve. */ - set_pte_at(mm, addr, ptep, pte); + set_pte_at(vma->vm_mm, addr, ptep, pte); } #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION @@ -644,22 +644,22 @@ static inline void __ptep_modify_prot_commit(struct mm_struct *mm, * queue the update to be done at some later time. The update must be * actually committed before the pte lock is released, however. */ -static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, +static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - return __ptep_modify_prot_start(mm, addr, ptep); + return __ptep_modify_prot_start(vma, addr, ptep); } /* * Commit an update to a pte, leaving any hardware-controlled bits in * the PTE unmodified. */ -static inline void ptep_modify_prot_commit(struct mm_struct *mm, +static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep, pte_t pte) + pte_t *ptep, pte_t old_pte, pte_t pte) { - __ptep_modify_prot_commit(mm, addr, ptep, pte); + __ptep_modify_prot_commit(vma, addr, ptep, pte); } #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ #endif /* CONFIG_MMU */ diff --git a/include/asm-generic/shmparam.h b/include/asm-generic/shmparam.h new file mode 100644 index 000000000000..8b78c0ba08b1 --- /dev/null +++ b/include/asm-generic/shmparam.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_GENERIC_SHMPARAM_H +#define __ASM_GENERIC_SHMPARAM_H + +#define SHMLBA PAGE_SIZE /* attach addr a multiple of this */ + +#endif /* _ASM_GENERIC_SHMPARAM_H */ diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d82c78a79da5..b3d2241e03f8 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -22,7 +22,6 @@ #endif #ifndef get_fs -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) static inline void set_fs(mm_segment_t fs) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 3d7a6a9c2370..f8f6f04c4453 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -733,7 +733,7 @@ KEEP(*(.orc_unwind_ip)) \ __stop_orc_unwind_ip = .; \ } \ - . = ALIGN(6); \ + . = ALIGN(2); \ .orc_unwind : AT(ADDR(.orc_unwind) - LOAD_OFFSET) { \ __start_orc_unwind = .; \ KEEP(*(.orc_unwind)) \ |