diff options
Diffstat (limited to 'scripts/atomic/fallbacks')
| -rwxr-xr-x | scripts/atomic/fallbacks/acquire | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/add_negative | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/add_unless | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/andnot | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/dec | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/dec_and_test | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/dec_if_positive | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/dec_unless_positive | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/fence | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/fetch_add_unless | 10 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/inc | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/inc_and_test | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/inc_not_zero | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/inc_unless_negative | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/read_acquire | 15 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/release | 6 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/set_release | 11 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/sub_and_test | 8 | ||||
| -rwxr-xr-x | scripts/atomic/fallbacks/try_cmpxchg | 6 |
19 files changed, 82 insertions, 68 deletions
diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire index e38871e64db6..ef764085c79a 100755 --- a/scripts/atomic/fallbacks/acquire +++ b/scripts/atomic/fallbacks/acquire @@ -1,8 +1,8 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}_acquire(${params}) +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}_acquire(${params}) { - ${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); __atomic_acquire_fence(); return ret; } diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative index e6f4815637de..15caa2eb2371 100755 --- a/scripts/atomic/fallbacks/add_negative +++ b/scripts/atomic/fallbacks/add_negative @@ -1,6 +1,6 @@ cat <<EOF /** - * ${atomic}_add_negative - add and test if negative + * arch_${atomic}_add_negative - add and test if negative * @i: integer value to add * @v: pointer of type ${atomic}_t * @@ -8,9 +8,9 @@ cat <<EOF * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline bool -${atomic}_add_negative(${int} i, ${atomic}_t *v) +static __always_inline bool +arch_${atomic}_add_negative(${int} i, ${atomic}_t *v) { - return ${atomic}_add_return(i, v) < 0; + return arch_${atomic}_add_return(i, v) < 0; } EOF diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless index 792533885fbf..9e5159c2ccfc 100755 --- a/scripts/atomic/fallbacks/add_unless +++ b/scripts/atomic/fallbacks/add_unless @@ -1,6 +1,6 @@ cat << EOF /** - * ${atomic}_add_unless - add unless the number is already a given value + * arch_${atomic}_add_unless - add unless the number is already a given value * @v: pointer of type ${atomic}_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -8,9 +8,9 @@ cat << EOF * Atomically adds @a to @v, if @v was not already @u. * Returns true if the addition was done. */ -static inline bool -${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) +static __always_inline bool +arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) { - return ${atomic}_fetch_add_unless(v, a, u) != u; + return arch_${atomic}_fetch_add_unless(v, a, u) != u; } EOF diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot index 9f3a3216b5e3..5a42f54a3595 100755 --- a/scripts/atomic/fallbacks/andnot +++ b/scripts/atomic/fallbacks/andnot @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) +static __always_inline ${ret} +arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v); + ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v); } EOF diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec index 10bbc82be31d..8c144c818e9e 100755 --- a/scripts/atomic/fallbacks/dec +++ b/scripts/atomic/fallbacks/dec @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) +static __always_inline ${ret} +arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v); + ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v); } EOF diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test index 0ce7103b3df2..8549f359bd0e 100755 --- a/scripts/atomic/fallbacks/dec_and_test +++ b/scripts/atomic/fallbacks/dec_and_test @@ -1,15 +1,15 @@ cat <<EOF /** - * ${atomic}_dec_and_test - decrement and test + * arch_${atomic}_dec_and_test - decrement and test * @v: pointer of type ${atomic}_t * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. */ -static inline bool -${atomic}_dec_and_test(${atomic}_t *v) +static __always_inline bool +arch_${atomic}_dec_and_test(${atomic}_t *v) { - return ${atomic}_dec_return(v) == 0; + return arch_${atomic}_dec_return(v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive index c52eacec43c8..86bdced3428d 100755 --- a/scripts/atomic/fallbacks/dec_if_positive +++ b/scripts/atomic/fallbacks/dec_if_positive @@ -1,14 +1,14 @@ cat <<EOF -static inline ${ret} -${atomic}_dec_if_positive(${atomic}_t *v) +static __always_inline ${ret} +arch_${atomic}_dec_if_positive(${atomic}_t *v) { - ${int} dec, c = ${atomic}_read(v); + ${int} dec, c = arch_${atomic}_read(v); do { dec = c - 1; if (unlikely(dec < 0)) break; - } while (!${atomic}_try_cmpxchg(v, &c, dec)); + } while (!arch_${atomic}_try_cmpxchg(v, &c, dec)); return dec; } diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive index 8a2578f14268..c531d5afecc4 100755 --- a/scripts/atomic/fallbacks/dec_unless_positive +++ b/scripts/atomic/fallbacks/dec_unless_positive @@ -1,13 +1,13 @@ cat <<EOF -static inline bool -${atomic}_dec_unless_positive(${atomic}_t *v) +static __always_inline bool +arch_${atomic}_dec_unless_positive(${atomic}_t *v) { - ${int} c = ${atomic}_read(v); + ${int} c = arch_${atomic}_read(v); do { if (unlikely(c > 0)) return false; - } while (!${atomic}_try_cmpxchg(v, &c, c - 1)); + } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1)); return true; } diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence index 82f68fa6931a..07757d8e338e 100755 --- a/scripts/atomic/fallbacks/fence +++ b/scripts/atomic/fallbacks/fence @@ -1,10 +1,10 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}(${params}) +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}(${params}) { ${ret} ret; __atomic_pre_full_fence(); - ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); __atomic_post_full_fence(); return ret; } diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless index d2c091db7eae..68ce13c8b9da 100755 --- a/scripts/atomic/fallbacks/fetch_add_unless +++ b/scripts/atomic/fallbacks/fetch_add_unless @@ -1,6 +1,6 @@ cat << EOF /** - * ${atomic}_fetch_add_unless - add unless the number is already a given value + * arch_${atomic}_fetch_add_unless - add unless the number is already a given value * @v: pointer of type ${atomic}_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -8,15 +8,15 @@ cat << EOF * Atomically adds @a to @v, so long as @v was not already @u. * Returns original value of @v */ -static inline ${int} -${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) +static __always_inline ${int} +arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) { - ${int} c = ${atomic}_read(v); + ${int} c = arch_${atomic}_read(v); do { if (unlikely(c == u)) break; - } while (!${atomic}_try_cmpxchg(v, &c, c + a)); + } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a)); return c; } diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc index f866b3ad2353..3c2c3739169e 100755 --- a/scripts/atomic/fallbacks/inc +++ b/scripts/atomic/fallbacks/inc @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) +static __always_inline ${ret} +arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v); + ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v); } EOF diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test index 4e2068869f7e..0cf23fe1efb8 100755 --- a/scripts/atomic/fallbacks/inc_and_test +++ b/scripts/atomic/fallbacks/inc_and_test @@ -1,15 +1,15 @@ cat <<EOF /** - * ${atomic}_inc_and_test - increment and test + * arch_${atomic}_inc_and_test - increment and test * @v: pointer of type ${atomic}_t * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. */ -static inline bool -${atomic}_inc_and_test(${atomic}_t *v) +static __always_inline bool +arch_${atomic}_inc_and_test(${atomic}_t *v) { - return ${atomic}_inc_return(v) == 0; + return arch_${atomic}_inc_return(v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero index a7c45c8d107c..ed8a1f562667 100755 --- a/scripts/atomic/fallbacks/inc_not_zero +++ b/scripts/atomic/fallbacks/inc_not_zero @@ -1,14 +1,14 @@ cat <<EOF /** - * ${atomic}_inc_not_zero - increment unless the number is zero + * arch_${atomic}_inc_not_zero - increment unless the number is zero * @v: pointer of type ${atomic}_t * * Atomically increments @v by 1, if @v is non-zero. * Returns true if the increment was done. */ -static inline bool -${atomic}_inc_not_zero(${atomic}_t *v) +static __always_inline bool +arch_${atomic}_inc_not_zero(${atomic}_t *v) { - return ${atomic}_add_unless(v, 1, 0); + return arch_${atomic}_add_unless(v, 1, 0); } EOF diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative index 0c266e71dbd4..95d8ce48233f 100755 --- a/scripts/atomic/fallbacks/inc_unless_negative +++ b/scripts/atomic/fallbacks/inc_unless_negative @@ -1,13 +1,13 @@ cat <<EOF -static inline bool -${atomic}_inc_unless_negative(${atomic}_t *v) +static __always_inline bool +arch_${atomic}_inc_unless_negative(${atomic}_t *v) { - ${int} c = ${atomic}_read(v); + ${int} c = arch_${atomic}_read(v); do { if (unlikely(c < 0)) return false; - } while (!${atomic}_try_cmpxchg(v, &c, c + 1)); + } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1)); return true; } diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire index 75863b5203f7..a0ea1d26e6b2 100755 --- a/scripts/atomic/fallbacks/read_acquire +++ b/scripts/atomic/fallbacks/read_acquire @@ -1,7 +1,16 @@ cat <<EOF -static inline ${ret} -${atomic}_read_acquire(const ${atomic}_t *v) +static __always_inline ${ret} +arch_${atomic}_read_acquire(const ${atomic}_t *v) { - return smp_load_acquire(&(v)->counter); + ${int} ret; + + if (__native_word(${atomic}_t)) { + ret = smp_load_acquire(&(v)->counter); + } else { + ret = arch_${atomic}_read(v); + __atomic_acquire_fence(); + } + + return ret; } EOF diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release index 3f628a3802d9..b46feb56d69c 100755 --- a/scripts/atomic/fallbacks/release +++ b/scripts/atomic/fallbacks/release @@ -1,8 +1,8 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}_release(${params}) +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}_release(${params}) { __atomic_release_fence(); - ${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); } EOF diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release index 45bb5e0cfc08..05cdb7f42477 100755 --- a/scripts/atomic/fallbacks/set_release +++ b/scripts/atomic/fallbacks/set_release @@ -1,7 +1,12 @@ cat <<EOF -static inline void -${atomic}_set_release(${atomic}_t *v, ${int} i) +static __always_inline void +arch_${atomic}_set_release(${atomic}_t *v, ${int} i) { - smp_store_release(&(v)->counter, i); + if (__native_word(${atomic}_t)) { + smp_store_release(&(v)->counter, i); + } else { + __atomic_release_fence(); + arch_${atomic}_set(v, i); + } } EOF diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test index 289ef17a2d7a..260f37341c88 100755 --- a/scripts/atomic/fallbacks/sub_and_test +++ b/scripts/atomic/fallbacks/sub_and_test @@ -1,6 +1,6 @@ cat <<EOF /** - * ${atomic}_sub_and_test - subtract value from variable and test result + * arch_${atomic}_sub_and_test - subtract value from variable and test result * @i: integer value to subtract * @v: pointer of type ${atomic}_t * @@ -8,9 +8,9 @@ cat <<EOF * true if the result is zero, or false for all * other cases. */ -static inline bool -${atomic}_sub_and_test(${int} i, ${atomic}_t *v) +static __always_inline bool +arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v) { - return ${atomic}_sub_return(i, v) == 0; + return arch_${atomic}_sub_return(i, v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg index 4ed85e2f5378..890f850ede37 100755 --- a/scripts/atomic/fallbacks/try_cmpxchg +++ b/scripts/atomic/fallbacks/try_cmpxchg @@ -1,9 +1,9 @@ cat <<EOF -static inline bool -${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) +static __always_inline bool +arch_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) { ${int} r, o = *old; - r = ${atomic}_cmpxchg${order}(v, o, new); + r = arch_${atomic}_cmpxchg${order}(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); |