Implement atomic operations using the atomic instructions available

since ARMv6K.  As we also support ARMs that are older than that,
guard the new atomic operations with an ifdef specifically for ARMv7.

ok jsg@

2 files changed, 298 insertions, 7 deletions

diff --git a/sys/arch/arm/include/atomic.h b/sys/arch/arm/include/atomic.h
index a57d0f49808..b50a12e9c12 100644
--- a/sys/arch/arm/include/atomic.h
+++ b/sys/arch/arm/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.13 2016/01/31 00:14:50 jsg Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.14 2016/04/25 08:00:43 patrick Exp $	*/
 
 /* Public Domain */
 
@@ -7,16 +7,14 @@
 
 #if defined(_KERNEL)
 
+#if !defined(CPU_ARMv7)
+
 #include <arm/cpufunc.h>
 #include <arm/armreg.h>
 
 /*
  * on pre-v6 arm processors, it is necessary to disable interrupts if
  * in the kernel and atomic updates are necessary without full mutexes
-*
- * eventually it would be interesting to have these functions
- * support the V6/V7+ atomic instructions ldrex/strex if available
- * on the CPU.
  */
 
 static inline unsigned int
@@ -194,7 +192,280 @@ atomic_clearbits_int(volatile unsigned int *uip, unsigned int v)
 	restore_interrupts(cpsr);
 }
 
-#if defined(CPU_ARMv7)
+#else /* !CPU_ARMv7 */
+
+/*
+ * Compare and set:
+ * ret = *ptr
+ * if (ret == expect)
+ * 	*ptr = new
+ * return (ret)
+ */
+#define def_atomic_cas(_f, _t)					\
+static inline _t						\
+_f(volatile _t *p, _t e, _t n)					\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	    "1:	ldrex %0, [%4]		\n\t"			\
+	    "	cmp %0, %3		\n\t"			\
+	    "	bne 2f			\n\t"			\
+	    "	strex %1, %2, [%4]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    "	b 3f			\n\t"			\
+	    "2:	clrex			\n\t"			\
+	    "3:				\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (n), "r" (e), "r" (p)				\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_cas(_atomic_cas_uint, unsigned int)
+def_atomic_cas(_atomic_cas_ulong, unsigned long)
+#undef def_atomic_cas
+
+#define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
+#define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
+
+static inline void *
+_atomic_cas_ptr(volatile void *p, void *e, void *n)
+{
+	void *ret;
+	uint32_t modified;
+
+	__asm volatile (
+	    "1:	ldrex %0, [%4]		\n\t"
+	    "	cmp %0, %3		\n\t"
+	    "	bne 2f			\n\t"
+	    "	strex %1, %2, [%4]	\n\t"
+	    "	cmp %1, #0		\n\t"
+	    "	bne 1b			\n\t"
+	    "	b 3f			\n\t"
+	    "2:	clrex			\n\t"
+	    "3:				\n\t"
+	    : "=&r" (ret), "=&r" (modified)
+	    : "r" (n), "r" (e), "r" (p)
+	    : "memory", "cc"
+	);
+	return (ret);
+}
+#define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
+
+/*
+ * Swap:
+ * ret = *p
+ * *p = val
+ * return (ret)
+ */
+#define def_atomic_swap(_f, _t)					\
+static inline _t						\
+_f(volatile _t *p, _t v)					\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	    "1:	ldrex %0, [%3]		\n\t"			\
+	    "	strex %1, %2, [%3]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (v), "r" (p)					\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_swap(_atomic_swap_uint, unsigned int)
+def_atomic_swap(_atomic_swap_ulong, unsigned long)
+#undef def_atomic_swap
+
+#define atomic_swap_uint(_p, _v) _atomic_swap_uint((_p), (_v))
+#define atomic_swap_ulong(_p, _v) _atomic_swap_ulong((_p), (_v))
+
+static inline void *
+_atomic_swap_ptr(volatile void *p, void *v)
+{
+	void *ret;
+	uint32_t modified;
+
+	__asm volatile (
+	    "1:	ldrex %0, [%3]		\n\t"
+	    "	strex %1, %2, [%3]	\n\t"
+	    "	cmp %1, #0		\n\t"
+	    "	bne 1b			\n\t"
+	    : "=&r" (ret), "=&r" (modified)
+	    : "r" (v), "r" (p)
+	    : "memory", "cc"
+	);
+	return (ret);
+}
+#define atomic_swap_ptr(_p, _v) _atomic_swap_ptr((_p), (_v))
+
+/*
+ * Increment returning the new value
+ * *p += 1
+ * return (*p)
+ */
+#define def_atomic_inc_nv(_f, _t)				\
+static inline _t						\
+_f(volatile _t *p)						\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	   "1:	ldrex %0, [%2]		\n\t"			\
+	    "	add %0, %0, #1		\n\t"			\
+	    "	strex %1, %0, [%2]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (p)						\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_inc_nv(_atomic_inc_int_nv, unsigned int)
+def_atomic_inc_nv(_atomic_inc_long_nv, unsigned long)
+#undef def_atomic_inc_nv
+
+#define atomic_inc_int_nv(_p) _atomic_inc_int_nv((_p))
+#define atomic_inc_long_nv(_p) _atomic_inc_long_nv((_p))
+
+/*
+ * Decrement returning the new value
+ * *p -= 1
+ * return (*p)
+ */
+#define def_atomic_dec_nv(_f, _t)				\
+static inline _t						\
+_f(volatile _t *p)						\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	    "1:	ldrex %0, [%2]		\n\t"			\
+	    "	sub %0, %0, #1		\n\t"			\
+	    "	strex %1, %0, [%2]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (p)						\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_dec_nv(_atomic_dec_int_nv, unsigned int)
+def_atomic_dec_nv(_atomic_dec_long_nv, unsigned long)
+#undef def_atomic_dec_nv
+
+#define atomic_dec_int_nv(_p) _atomic_dec_int_nv((_p))
+#define atomic_dec_long_nv(_p) _atomic_dec_long_nv((_p))
+
+
+/*
+ * Addition returning the new value
+ * *p += v
+ * return (*p)
+ */
+#define def_atomic_add_nv(_f, _t)				\
+static inline _t						\
+_f(volatile _t *p, _t v)					\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	    "1:	ldrex %0, [%2]		\n\t"			\
+	    "	add %0, %0, %3		\n\t"			\
+	    "	strex %1, %0, [%2]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (p), "r" (v)					\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_add_nv(_atomic_add_int_nv, unsigned int)
+def_atomic_add_nv(_atomic_add_long_nv, unsigned long)
+#undef def_atomic_add_nv
+
+#define atomic_add_int_nv(_p, _v) _atomic_add_int_nv((_p), (_v))
+#define atomic_add_long_nv(_p, _v) _atomic_add_long_nv((_p), (_v))
+
+/*
+ * Subtraction returning the new value
+ * *p -= v
+ * return (*p)
+ */
+#define def_atomic_sub_nv(_f, _t)				\
+static inline _t						\
+_f(volatile _t *p, _t v)					\
+{								\
+	_t ret, modified;					\
+								\
+	__asm volatile (					\
+	    "1:	ldrex %0, [%2]		\n\t"			\
+	    "	sub %0, %0, %3		\n\t"			\
+	    "	strex %1, %0, [%2]	\n\t"			\
+	    "	cmp %1, #0		\n\t"			\
+	    "	bne 1b			\n\t"			\
+	    : "=&r" (ret), "=&r" (modified)			\
+	    : "r" (p), "r" (v)					\
+	    : "memory", "cc"					\
+	);							\
+	return (ret);						\
+}
+def_atomic_sub_nv(_atomic_sub_int_nv, unsigned int)
+def_atomic_sub_nv(_atomic_sub_long_nv, unsigned long)
+#undef def_atomic_sub_nv
+
+#define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv((_p), (_v))
+#define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv((_p), (_v))
+
+/*
+ * Set bits
+ * *p = *p | v
+ */
+static inline void
+atomic_setbits_int(volatile unsigned int *p, unsigned int v)
+{
+	unsigned int modified, tmp;
+
+	__asm volatile (
+	    "1:	ldrex %0, [%3]		\n\t"
+	    "	orr %0, %0, %2		\n\t"
+	    "	strex %1, %0, [%3]	\n\t"
+	    "	cmp %1, #0		\n\t"
+	    "	bne 1b			\n\t"
+	    : "=&r" (tmp), "=&r" (modified)
+	    : "r" (v), "r" (p)
+	    : "memory", "cc"
+	);
+}
+
+/*
+ * Clear bits
+ * *p = *p & (~v)
+ */
+static inline void
+atomic_clearbits_int(volatile unsigned int *p, unsigned int v)
+{
+	unsigned int modified, tmp;
+
+	__asm volatile (
+	    "1:	ldrex %0, [%3]		\n\t"
+	    "	bic %0, %0, %2		\n\t"
+	    "	strex %1, %0, [%3]	\n\t"
+	    "	cmp %1, #0		\n\t"
+	    "	bne 1b			\n\t"
+	    : "=&r" (tmp), "=&r" (modified)
+	    : "r" (v), "r" (p)
+	    : "memory", "cc"
+	);
+}
+
 #define __membar(_f) do { __asm __volatile(_f ::: "memory"); } while (0)
 
 #define membar_enter()		__membar("dmb sy")
diff --git a/sys/arch/arm/include/frame.h b/sys/arch/arm/include/frame.h
index 10a476fe991..8aaaec07712 100644
--- a/sys/arch/arm/include/frame.h
+++ b/sys/arch/arm/include/frame.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: frame.h,v 1.9 2016/04/24 12:11:46 patrick Exp $	*/
+/*	$OpenBSD: frame.h,v 1.10 2016/04/25 08:00:43 patrick Exp $	*/
 /*	$NetBSD: frame.h,v 1.9 2003/12/01 08:48:33 scw Exp $	*/
 
 /*
@@ -201,11 +201,28 @@ struct frame {
  */
 
 /*
+ * CLREX - On ARMv7 machines that support atomic instructions, we need
+ * to clear the exclusive monitors on kernel exit, so that a userland
+ * atomic store can't succeed due to an unrelated outstanding atomic
+ * operation. ARM also highly recommends clearing the monitor on data
+ * aborts, as the monitor state after taking a data abort is unknown.
+ * Issuing a clrex on kernel entry and on kernel exit is the easiest
+ * way to take care of both issues and to make sure that the kernel
+ * and userland do not leave any outstanding reserves active.
+ */
+#if defined(CPU_ARMv7)
+#define CLREX clrex
+#else
+#define CLREX
+#endif
+
+/*
  * PUSHFRAME - macro to push a trap frame on the stack in the current mode
  * Since the current mode is used, the SVC lr field is not defined.
  */
 
 #define PUSHFRAME							   \
+	CLREX;								   \
 	sub	sp, sp, #4;		/* Align the stack */		   \
 	str	lr, [sp, #-4]!;		/* Push the return address */	   \
 	sub	sp, sp, #(4*17);	/* Adjust the stack pointer */	   \
@@ -220,6 +237,7 @@ struct frame {
  */
 
 #define PULLFRAME							   \
+	CLREX;								   \
 	ldr	r0, [sp], #0x0004;	/* Get the SPSR from stack */	   \
 	msr	spsr_fsxc, r0;						   \
 	ldmia	sp, {r0-r14}^;		/* Restore registers (usr mode) */ \
@@ -237,6 +255,7 @@ struct frame {
  */
 
 #define PUSHFRAMEINSVC							   \
+	CLREX;								   \
 	stmdb	sp, {r0-r3};		/* Save 4 registers */		   \
 	mov	r0, lr;			/* Save xxx32 r14 */		   \
 	mov	r1, sp;			/* Save xxx32 sp */		   \
@@ -267,6 +286,7 @@ struct frame {
  */
 
 #define PULLFRAMEFROMSVCANDEXIT						   \
+	CLREX;								   \
 	ldr	r0, [sp], #0x0004;	/* Get the SPSR from stack */	   \
 	msr	spsr_fsxc, r0;		/* restore SPSR */		   \
 	ldmia	sp, {r0-r14}^;		/* Restore registers (usr mode) */ \