#ifndef _ASM_GENERIC_PERCPU_H_ #define _ASM_GENERIC_PERCPU_H_ #include #include #include #ifdef CONFIG_SMP /* * per_cpu_offset() is the offset that has to be added to a * percpu variable to get to the instance for a certain processor. * * Most arches use the __per_cpu_offset array for those offsets but * some arches have their own ways of determining the offset (x86_64, s390). */ #ifndef __per_cpu_offset extern unsigned long __per_cpu_offset[NR_CPUS]; #define per_cpu_offset(x) (__per_cpu_offset[x]) #endif /* * Determine the offset for the currently active processor. * An arch may define __my_cpu_offset to provide a more effective * means of obtaining the offset to the per cpu variables of the * current processor. */ #ifndef __my_cpu_offset #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id()) #endif #ifdef CONFIG_DEBUG_PREEMPT #define my_cpu_offset per_cpu_offset(smp_processor_id()) #else #define my_cpu_offset __my_cpu_offset #endif /* * Arch may define arch_raw_cpu_ptr() to provide more efficient address * translations for raw_cpu_ptr(). */ #ifndef arch_raw_cpu_ptr #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) #endif #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas(void); #endif #endif /* SMP */ #ifndef PER_CPU_BASE_SECTION #ifdef CONFIG_SMP #define PER_CPU_BASE_SECTION ".data..percpu" #else #define PER_CPU_BASE_SECTION ".data" #endif #endif #ifndef PER_CPU_ATTRIBUTES #define PER_CPU_ATTRIBUTES #endif #ifndef PER_CPU_DEF_ATTRIBUTES #define PER_CPU_DEF_ATTRIBUTES #endif # ifndef raw_cpu_read_1 # define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp))) # endif # ifndef raw_cpu_read_2 # define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp))) # endif # ifndef raw_cpu_read_4 # define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp))) # endif # ifndef raw_cpu_read_8 # define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp))) # endif #define raw_cpu_generic_to_op(pcp, val, op) \ do { \ *raw_cpu_ptr(&(pcp)) op val; \ } while (0) # ifndef raw_cpu_write_1 # define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif # ifndef raw_cpu_write_2 # define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif # ifndef raw_cpu_write_4 # define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif # ifndef raw_cpu_write_8 # define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif # ifndef raw_cpu_add_1 # define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef raw_cpu_add_2 # define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef raw_cpu_add_4 # define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef raw_cpu_add_8 # define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef raw_cpu_and_1 # define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef raw_cpu_and_2 # define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef raw_cpu_and_4 # define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef raw_cpu_and_8 # define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef raw_cpu_or_1 # define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef raw_cpu_or_2 # define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef raw_cpu_or_4 # define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef raw_cpu_or_8 # define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif #define raw_cpu_generic_add_return(pcp, val) \ ({ \ raw_cpu_add(pcp, val); \ raw_cpu_read(pcp); \ }) # ifndef raw_cpu_add_return_1 # define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) # endif # ifndef raw_cpu_add_return_2 # define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val) # endif # ifndef raw_cpu_add_return_4 # define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val) # endif # ifndef raw_cpu_add_return_8 # define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val) # endif #define raw_cpu_generic_xchg(pcp, nval) \ ({ typeof(pcp) ret__; \ ret__ = raw_cpu_read(pcp); \ raw_cpu_write(pcp, nval); \ ret__; \ }) # ifndef raw_cpu_xchg_1 # define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) # endif # ifndef raw_cpu_xchg_2 # define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval) # endif # ifndef raw_cpu_xchg_4 # define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval) # endif # ifndef raw_cpu_xchg_8 # define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) # endif #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ typeof(pcp) ret__; \ ret__ = raw_cpu_read(pcp); \ if (ret__ == (oval)) \ raw_cpu_write(pcp, nval); \ ret__; \ }) # ifndef raw_cpu_cmpxchg_1 # define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef raw_cpu_cmpxchg_2 # define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef raw_cpu_cmpxchg_4 # define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef raw_cpu_cmpxchg_8 # define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) # endif #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ int __ret = 0; \ if (raw_cpu_read(pcp1) == (oval1) && \ raw_cpu_read(pcp2) == (oval2)) { \ raw_cpu_write(pcp1, (nval1)); \ raw_cpu_write(pcp2, (nval2)); \ __ret = 1; \ } \ (__ret); \ }) # ifndef raw_cpu_cmpxchg_double_1 # define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef raw_cpu_cmpxchg_double_2 # define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef raw_cpu_cmpxchg_double_4 # define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef raw_cpu_cmpxchg_double_8 # define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif #define _this_cpu_generic_read(pcp) \ ({ typeof(pcp) ret__; \ preempt_disable(); \ ret__ = *this_cpu_ptr(&(pcp)); \ preempt_enable(); \ ret__; \ }) # ifndef this_cpu_read_1 # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) # endif # ifndef this_cpu_read_2 # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp) # endif # ifndef this_cpu_read_4 # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp) # endif # ifndef this_cpu_read_8 # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) # endif #define _this_cpu_generic_to_op(pcp, val, op) \ do { \ unsigned long flags; \ raw_local_irq_save(flags); \ *raw_cpu_ptr(&(pcp)) op val; \ raw_local_irq_restore(flags); \ } while (0) # ifndef this_cpu_write_1 # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif # ifndef this_cpu_write_2 # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif # ifndef this_cpu_write_4 # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif # ifndef this_cpu_write_8 # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif # ifndef this_cpu_add_1 # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef this_cpu_add_2 # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef this_cpu_add_4 # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef this_cpu_add_8 # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif # ifndef this_cpu_and_1 # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef this_cpu_and_2 # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef this_cpu_and_4 # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef this_cpu_and_8 # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif # ifndef this_cpu_or_1 # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef this_cpu_or_2 # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef this_cpu_or_4 # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif # ifndef this_cpu_or_8 # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif #define _this_cpu_generic_add_return(pcp, val) \ ({ \ typeof(pcp) ret__; \ unsigned long flags; \ raw_local_irq_save(flags); \ raw_cpu_add(pcp, val); \ ret__ = raw_cpu_read(pcp); \ raw_local_irq_restore(flags); \ ret__; \ }) # ifndef this_cpu_add_return_1 # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif # ifndef this_cpu_add_return_2 # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif # ifndef this_cpu_add_return_4 # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif # ifndef this_cpu_add_return_8 # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif #define _this_cpu_generic_xchg(pcp, nval) \ ({ typeof(pcp) ret__; \ unsigned long flags; \ raw_local_irq_save(flags); \ ret__ = raw_cpu_read(pcp); \ raw_cpu_write(pcp, nval); \ raw_local_irq_restore(flags); \ ret__; \ }) # ifndef this_cpu_xchg_1 # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval) # endif # ifndef this_cpu_xchg_2 # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval) # endif # ifndef this_cpu_xchg_4 # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval) # endif # ifndef this_cpu_xchg_8 # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval) # endif #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ typeof(pcp) ret__; \ unsigned long flags; \ raw_local_irq_save(flags); \ ret__ = raw_cpu_read(pcp); \ if (ret__ == (oval)) \ raw_cpu_write(pcp, nval); \ raw_local_irq_restore(flags); \ ret__; \ }) # ifndef this_cpu_cmpxchg_1 # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef this_cpu_cmpxchg_2 # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef this_cpu_cmpxchg_4 # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) # endif # ifndef this_cpu_cmpxchg_8 # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) # endif #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ int ret__; \ unsigned long flags; \ raw_local_irq_save(flags); \ ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ oval1, oval2, nval1, nval2); \ raw_local_irq_restore(flags); \ ret__; \ }) # ifndef this_cpu_cmpxchg_double_1 # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef this_cpu_cmpxchg_double_2 # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef this_cpu_cmpxchg_double_4 # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif # ifndef this_cpu_cmpxchg_double_8 # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) # endif #endif /* _ASM_GENERIC_PERCPU_H_ */