diff options
Diffstat (limited to 'include/linux/vmstat.h')
| -rw-r--r-- | include/linux/vmstat.h | 25 |
1 files changed, 15 insertions, 10 deletions
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 67ce70c8279b..45c9cd1daf7a 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -27,9 +27,13 @@ struct vm_event_state { DECLARE_PER_CPU(struct vm_event_state, vm_event_states); +/* + * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the + * local_irq_disable overhead. + */ static inline void __count_vm_event(enum vm_event_item item) { - __this_cpu_inc(vm_event_states.event[item]); + raw_cpu_inc(vm_event_states.event[item]); } static inline void count_vm_event(enum vm_event_item item) @@ -39,7 +43,7 @@ static inline void count_vm_event(enum vm_event_item item) static inline void __count_vm_events(enum vm_event_item item, long delta) { - __this_cpu_add(vm_event_states.event[item], delta); + raw_cpu_add(vm_event_states.event[item], delta); } static inline void count_vm_events(enum vm_event_item item, long delta) @@ -187,8 +191,6 @@ extern void zone_statistics(struct zone *, struct zone *, gfp_t gfp); #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d) #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d)) -extern void inc_zone_state(struct zone *, enum zone_stat_item); - #ifdef CONFIG_SMP void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int); void __inc_zone_page_state(struct page *, enum zone_stat_item); @@ -230,18 +232,18 @@ static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item) atomic_long_inc(&vm_stat[item]); } -static inline void __inc_zone_page_state(struct page *page, - enum zone_stat_item item) -{ - __inc_zone_state(page_zone(page), item); -} - static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item) { atomic_long_dec(&zone->vm_stat[item]); atomic_long_dec(&vm_stat[item]); } +static inline void __inc_zone_page_state(struct page *page, + enum zone_stat_item item) +{ + __inc_zone_state(page_zone(page), item); +} + static inline void __dec_zone_page_state(struct page *page, enum zone_stat_item item) { @@ -256,6 +258,9 @@ static inline void __dec_zone_page_state(struct page *page, #define dec_zone_page_state __dec_zone_page_state #define mod_zone_page_state __mod_zone_page_state +#define inc_zone_state __inc_zone_state +#define dec_zone_state __dec_zone_state + #define set_pgdat_percpu_threshold(pgdat, callback) { } static inline void refresh_cpu_vm_stats(int cpu) { } |