From 0b9c99b6f21c2e9e00938e9c57942ed71bfe4d21 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 30 Jan 2008 13:30:35 +0100 Subject: x86: cleanup tlbflush.h variants Bring the tlbflush.h variants into sync to prepare merging and paravirt support. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/cpufeature.h | 9 ++++ include/asm-x86/tlbflush_32.h | 120 ++++++++++++++++++------------------------ include/asm-x86/tlbflush_64.h | 78 ++++++++++++++++++++------- 3 files changed, 118 insertions(+), 89 deletions(-) (limited to 'include') diff --git a/include/asm-x86/cpufeature.h b/include/asm-x86/cpufeature.h index 4c7875554d01..acbf6681740d 100644 --- a/include/asm-x86/cpufeature.h +++ b/include/asm-x86/cpufeature.h @@ -163,6 +163,12 @@ #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH) #define cpu_has_bts boot_cpu_has(X86_FEATURE_BTS) +#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) +# define cpu_has_invlpg 1 +#else +# define cpu_has_invlpg (boot_cpu_data.x86 > 3) +#endif + #ifdef CONFIG_X86_64 #undef cpu_has_vme @@ -183,6 +189,9 @@ #undef cpu_has_centaur_mcr #define cpu_has_centaur_mcr 0 +#undef cpu_has_pge +#define cpu_has_pge 1 + #endif /* CONFIG_X86_64 */ #endif /* _ASM_X86_CPUFEATURE_H */ diff --git a/include/asm-x86/tlbflush_32.h b/include/asm-x86/tlbflush_32.h index 2bd5b95e2048..9e07cc8f2d94 100644 --- a/include/asm-x86/tlbflush_32.h +++ b/include/asm-x86/tlbflush_32.h @@ -1,8 +1,11 @@ -#ifndef _I386_TLBFLUSH_H -#define _I386_TLBFLUSH_H +#ifndef _X86_TLBFLUSH_H +#define _X86_TLBFLUSH_H #include +#include + #include +#include #ifdef CONFIG_PARAVIRT #include @@ -12,62 +15,41 @@ #define __flush_tlb_single(addr) __native_flush_tlb_single(addr) #endif -#define __native_flush_tlb() \ - do { \ - unsigned int tmpreg; \ - \ - __asm__ __volatile__( \ - "movl %%cr3, %0; \n" \ - "movl %0, %%cr3; # flush TLB \n" \ - : "=r" (tmpreg) \ - :: "memory"); \ - } while (0) +static inline void __native_flush_tlb(void) +{ + write_cr3(read_cr3()); +} -/* - * Global pages have to be flushed a bit differently. Not a real - * performance problem because this does not happen often. - */ -#define __native_flush_tlb_global() \ - do { \ - unsigned int tmpreg, cr4, cr4_orig; \ - \ - __asm__ __volatile__( \ - "movl %%cr4, %2; # turn off PGE \n" \ - "movl %2, %1; \n" \ - "andl %3, %1; \n" \ - "movl %1, %%cr4; \n" \ - "movl %%cr3, %0; \n" \ - "movl %0, %%cr3; # flush TLB \n" \ - "movl %2, %%cr4; # turn PGE back on \n" \ - : "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \ - : "i" (~X86_CR4_PGE) \ - : "memory"); \ - } while (0) - -#define __native_flush_tlb_single(addr) \ - __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") - -# define __flush_tlb_all() \ - do { \ - if (cpu_has_pge) \ - __flush_tlb_global(); \ - else \ - __flush_tlb(); \ - } while (0) - -#define cpu_has_invlpg (boot_cpu_data.x86 > 3) - -#ifdef CONFIG_X86_INVLPG -# define __flush_tlb_one(addr) __flush_tlb_single(addr) -#else -# define __flush_tlb_one(addr) \ - do { \ - if (cpu_has_invlpg) \ - __flush_tlb_single(addr); \ - else \ - __flush_tlb(); \ - } while (0) -#endif +static inline void __native_flush_tlb_global(void) +{ + unsigned long cr4 = read_cr4(); + + /* clear PGE */ + write_cr4(cr4 & ~X86_CR4_PGE); + /* write old PGE again and flush TLBs */ + write_cr4(cr4); +} + +static inline void __native_flush_tlb_single(unsigned long addr) +{ + __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory"); +} + +static inline void __flush_tlb_all(void) +{ + if (cpu_has_pge) + __flush_tlb_global(); + else + __flush_tlb(); +} + +static inline void __flush_tlb_one(unsigned long addr) +{ + if (cpu_has_invlpg) + __flush_tlb_single(addr); + else + __flush_tlb(); +} /* * TLB flushing: @@ -86,11 +68,8 @@ #define TLB_FLUSH_ALL 0xffffffff - #ifndef CONFIG_SMP -#include - #define flush_tlb() __flush_tlb() #define flush_tlb_all() __flush_tlb_all() #define local_flush_tlb() __flush_tlb() @@ -102,21 +81,22 @@ static inline void flush_tlb_mm(struct mm_struct *mm) } static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long addr) + unsigned long addr) { if (vma->vm_mm == current->active_mm) __flush_tlb_one(addr); } static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end) { if (vma->vm_mm == current->active_mm) __flush_tlb(); } static inline void native_flush_tlb_others(const cpumask_t *cpumask, - struct mm_struct *mm, unsigned long va) + struct mm_struct *mm, + unsigned long va) { } @@ -124,8 +104,7 @@ static inline void native_flush_tlb_others(const cpumask_t *cpumask, #include -#define local_flush_tlb() \ - __flush_tlb() +#define local_flush_tlb() __flush_tlb() extern void flush_tlb_all(void); extern void flush_tlb_current_task(void); @@ -134,7 +113,8 @@ extern void flush_tlb_page(struct vm_area_struct *, unsigned long); #define flush_tlb() flush_tlb_current_task() -static inline void flush_tlb_range(struct vm_area_struct * vma, unsigned long start, unsigned long end) +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) { flush_tlb_mm(vma->vm_mm); } @@ -152,17 +132,17 @@ struct tlb_state char __cacheline_padding[L1_CACHE_BYTES-8]; }; DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate); + #endif /* SMP */ #ifndef CONFIG_PARAVIRT -#define flush_tlb_others(mask, mm, va) \ - native_flush_tlb_others(&mask, mm, va) +#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(&mask, mm, va) #endif static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) + unsigned long end) { flush_tlb_all(); } -#endif /* _I386_TLBFLUSH_H */ +#endif /* _X86_TLBFLUSH_H */ diff --git a/include/asm-x86/tlbflush_64.h b/include/asm-x86/tlbflush_64.h index 7731fd23d572..0bed440ba9fe 100644 --- a/include/asm-x86/tlbflush_64.h +++ b/include/asm-x86/tlbflush_64.h @@ -1,26 +1,55 @@ -#ifndef _X8664_TLBFLUSH_H -#define _X8664_TLBFLUSH_H +#ifndef _X86_TLBFLUSH_H +#define _X86_TLBFLUSH_H #include #include + #include #include -static inline void __flush_tlb(void) +#ifdef CONFIG_PARAVIRT +#include +#else +#define __flush_tlb() __native_flush_tlb() +#define __flush_tlb_global() __native_flush_tlb_global() +#define __flush_tlb_single(addr) __native_flush_tlb_single(addr) +#endif + +static inline void __native_flush_tlb(void) { write_cr3(read_cr3()); } -static inline void __flush_tlb_all(void) +static inline void __native_flush_tlb_global(void) { unsigned long cr4 = read_cr4(); - write_cr4(cr4 & ~X86_CR4_PGE); /* clear PGE */ - write_cr4(cr4); /* write old PGE again and flush TLBs */ + + /* clear PGE */ + write_cr4(cr4 & ~X86_CR4_PGE); + /* write old PGE again and flush TLBs */ + write_cr4(cr4); } -#define __flush_tlb_one(addr) \ - __asm__ __volatile__("invlpg (%0)" :: "r" (addr) : "memory") +static inline void __native_flush_tlb_single(unsigned long addr) +{ + __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory"); +} + +static inline void __flush_tlb_all(void) +{ + if (cpu_has_pge) + __flush_tlb_global(); + else + __flush_tlb(); +} +static inline void __flush_tlb_one(unsigned long addr) +{ + if (cpu_has_invlpg) + __flush_tlb_single(addr); + else + __flush_tlb(); +} /* * TLB flushing: @@ -37,6 +66,8 @@ static inline void __flush_tlb_all(void) * range a few INVLPGs in a row are a win. */ +#define TLB_FLUSH_ALL -1ULL + #ifndef CONFIG_SMP #define flush_tlb() __flush_tlb() @@ -50,25 +81,30 @@ static inline void flush_tlb_mm(struct mm_struct *mm) } static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long addr) + unsigned long addr) { if (vma->vm_mm == current->active_mm) __flush_tlb_one(addr); } static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end) { if (vma->vm_mm == current->active_mm) __flush_tlb(); } -#else +static inline void native_flush_tlb_others(const cpumask_t *cpumask, + struct mm_struct *mm, + unsigned long va) +{ +} + +#else /* SMP */ #include -#define local_flush_tlb() \ - __flush_tlb() +#define local_flush_tlb() __flush_tlb() extern void flush_tlb_all(void); extern void flush_tlb_current_task(void); @@ -77,24 +113,28 @@ extern void flush_tlb_page(struct vm_area_struct *, unsigned long); #define flush_tlb() flush_tlb_current_task() -static inline void flush_tlb_range(struct vm_area_struct * vma, unsigned long start, unsigned long end) +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) { flush_tlb_mm(vma->vm_mm); } +void native_flush_tlb_others(const cpumask_t *cpumask, struct mm_struct *mm, + unsigned long va); + #define TLBSTATE_OK 1 #define TLBSTATE_LAZY 2 -/* Roughly an IPI every 20MB with 4k pages for freeing page table - ranges. Cost is about 42k of memory for each CPU. */ -#define ARCH_FREE_PTE_NR 5350 +#endif /* SMP */ +#ifndef CONFIG_PARAVIRT +#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(&mask, mm, va) #endif static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) + unsigned long end) { flush_tlb_all(); } -#endif /* _X8664_TLBFLUSH_H */ +#endif /* _X86_TLBFLUSH_H */ -- cgit v1.2.3-59-g8ed1b