From 4dedbf8dc1c825f2e4aa2d2624058533b5d76f85 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 25 Jul 2011 17:12:20 -0700 Subject: sparc64: kill page table quicklists With the recent mmu_gather changes that included generic RCU freeing of page-tables, it is now quite straightforward to implement gup_fast() on sparc64. This patch: Remove the page table quicklists. They are pointless and make it harder to use RCU page table freeing and share code with other architectures. BTW, this is the second time this has happened, see commit 3c936465249f ("[SPARC64]: Kill pgtable quicklists and use SLAB.") Signed-off-by: David S. Miller Signed-off-by: Peter Zijlstra Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sparc/include/asm/pgalloc_64.h | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'arch/sparc/include') diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 4e5e0878144f..cb4867de06a2 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -14,69 +13,68 @@ /* Page table allocation/freeing. */ +extern struct kmem_cache *pgtable_cache; + static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - return quicklist_alloc(0, GFP_KERNEL, NULL); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - quicklist_free(0, NULL, pgd); + kmem_cache_free(pgtable_cache, pgd); } #define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return quicklist_alloc(0, GFP_KERNEL, NULL); + return kmem_cache_alloc(pgtable_cache, + GFP_KERNEL|__GFP_REPEAT); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { - quicklist_free(0, NULL, pmd); + kmem_cache_free(pgtable_cache, pmd); } static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return quicklist_alloc(0, GFP_KERNEL, NULL); + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *page; - void *pg; + pte_t *pte; - pg = quicklist_alloc(0, GFP_KERNEL, NULL); - if (!pg) + pte = pte_alloc_one_kernel(mm, address); + if (!pte) return NULL; - page = virt_to_page(pg); + page = virt_to_page(pte); pgtable_page_ctor(page); return page; } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { - quicklist_free(0, NULL, pte); + free_page((unsigned long)pte); } static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) { pgtable_page_dtor(ptepage); - quicklist_free_page(0, NULL, ptepage); + __free_page(ptepage); } - #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) #define pmd_populate(MM,PMD,PTE_PAGE) \ pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) #define pmd_pgtable(pmd) pmd_page(pmd) -static inline void check_pgt_cache(void) -{ - quicklist_trim(0, NULL, 25, 16); -} +#define check_pgt_cache() do { } while (0) #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) -- cgit v1.2.3-59-g8ed1b From 4a0100f7546fb642e0e04a38fa7ca198cd016b47 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 25 Jul 2011 17:12:21 -0700 Subject: sparc64: use RCU page table freeing Make use of the generic RCU page table freeing on Sparc64, doing so allows for race-free software page-table walkers like gup_fast(). Signed-off-by: David S. Miller Signed-off-by: Peter Zijlstra Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sparc/Kconfig | 1 + arch/sparc/include/asm/pgalloc_64.h | 48 +++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) (limited to 'arch/sparc/include') diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index af64348775e2..1074dddcb104 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -41,6 +41,7 @@ config SPARC64 select HAVE_FUNCTION_TRACE_MCOUNT_TEST select HAVE_KRETPROBES select HAVE_KPROBES + select HAVE_RCU_TABLE_FREE if SMP select HAVE_MEMBLOCK select HAVE_SYSCALL_WRAPPERS select HAVE_DYNAMIC_FTRACE diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index cb4867de06a2..40b2d7a7023d 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -76,7 +76,51 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) #define check_pgt_cache() do { } while (0) -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +static inline void pgtable_free(void *table, bool is_page) +{ + if (is_page) + free_page((unsigned long)table); + else + kmem_cache_free(pgtable_cache, table); +} + +#ifdef CONFIG_SMP + +struct mmu_gather; +extern void tlb_remove_table(struct mmu_gather *, void *); + +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) +{ + unsigned long pgf = (unsigned long)table; + if (is_page) + pgf |= 0x1UL; + tlb_remove_table(tlb, (void *)pgf); +} + +static inline void __tlb_remove_table(void *_table) +{ + void *table = (void *)((unsigned long)_table & ~0x1UL); + bool is_page = false; + + if ((unsigned long)_table & 0x1UL) + is_page = true; + pgtable_free(table, is_page); +} +#else /* CONFIG_SMP */ +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) +{ + pgtable_free(table, is_page); +} +#endif /* !CONFIG_SMP */ + +static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage, + unsigned long address) +{ + pgtable_page_dtor(ptepage); + pgtable_free_tlb(tlb, page_address(ptepage), true); +} + +#define __pmd_free_tlb(tlb, pmd, addr) \ + pgtable_free_tlb(tlb, pmd, false) #endif /* _SPARC64_PGALLOC_H */ -- cgit v1.2.3-59-g8ed1b From 683d2fa672da5e3b4fe96f13c43eba32b068d64b Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 25 Jul 2011 17:12:21 -0700 Subject: sparc64: add support for _PAGE_SPECIAL Luckily there are still a few software PTE bits remaining and they even match up in both the sun4u and sun4v pte layouts. Signed-off-by: David S. Miller Signed-off-by: Peter Zijlstra Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sparc/include/asm/pgtable_64.h | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'arch/sparc/include') diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 1e03c5a6b4f7..adf89329af59 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -95,6 +95,10 @@ /* PTE bits which are the same in SUN4U and SUN4V format. */ #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ +#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ + +/* Advertise support for _PAGE_SPECIAL */ +#define __HAVE_ARCH_PTE_SPECIAL /* SUN4U pte bits... */ #define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ @@ -104,6 +108,7 @@ #define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */ #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ +#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ @@ -133,6 +138,7 @@ #define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */ #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ +#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ @@ -302,10 +308,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) : "=r" (mask), "=r" (tmp) : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | - _PAGE_SZBITS_4U), + _PAGE_SZBITS_4U | _PAGE_SPECIAL), "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | - _PAGE_SZBITS_4V)); + _PAGE_SZBITS_4V | _PAGE_SPECIAL)); return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); } @@ -502,6 +508,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkspecial(pte_t pte) { + pte_val(pte) |= _PAGE_SPECIAL; return pte; } @@ -607,9 +614,9 @@ static inline unsigned long pte_present(pte_t pte) return val; } -static inline int pte_special(pte_t pte) +static inline unsigned long pte_special(pte_t pte) { - return 0; + return pte_val(pte) & _PAGE_SPECIAL; } #define pmd_set(pmdp, ptep) \ -- cgit v1.2.3-59-g8ed1b