From 195568a10a264a733ec7151a657ab054a0af768f Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 5 Nov 2018 10:00:15 +0800 Subject: nios2: flush_tlb_page use PID based flush flush_tlb_page is for flushing user pages, so it should not be using flush_tlb_one (which flushes all pages). This patch implements it with the flush_tlb_range, which is a user flush that does the right thing. flush_tlb_one is made static to mm/tlb.c because it's a bit confusing. It is used in do_page_fault to flush the kernel non-linear mappings, so that is replaced with flush_tlb_kernel_page. The end result is that functionality is identical. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index cf10326aab1c..99e047082993 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -102,19 +102,11 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, } } -void flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - while (start < end) { - flush_tlb_one(start); - start += PAGE_SIZE; - } -} - /* * This one is only used for pages with the global bit set so we don't care * much about the ASID. */ -void flush_tlb_one(unsigned long addr) +static void flush_tlb_one(unsigned long addr) { unsigned int way; unsigned long org_misc, pid_misc; @@ -154,6 +146,14 @@ void flush_tlb_one(unsigned long addr) WRCTL(CTL_TLBMISC, org_misc); } +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + while (start < end) { + flush_tlb_one(start); + start += PAGE_SIZE; + } +} + void dump_tlb_line(unsigned long line) { unsigned int way; -- cgit v1.2.3-59-g8ed1b From 3437d3c886ed07863acde923a627395abb177aa9 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 5 Nov 2018 10:35:12 +0800 Subject: nios2: Use an invalid TLB entry address helper function There is no need for complicated calculation for an invalid address that maps to the same TLB index as the entry to be invalidated. Using the TLB address plus the two top bits set puts the address into the kernel TLB bypass range and still maps to the same cache line. This is also a bug fix for flush_tlb_pid, which is currently unused, but does not set PTEADDR to invalid. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 68 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 35 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index 99e047082993..eb568a919576 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -23,10 +23,6 @@ ((((1UL << (cpuinfo.tlb_ptr_sz - cpuinfo.tlb_num_ways_log2))) - 1) \ << PAGE_SHIFT) -/* Used as illegal PHYS_ADDR for TLB mappings - */ -#define MAX_PHYS_ADDR 0 - static void get_misc_and_pid(unsigned long *misc, unsigned long *pid) { *misc = RDCTL(CTL_TLBMISC); @@ -34,6 +30,15 @@ static void get_misc_and_pid(unsigned long *misc, unsigned long *pid) *pid = *misc & TLBMISC_PID; } +/* + * This provides a PTEADDR value for addr that will cause a TLB miss + * (fast TLB miss). TLB invalidation replaces entries with this value. + */ +static unsigned long pteaddr_invalid(unsigned long addr) +{ + return ((addr | 0xC0000000UL) >> PAGE_SHIFT) << 2; +} + /* * All entries common to a mm share an asid. To effectively flush these * entries, we just bump the asid. @@ -74,17 +79,14 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK; if (((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) && pid == mmu_pid) { - unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE + - ((PAGE_SIZE * cpuinfo.tlb_num_lines) * way) + - (addr & TLB_INDEX_MASK); - pr_debug("Flush entry by writing %#lx way=%dl pid=%ld\n", - vaddr, way, (pid_misc >> TLBMISC_PID_SHIFT)); + pr_debug("Flush entry by writing way=%dl pid=%ld\n", + way, (pid_misc >> TLBMISC_PID_SHIFT)); - WRCTL(CTL_PTEADDR, (vaddr >> 12) << 2); + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); tlbmisc = pid_misc | TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT)); + WRCTL(CTL_TLBACC, 0); } } @@ -128,18 +130,14 @@ static void flush_tlb_one(unsigned long addr) tlbmisc = RDCTL(CTL_TLBMISC); if ((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) { - unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE + - ((PAGE_SIZE * cpuinfo.tlb_num_lines) * way) + - (addr & TLB_INDEX_MASK); - - pr_debug("Flush entry by writing %#lx way=%dl pid=%ld\n", - vaddr, way, (pid_misc >> TLBMISC_PID_SHIFT)); + pr_debug("Flush entry by writing way=%dl pid=%ld\n", + way, (pid_misc >> TLBMISC_PID_SHIFT)); + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); tlbmisc = pid_misc | TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); - WRCTL(CTL_PTEADDR, (vaddr >> 12) << 2); WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT)); + WRCTL(CTL_TLBACC, 0); } } @@ -177,7 +175,7 @@ void dump_tlb_line(unsigned long line) tlbmisc = RDCTL(CTL_TLBMISC); tlbacc = RDCTL(CTL_TLBACC); - if ((tlbacc << PAGE_SHIFT) != (MAX_PHYS_ADDR & PAGE_MASK)) { + if ((tlbacc << PAGE_SHIFT) != 0) { pr_debug("-- way:%02x vpn:0x%08lx phys:0x%08lx pid:0x%02lx flags:%c%c%c%c%c\n", way, (pteaddr << (PAGE_SHIFT-2)), @@ -205,6 +203,7 @@ void dump_tlb(void) void flush_tlb_pid(unsigned long pid) { + unsigned long addr = 0; unsigned int line; unsigned int way; unsigned long org_misc, pid_misc; @@ -213,38 +212,35 @@ void flush_tlb_pid(unsigned long pid) get_misc_and_pid(&org_misc, &pid_misc); for (line = 0; line < cpuinfo.tlb_num_lines; line++) { - WRCTL(CTL_PTEADDR, line << 2); + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); for (way = 0; way < cpuinfo.tlb_num_ways; way++) { - unsigned long pteaddr; unsigned long tlbmisc; - unsigned long tlbacc; tlbmisc = pid_misc | TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); - pteaddr = RDCTL(CTL_PTEADDR); tlbmisc = RDCTL(CTL_TLBMISC); - tlbacc = RDCTL(CTL_TLBACC); if (((tlbmisc>>TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK) == pid) { tlbmisc = pid_misc | TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, - (MAX_PHYS_ADDR >> PAGE_SHIFT)); + WRCTL(CTL_TLBACC, 0); } } + addr += PAGE_SIZE; + WRCTL(CTL_TLBMISC, org_misc); } } void flush_tlb_all(void) { - int i; - unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE; + unsigned long addr = 0; + unsigned int line; unsigned int way; unsigned long org_misc, pid_misc, tlbmisc; @@ -254,14 +250,16 @@ void flush_tlb_all(void) /* Map each TLB entry to physcal address 0 with no-access and a bad ptbase */ - for (way = 0; way < cpuinfo.tlb_num_ways; way++) { - tlbmisc = pid_misc | (way << TLBMISC_WAY_SHIFT); - for (i = 0; i < cpuinfo.tlb_num_lines; i++) { - WRCTL(CTL_PTEADDR, ((vaddr) >> PAGE_SHIFT) << 2); + for (line = 0; line < cpuinfo.tlb_num_lines; line++) { + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); + + for (way = 0; way < cpuinfo.tlb_num_ways; way++) { + tlbmisc = pid_misc | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT)); - vaddr += 1UL << 12; + WRCTL(CTL_TLBACC, 0); } + + addr += PAGE_SIZE; } /* restore pid/way */ -- cgit v1.2.3-59-g8ed1b From 7d1730708b8aa345a9b2e9e1ffa2d9aa7019d4e2 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 12 Nov 2018 23:03:35 +0800 Subject: nios2: TLBMISC writes do not require PID bits to be set TLBMISC_RD does not use PID bits, and when setting invalid TLBs, the PID is not required because the address will not match. This is just a tidy up. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index eb568a919576..6e0fcaa0230a 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -72,7 +72,7 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) unsigned long tlbmisc; unsigned long pid; - tlbmisc = pid_misc | TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); pteaddr = RDCTL(CTL_PTEADDR); tlbmisc = RDCTL(CTL_TLBMISC); @@ -83,8 +83,7 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) way, (pid_misc >> TLBMISC_PID_SHIFT)); WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - tlbmisc = pid_misc | TLBMISC_WE | - (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); WRCTL(CTL_TLBACC, 0); } @@ -124,7 +123,7 @@ static void flush_tlb_one(unsigned long addr) unsigned long pteaddr; unsigned long tlbmisc; - tlbmisc = pid_misc | TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); pteaddr = RDCTL(CTL_PTEADDR); tlbmisc = RDCTL(CTL_TLBMISC); @@ -134,8 +133,7 @@ static void flush_tlb_one(unsigned long addr) way, (pid_misc >> TLBMISC_PID_SHIFT)); WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - tlbmisc = pid_misc | TLBMISC_WE | - (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); WRCTL(CTL_TLBACC, 0); } @@ -217,15 +215,13 @@ void flush_tlb_pid(unsigned long pid) for (way = 0; way < cpuinfo.tlb_num_ways; way++) { unsigned long tlbmisc; - tlbmisc = pid_misc | TLBMISC_RD | - (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); tlbmisc = RDCTL(CTL_TLBMISC); if (((tlbmisc>>TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK) == pid) { - tlbmisc = pid_misc | TLBMISC_WE | - (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); WRCTL(CTL_TLBACC, 0); } @@ -246,7 +242,6 @@ void flush_tlb_all(void) /* remember pid/way until we return */ get_misc_and_pid(&org_misc, &pid_misc); - pid_misc |= TLBMISC_WE; /* Map each TLB entry to physcal address 0 with no-access and a bad ptbase */ @@ -254,7 +249,7 @@ void flush_tlb_all(void) WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); for (way = 0; way < cpuinfo.tlb_num_ways; way++) { - tlbmisc = pid_misc | (way << TLBMISC_WAY_SHIFT); + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); WRCTL(CTL_TLBACC, 0); } -- cgit v1.2.3-59-g8ed1b From 58fd4766787eacd89a0859b0c4b90bd24258f971 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 5 Nov 2018 10:47:32 +0800 Subject: nios2: flush_tlb_pid can just restore TLBMISC once This matches the other functions in this file that use TLBMISC. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index 6e0fcaa0230a..b87387d664d1 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -228,9 +228,9 @@ void flush_tlb_pid(unsigned long pid) } addr += PAGE_SIZE; - - WRCTL(CTL_TLBMISC, org_misc); } + + WRCTL(CTL_TLBMISC, org_misc); } void flush_tlb_all(void) -- cgit v1.2.3-59-g8ed1b From e71c99fe8da9b8ab07b837a6f317c327d6a2b3df Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 5 Nov 2018 10:51:44 +0800 Subject: nios2: flush_tlb_mm flush only the pid Currently flush_tlb_mm flushes the entire TLB. Switch it to doing a PID aware flush. This also improves the readibility of flush_tlb_pid. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index b87387d664d1..e87304e1fc3f 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -39,18 +39,6 @@ static unsigned long pteaddr_invalid(unsigned long addr) return ((addr | 0xC0000000UL) >> PAGE_SHIFT) << 2; } -/* - * All entries common to a mm share an asid. To effectively flush these - * entries, we just bump the asid. - */ -void flush_tlb_mm(struct mm_struct *mm) -{ - if (current->mm == mm) - flush_tlb_all(); - else - memset(&mm->context, 0, sizeof(mm_context_t)); -} - /* * This one is only used for pages with the global bit set so we don't care * much about the ASID. @@ -233,6 +221,20 @@ void flush_tlb_pid(unsigned long pid) WRCTL(CTL_TLBMISC, org_misc); } +/* + * All entries common to a mm share an asid. To effectively flush these + * entries, we just bump the asid. + */ +void flush_tlb_mm(struct mm_struct *mm) +{ + if (current->mm == mm) { + unsigned long mmu_pid = get_pid_from_context(&mm->context); + flush_tlb_pid(mmu_pid); + } else { + memset(&mm->context, 0, sizeof(mm_context_t)); + } +} + void flush_tlb_all(void) { unsigned long addr = 0; -- cgit v1.2.3-59-g8ed1b From c6b1d363f883996e62bdb8655e472890d7dfd7ca Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 16 Nov 2018 07:04:05 +0800 Subject: nios2: improve readability of tlb functions Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 60 +++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 27 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index e87304e1fc3f..58c16d35f501 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -62,19 +62,20 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); + pteaddr = RDCTL(CTL_PTEADDR); + if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT)) + continue; + tlbmisc = RDCTL(CTL_TLBMISC); pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK; - if (((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) && - pid == mmu_pid) { - pr_debug("Flush entry by writing way=%dl pid=%ld\n", - way, (pid_misc >> TLBMISC_PID_SHIFT)); + if (pid != mmu_pid) + continue; - WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); - WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, 0); - } + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); + WRCTL(CTL_TLBMISC, tlbmisc); + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); + WRCTL(CTL_TLBACC, 0); } WRCTL(CTL_TLBMISC, org_misc); @@ -113,18 +114,18 @@ static void flush_tlb_one(unsigned long addr) tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); + pteaddr = RDCTL(CTL_PTEADDR); - tlbmisc = RDCTL(CTL_TLBMISC); + if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT)) + continue; - if ((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) { - pr_debug("Flush entry by writing way=%dl pid=%ld\n", - way, (pid_misc >> TLBMISC_PID_SHIFT)); + pr_debug("Flush entry by writing way=%dl pid=%ld\n", + way, (pid_misc >> TLBMISC_PID_SHIFT)); - WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); - WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, 0); - } + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); + WRCTL(CTL_TLBMISC, tlbmisc); + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); + WRCTL(CTL_TLBACC, 0); } WRCTL(CTL_TLBMISC, org_misc); @@ -187,7 +188,7 @@ void dump_tlb(void) dump_tlb_line(i); } -void flush_tlb_pid(unsigned long pid) +void flush_tlb_pid(unsigned long mmu_pid) { unsigned long addr = 0; unsigned int line; @@ -202,17 +203,18 @@ void flush_tlb_pid(unsigned long pid) for (way = 0; way < cpuinfo.tlb_num_ways; way++) { unsigned long tlbmisc; + unsigned long pid; tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); tlbmisc = RDCTL(CTL_TLBMISC); + pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK; + if (pid != mmu_pid) + continue; - if (((tlbmisc>>TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK) - == pid) { - tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); - WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_TLBACC, 0); - } + tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); + WRCTL(CTL_TLBMISC, tlbmisc); + WRCTL(CTL_TLBACC, 0); } addr += PAGE_SIZE; @@ -265,6 +267,10 @@ void flush_tlb_all(void) void set_mmu_pid(unsigned long pid) { - WRCTL(CTL_TLBMISC, (RDCTL(CTL_TLBMISC) & TLBMISC_WAY) | - ((pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT)); + unsigned long tlbmisc; + + tlbmisc = RDCTL(CTL_TLBMISC); + tlbmisc = (tlbmisc & TLBMISC_WAY); + tlbmisc |= (pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT; + WRCTL(CTL_TLBMISC, tlbmisc); } -- cgit v1.2.3-59-g8ed1b From 737a3fa20f2e195d94d9501ab5d76c29194d8176 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 7 Nov 2018 10:21:02 +0800 Subject: nios2: flush_tlb_all use TLBMISC way auto-increment feature Writes to TLBACC cause TLBMISC way to be incremented, which can be used to iterate over ways in a set, then wrap back to zero ready for the next set. This reduces register writes significantly. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index 58c16d35f501..2469f88ef7f3 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -242,21 +242,20 @@ void flush_tlb_all(void) unsigned long addr = 0; unsigned int line; unsigned int way; - unsigned long org_misc, pid_misc, tlbmisc; + unsigned long org_misc, pid_misc; /* remember pid/way until we return */ get_misc_and_pid(&org_misc, &pid_misc); + /* Start at way 0, way is auto-incremented after each TLBACC write */ + WRCTL(CTL_TLBMISC, TLBMISC_WE); + /* Map each TLB entry to physcal address 0 with no-access and a bad ptbase */ for (line = 0; line < cpuinfo.tlb_num_lines; line++) { WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - - for (way = 0; way < cpuinfo.tlb_num_ways; way++) { - tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); - WRCTL(CTL_TLBMISC, tlbmisc); + for (way = 0; way < cpuinfo.tlb_num_ways; way++) WRCTL(CTL_TLBACC, 0); - } addr += PAGE_SIZE; } -- cgit v1.2.3-59-g8ed1b From b6a10463438d8775aa6aa09ece46e8af14345712 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 7 Nov 2018 10:35:20 +0800 Subject: nios2: User address TLB flush break after finding the matching entry Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/mm/tlb.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index 2469f88ef7f3..2e49993d29ef 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -76,6 +76,11 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) WRCTL(CTL_TLBMISC, tlbmisc); WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); WRCTL(CTL_TLBACC, 0); + /* + * There should be only a single entry that maps a + * particular {address,pid} so break after a match. + */ + break; } WRCTL(CTL_TLBMISC, org_misc); -- cgit v1.2.3-59-g8ed1b From 3ac23944de570df7a6309425aeef063be38f37c4 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 7 Nov 2018 10:35:34 +0800 Subject: nios2: update_mmu_cache preload the TLB with the new PTE Rather than flush the TLB entry when installing a new PTE to allow the fast TLB reload to re-fill the TLB, just refill the TLB entry when removing the old one. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan --- arch/nios2/include/asm/tlbflush.h | 6 ++++++ arch/nios2/mm/cacheflush.c | 7 ++++--- arch/nios2/mm/tlb.c | 32 ++++++++++++++++++++++++++------ 3 files changed, 36 insertions(+), 9 deletions(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/include/asm/tlbflush.h b/arch/nios2/include/asm/tlbflush.h index 982a7ecf221f..b4bf487b9832 100644 --- a/arch/nios2/include/asm/tlbflush.h +++ b/arch/nios2/include/asm/tlbflush.h @@ -30,6 +30,9 @@ struct mm_struct; * - flush_tlb_page(vma, address) flushes a page * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages * - flush_tlb_kernel_page(address) flushes a kernel page + * + * - reload_tlb_page(vma, address, pte) flushes the TLB for address like + * flush_tlb_page, then replaces it with a TLB for pte. */ extern void flush_tlb_all(void); extern void flush_tlb_mm(struct mm_struct *mm); @@ -48,4 +51,7 @@ static inline void flush_tlb_kernel_page(unsigned long address) flush_tlb_kernel_range(address, address + PAGE_SIZE); } +extern void reload_tlb_page(struct vm_area_struct *vma, unsigned long addr, + pte_t pte); + #endif /* _ASM_NIOS2_TLBFLUSH_H */ diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index d58e7e80dc0d..65de1bd6a760 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -198,13 +198,14 @@ void flush_dcache_page(struct page *page) EXPORT_SYMBOL(flush_dcache_page); void update_mmu_cache(struct vm_area_struct *vma, - unsigned long address, pte_t *pte) + unsigned long address, pte_t *ptep) { - unsigned long pfn = pte_pfn(*pte); + pte_t pte = *ptep; + unsigned long pfn = pte_pfn(pte); struct page *page; struct address_space *mapping; - flush_tlb_page(vma, address); + reload_tlb_page(vma, address, pte); if (!pfn_valid(pfn)) return; diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index 2e49993d29ef..af8711885569 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -43,13 +43,11 @@ static unsigned long pteaddr_invalid(unsigned long addr) * This one is only used for pages with the global bit set so we don't care * much about the ASID. */ -void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) +static void replace_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, unsigned long tlbacc) { unsigned int way; unsigned long org_misc, pid_misc; - pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr); - /* remember pid/way until we return. */ get_misc_and_pid(&org_misc, &pid_misc); @@ -72,10 +70,11 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) if (pid != mmu_pid) continue; - tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); + tlbmisc = mmu_pid | TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); - WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); - WRCTL(CTL_TLBACC, 0); + if (tlbacc == 0) + WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); + WRCTL(CTL_TLBACC, tlbacc); /* * There should be only a single entry that maps a * particular {address,pid} so break after a match. @@ -86,6 +85,20 @@ void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) WRCTL(CTL_TLBMISC, org_misc); } +static void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) +{ + pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr); + + replace_tlb_one_pid(addr, mmu_pid, 0); +} + +static void reload_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, pte_t pte) +{ + pr_debug("Reload tlb-entry for vaddr=%#lx\n", addr); + + replace_tlb_one_pid(addr, mmu_pid, pte_val(pte)); +} + void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { @@ -97,6 +110,13 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, } } +void reload_tlb_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +{ + unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context); + + reload_tlb_one_pid(addr, mmu_pid, pte); +} + /* * This one is only used for pages with the global bit set so we don't care * much about the ASID. -- cgit v1.2.3-59-g8ed1b From 21e6bff5e0ef0033d776e64c40e6873d7c75e74b Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Thu, 7 Mar 2019 05:28:31 +0800 Subject: nios2: Fix update_mmu_cache preload the TLB with the new PTE There is a bug in the TLB preload caused by the pid not being shifted to the correct location in tlbmisc register. Signed-off-by: Nicholas Piggin Signed-off-by: Ley Foon Tan Tested-by: Guenter Roeck --- arch/nios2/mm/tlb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/nios2/mm/tlb.c') diff --git a/arch/nios2/mm/tlb.c b/arch/nios2/mm/tlb.c index af8711885569..7fea59e53f94 100644 --- a/arch/nios2/mm/tlb.c +++ b/arch/nios2/mm/tlb.c @@ -70,7 +70,8 @@ static void replace_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, unsig if (pid != mmu_pid) continue; - tlbmisc = mmu_pid | TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); + tlbmisc = (mmu_pid << TLBMISC_PID_SHIFT) | TLBMISC_WE | + (way << TLBMISC_WAY_SHIFT); WRCTL(CTL_TLBMISC, tlbmisc); if (tlbacc == 0) WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); -- cgit v1.2.3-59-g8ed1b