From 60b2249d45d44bd3494d55f5ea4bccd25c7f8281 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 17:07:06 +0900 Subject: sh: Move over SH-5 TLB and cache support code. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_64 | 7 +- arch/sh/mm/cache-sh5.c | 1032 ++++++++++++++++++++++++++++++++++++++++++++++++ arch/sh/mm/tlb-sh5.c | 166 ++++++++ arch/sh64/mm/cache.c | 1032 ------------------------------------------------ arch/sh64/mm/tlb.c | 166 -------- 5 files changed, 1203 insertions(+), 1200 deletions(-) create mode 100644 arch/sh/mm/cache-sh5.c create mode 100644 arch/sh/mm/tlb-sh5.c delete mode 100644 arch/sh64/mm/cache.c delete mode 100644 arch/sh64/mm/tlb.c (limited to 'arch') diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index 82fe9072fd5c..5ff83583593e 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -5,12 +5,15 @@ obj-y := init.o extable_64.o consistent.o mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o +mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o tlb-sh5.o + +ifndef CONFIG_CACHE_OFF +obj-y += cache-sh5.o +endif obj-y += $(mmu-y) obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_PMB) += pmb.o obj-$(CONFIG_NUMA) += numa.o EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c new file mode 100644 index 000000000000..421487cfff4c --- /dev/null +++ b/arch/sh/mm/cache-sh5.c @@ -0,0 +1,1032 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * arch/sh64/mm/cache.c + * + * Original version Copyright (C) 2000, 2001 Paolo Alberelli + * Second version Copyright (C) benedict.gaster@superh.com 2002 + * Third version Copyright Richard.Curnow@superh.com 2003 + * Hacks to third version Copyright (C) 2003 Paul Mundt + */ + +/****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for flush_itlb_range */ + +#include + +/* This function is in entry.S */ +extern unsigned long switch_and_save_asid(unsigned long new_asid); + +/* Wired TLB entry for the D-cache */ +static unsigned long long dtlb_cache_slot; + +/** + * sh64_cache_init() + * + * This is pretty much just a straightforward clone of the SH + * detect_cpu_and_cache_system(). + * + * This function is responsible for setting up all of the cache + * info dynamically as well as taking care of CPU probing and + * setting up the relevant subtype data. + * + * FIXME: For the time being, we only really support the SH5-101 + * out of the box, and don't support dynamic probing for things + * like the SH5-103 or even cut2 of the SH5-101. Implement this + * later! + */ +int __init sh64_cache_init(void) +{ + /* + * First, setup some sane values for the I-cache. + */ + cpu_data->icache.ways = 4; + cpu_data->icache.sets = 256; + cpu_data->icache.linesz = L1_CACHE_BYTES; + + /* + * FIXME: This can probably be cleaned up a bit as well.. for example, + * do we really need the way shift _and_ the way_step_shift ?? Judging + * by the existing code, I would guess no.. is there any valid reason + * why we need to be tracking this around? + */ + cpu_data->icache.way_shift = 13; + cpu_data->icache.entry_shift = 5; + cpu_data->icache.set_shift = 4; + cpu_data->icache.way_step_shift = 16; + cpu_data->icache.asid_shift = 2; + + /* + * way offset = cache size / associativity, so just don't factor in + * associativity in the first place.. + */ + cpu_data->icache.way_ofs = cpu_data->icache.sets * + cpu_data->icache.linesz; + + cpu_data->icache.asid_mask = 0x3fc; + cpu_data->icache.idx_mask = 0x1fe0; + cpu_data->icache.epn_mask = 0xffffe000; + cpu_data->icache.flags = 0; + + /* + * Next, setup some sane values for the D-cache. + * + * On the SH5, these are pretty consistent with the I-cache settings, + * so we just copy over the existing definitions.. these can be fixed + * up later, especially if we add runtime CPU probing. + * + * Though in the meantime it saves us from having to duplicate all of + * the above definitions.. + */ + cpu_data->dcache = cpu_data->icache; + + /* + * Setup any cache-related flags here + */ +#if defined(CONFIG_DCACHE_WRITE_THROUGH) + set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)); +#elif defined(CONFIG_DCACHE_WRITE_BACK) + set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags)); +#endif + + /* + * We also need to reserve a slot for the D-cache in the DTLB, so we + * do this now .. + */ + dtlb_cache_slot = sh64_get_wired_dtlb_entry(); + + return 0; +} + +#ifdef CONFIG_DCACHE_DISABLED +#define sh64_dcache_purge_all() do { } while (0) +#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0) +#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0) +#define sh64_dcache_purge_phy_page(paddr) do { } while (0) +#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0) +#define sh64_dcache_purge_kernel_range(start, end) do { } while (0) +#define sh64_dcache_wback_current_user_range(start, end) do { } while (0) +#endif + +/*##########################################################################*/ + +/* From here onwards, a rewrite of the implementation, + by Richard.Curnow@superh.com. + + The major changes in this compared to the old version are; + 1. use more selective purging through OCBP instead of using ALLOCO to purge + by natural replacement. This avoids purging out unrelated cache lines + that happen to be in the same set. + 2. exploit the APIs copy_user_page and clear_user_page better + 3. be more selective about I-cache purging, in particular use invalidate_all + more sparingly. + + */ + +/*########################################################################## + SUPPORT FUNCTIONS + ##########################################################################*/ + +/****************************************************************************/ +/* The following group of functions deal with mapping and unmapping a temporary + page into the DTLB slot that have been set aside for our exclusive use. */ +/* In order to accomplish this, we use the generic interface for adding and + removing a wired slot entry as defined in arch/sh64/mm/tlb.c */ +/****************************************************************************/ + +static unsigned long slot_own_flags; + +static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr) +{ + local_irq_save(slot_own_flags); + sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr); +} + +static inline void sh64_teardown_dtlb_cache_slot(void) +{ + sh64_teardown_tlb_slot(dtlb_cache_slot); + local_irq_restore(slot_own_flags); +} + +/****************************************************************************/ + +#ifndef CONFIG_ICACHE_DISABLED + +static void __inline__ sh64_icache_inv_all(void) +{ + unsigned long long addr, flag, data; + unsigned int flags; + + addr=ICCR0; + flag=ICCR0_ICI; + data=0; + + /* Make this a critical section for safety (probably not strictly necessary.) */ + local_irq_save(flags); + + /* Without %1 it gets unexplicably wrong */ + asm volatile("getcfg %3, 0, %0\n\t" + "or %0, %2, %0\n\t" + "putcfg %3, 0, %0\n\t" + "synci" + : "=&r" (data) + : "0" (data), "r" (flag), "r" (addr)); + + local_irq_restore(flags); +} + +static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end) +{ + /* Invalidate range of addresses [start,end] from the I-cache, where + * the addresses lie in the kernel superpage. */ + + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr) +{ + /* If we get called, we know that vma->vm_flags contains VM_EXEC. + Also, eaddr is page-aligned. */ + + unsigned long long addr, end_addr; + unsigned long flags = 0; + unsigned long running_asid, vma_asid; + addr = eaddr; + end_addr = addr + PAGE_SIZE; + + /* Check whether we can use the current ASID for the I-cache + invalidation. For example, if we're called via + access_process_vm->flush_cache_page->here, (e.g. when reading from + /proc), 'running_asid' will be that of the reader, not of the + victim. + + Also, note the risk that we might get pre-empted between the ASID + compare and blocking IRQs, and before we regain control, the + pid->ASID mapping changes. However, the whole cache will get + invalidated when the mapping is renewed, so the worst that can + happen is that the loop below ends up invalidating somebody else's + cache entries. + */ + + running_asid = get_asid(); + vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK); + if (running_asid != vma_asid) { + local_irq_save(flags); + switch_and_save_asid(vma_asid); + } + while (addr < end_addr) { + /* Worth unrolling a little */ + asm __volatile__("icbi %0, 0" : : "r" (addr)); + asm __volatile__("icbi %0, 32" : : "r" (addr)); + asm __volatile__("icbi %0, 64" : : "r" (addr)); + asm __volatile__("icbi %0, 96" : : "r" (addr)); + addr += 128; + } + if (running_asid != vma_asid) { + switch_and_save_asid(running_asid); + local_irq_restore(flags); + } +} + +/****************************************************************************/ + +static void sh64_icache_inv_user_page_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* Used for invalidating big chunks of I-cache, i.e. assume the range + is whole pages. If 'start' or 'end' is not page aligned, the code + is conservative and invalidates to the ends of the enclosing pages. + This is functionally OK, just a performance loss. */ + + /* See the comments below in sh64_dcache_purge_user_range() regarding + the choice of algorithm. However, for the I-cache option (2) isn't + available because there are no physical tags so aliases can't be + resolved. The icbi instruction has to be used through the user + mapping. Because icbi is cheaper than ocbp on a cache hit, it + would be cheaper to use the selective code for a large range than is + possible with the D-cache. Just assume 64 for now as a working + figure. + */ + + int n_pages; + + if (!mm) return; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64) { + sh64_icache_inv_all(); + } else { + unsigned long aligned_start; + unsigned long eaddr; + unsigned long after_last_page_start; + unsigned long mm_asid, current_asid; + unsigned long long flags = 0ULL; + + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = get_asid(); + + if (mm_asid != current_asid) { + /* Switch ASID and run the invalidate loop under cli */ + local_irq_save(flags); + switch_and_save_asid(mm_asid); + } + + aligned_start = start & PAGE_MASK; + after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK); + + while (aligned_start < after_last_page_start) { + struct vm_area_struct *vma; + unsigned long vma_end; + vma = find_vma(mm, aligned_start); + if (!vma || (aligned_start <= vma->vm_end)) { + /* Avoid getting stuck in an error condition */ + aligned_start += PAGE_SIZE; + continue; + } + vma_end = vma->vm_end; + if (vma->vm_flags & VM_EXEC) { + /* Executable */ + eaddr = aligned_start; + while (eaddr < vma_end) { + sh64_icache_inv_user_page(vma, eaddr); + eaddr += PAGE_SIZE; + } + } + aligned_start = vma->vm_end; /* Skip to start of next region */ + } + if (mm_asid != current_asid) { + switch_and_save_asid(current_asid); + local_irq_restore(flags); + } + } +} + +static void sh64_icache_inv_user_small_range(struct mm_struct *mm, + unsigned long start, int len) +{ + + /* Invalidate a small range of user context I-cache, not necessarily + page (or even cache-line) aligned. */ + + unsigned long long eaddr = start; + unsigned long long eaddr_end = start + len; + unsigned long current_asid, mm_asid; + unsigned long long flags; + unsigned long long epage_start; + + /* Since this is used inside ptrace, the ASID in the mm context + typically won't match current_asid. We'll have to switch ASID to do + this. For safety, and given that the range will be small, do all + this under cli. + + Note, there is a hazard that the ASID in mm->context is no longer + actually associated with mm, i.e. if the mm->context has started a + new cycle since mm was last active. However, this is just a + performance issue: all that happens is that we invalidate lines + belonging to another mm, so the owning process has to refill them + when that mm goes live again. mm itself can't have any cache + entries because there will have been a flush_cache_all when the new + mm->context cycle started. */ + + /* Align to start of cache line. Otherwise, suppose len==8 and start + was at 32N+28 : the last 4 bytes wouldn't get invalidated. */ + eaddr = start & L1_CACHE_ALIGN_MASK; + eaddr_end = start + len; + + local_irq_save(flags); + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = switch_and_save_asid(mm_asid); + + epage_start = eaddr & PAGE_MASK; + + while (eaddr < eaddr_end) + { + asm __volatile__("icbi %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + switch_and_save_asid(current_asid); + local_irq_restore(flags); +} + +static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end) +{ + /* The icbi instruction never raises ITLBMISS. i.e. if there's not a + cache hit on the virtual tag the instruction ends there, without a + TLB lookup. */ + + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just invalidate over the range using the natural addresses. TLB + miss handling will be OK (TBC). Since it's for the current process, + either we're already in the right ASID context, or the ASIDs have + been recycled since we were last active in which case we might just + invalidate another processes I-cache entries : no worries, just a + performance drop for him. */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + asm __volatile__ ("nop"); + asm __volatile__ ("nop"); + addr += L1_CACHE_BYTES; + } +} + +#endif /* !CONFIG_ICACHE_DISABLED */ + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +/* Buffer used as the target of alloco instructions to purge data from cache + sets by natural eviction. -- RPC */ +#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4) +static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, }; + +/****************************************************************************/ + +static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets) +{ + /* Purge all ways in a particular block of sets, specified by the base + set number and number of sets. Can handle wrap-around, if that's + needed. */ + + int dummy_buffer_base_set; + unsigned long long eaddr, eaddr0, eaddr1; + int j; + int set_offset; + + dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift; + set_offset = sets_to_purge_base - dummy_buffer_base_set; + + for (j=0; jdcache.sets - 1); + eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift); + + /* Do one alloco which hits the required set per cache way. For + write-back mode, this will purge the #ways resident lines. There's + little point unrolling this loop because the allocos stall more if + they're too close together. */ + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddrdcache.way_ofs) { + asm __volatile__ ("alloco %0, 0" : : "r" (eaddr)); + asm __volatile__ ("synco"); /* TAKum03020 */ + } + + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddrdcache.way_ofs) { + /* Load from each address. Required because alloco is a NOP if + the cache is write-through. Write-through is a config option. */ + if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags))) + *(volatile unsigned char *)(int)eaddr; + } + } + + /* Don't use OCBI to invalidate the lines. That costs cycles directly. + If the dummy block is just left resident, it will naturally get + evicted as required. */ + + return; +} + +/****************************************************************************/ + +static void sh64_dcache_purge_all(void) +{ + /* Purge the entire contents of the dcache. The most efficient way to + achieve this is to use alloco instructions on a region of unused + memory equal in size to the cache, thereby causing the current + contents to be discarded by natural eviction. The alternative, + namely reading every tag, setting up a mapping for the corresponding + page and doing an OCBP for the line, would be much more expensive. + */ + + sh64_dcache_purge_sets(0, cpu_data->dcache.sets); + + return; + +} + +/****************************************************************************/ + +static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end) +{ + /* Purge the range of addresses [start,end] from the D-cache. The + addresses lie in the superpage mapping. There's no harm if we + overpurge at either end - just a small performance loss. */ + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("ocbp %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } + return; +} + +/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for + anything else in the kernel */ +#define MAGIC_PAGE0_START 0xffffffffec000000ULL + +static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr) +{ + /* Purge the physical page 'paddr' from the cache. It's known that any + cache lines requiring attention have the same page colour as the the + address 'eaddr'. + + This relies on the fact that the D-cache matches on physical tags + when no virtual tag matches. So we create an alias for the original + page and purge through that. (Alternatively, we could have done + this by switching ASID to match the original mapping and purged + through that, but that involves ASID switching cost + probably a + TLBMISS + refill anyway.) + */ + + unsigned long long magic_page_start; + unsigned long long magic_eaddr, magic_eaddr_end; + + magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK); + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr); + + magic_eaddr = magic_page_start; + magic_eaddr_end = magic_eaddr + PAGE_SIZE; + while (magic_eaddr < magic_eaddr_end) { + /* Little point in unrolling this loop - the OCBPs are blocking + and won't go any quicker (i.e. the loop overhead is parallel + to part of the OCBP execution.) */ + asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr)); + magic_eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); +} + +/****************************************************************************/ + +static void sh64_dcache_purge_phy_page(unsigned long paddr) +{ + /* Pure a page given its physical start address, by creating a + temporary 1 page mapping and purging across that. Even if we know + the virtual address (& vma or mm) of the page, the method here is + more elegant because it avoids issues of coping with page faults on + the purge instructions (i.e. no special-case code required in the + critical path in the TLB miss handling). */ + + unsigned long long eaddr_start, eaddr, eaddr_end; + int i; + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + eaddr_start = MAGIC_PAGE0_START; + for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) { + sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr); + + eaddr = eaddr_start; + eaddr_end = eaddr + PAGE_SIZE; + while (eaddr < eaddr_end) { + asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); + eaddr_start += PAGE_SIZE; + } +} + +static void sh64_dcache_purge_user_pages(struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + spinlock_t *ptl; + unsigned long paddr; + + if (!mm) + return; /* No way to find physical address of page */ + + pgd = pgd_offset(mm, addr); + if (pgd_bad(*pgd)) + return; + + pmd = pmd_offset(pgd, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return; + + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + do { + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + continue; + paddr = pte_val(entry) & PAGE_MASK; + sh64_dcache_purge_coloured_phy_page(paddr, addr); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(pte - 1, ptl); +} +/****************************************************************************/ + +static void sh64_dcache_purge_user_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* There are at least 5 choices for the implementation of this, with + pros (+), cons(-), comments(*): + + 1. ocbp each line in the range through the original user's ASID + + no lines spuriously evicted + - tlbmiss handling (must either handle faults on demand => extra + special-case code in tlbmiss critical path), or map the page in + advance (=> flush_tlb_range in advance to avoid multiple hits) + - ASID switching + - expensive for large ranges + + 2. temporarily map each page in the range to a special effective + address and ocbp through the temporary mapping; relies on the + fact that SH-5 OCB* always do TLB lookup and match on ptags (they + never look at the etags) + + no spurious evictions + - expensive for large ranges + * surely cheaper than (1) + + 3. walk all the lines in the cache, check the tags, if a match + occurs create a page mapping to ocbp the line through + + no spurious evictions + - tag inspection overhead + - (especially for small ranges) + - potential cost of setting up/tearing down page mapping for + every line that matches the range + * cost partly independent of range size + + 4. walk all the lines in the cache, check the tags, if a match + occurs use 4 * alloco to purge the line (+3 other probably + innocent victims) by natural eviction + + no tlb mapping overheads + - spurious evictions + - tag inspection overhead + + 5. implement like flush_cache_all + + no tag inspection overhead + - spurious evictions + - bad for small ranges + + (1) can be ruled out as more expensive than (2). (2) appears best + for small ranges. The choice between (3), (4) and (5) for large + ranges and the range size for the large/small boundary need + benchmarking to determine. + + For now use approach (2) for small ranges and (5) for large ones. + + */ + + int n_pages; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) { +#if 1 + sh64_dcache_purge_all(); +#else + unsigned long long set, way; + unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + for (set = 0; set < cpu_data->dcache.sets; set++) { + unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift); + for (way = 0; way < cpu_data->dcache.ways; way++) { + unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift); + unsigned long long tag0; + unsigned long line_valid; + + asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr)); + line_valid = tag0 & SH_CACHE_VALID; + if (line_valid) { + unsigned long cache_asid; + unsigned long epn; + + cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift; + /* The next line needs some + explanation. The virtual tags + encode bits [31:13] of the virtual + address, bit [12] of the 'tag' being + implied by the cache set index. */ + epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift); + + if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) { + /* TODO : could optimise this + call by batching multiple + adjacent sets together. */ + sh64_dcache_purge_sets(set, 1); + break; /* Don't waste time inspecting other ways for this set */ + } + } + } + } +#endif + } else { + /* Small range, covered by a single page table page */ + start &= PAGE_MASK; /* should already be so */ + end = PAGE_ALIGN(end); /* should already be so */ + sh64_dcache_purge_user_pages(mm, start, end); + } + return; +} + +static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end) +{ + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just wback over the range using the natural addresses. TLB miss + handling will be OK (TBC) : the range has just been written to by + the signal frame setup code, so the PTEs must exist. + + Note, if we have CONFIG_PREEMPT and get preempted inside this loop, + it doesn't matter, even if the pid->ASID mapping changes whilst + we're away. In that case the cache will have been flushed when the + mapping was renewed. So the writebacks below will be nugatory (and + we'll doubtless have to fault the TLB entry/ies in again with the + new ASID), but it's a rare case. + */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("ocbwb %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +/****************************************************************************/ + +/* These *MUST* lie in an area of virtual address space that's otherwise unused. */ +#define UNIQUE_EADDR_START 0xe0000000UL +#define UNIQUE_EADDR_END 0xe8000000UL + +static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr) +{ + /* Given a physical address paddr, and a user virtual address + user_eaddr which will eventually be mapped to it, create a one-off + kernel-private eaddr mapped to the same paddr. This is used for + creating special destination pages for copy_user_page and + clear_user_page */ + + static unsigned long current_pointer = UNIQUE_EADDR_START; + unsigned long coloured_pointer; + + if (current_pointer == UNIQUE_EADDR_END) { + sh64_dcache_purge_all(); + current_pointer = UNIQUE_EADDR_START; + } + + coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK); + sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr); + + current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS); + + return coloured_pointer; +} + +/****************************************************************************/ + +static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing cache entries of the wrong colour. These are + present quite often, if the kernel has recently used the page + internally, then given it up, then it's been allocated to the user. + */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_copy(from, coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +static void sh64_clear_user_page_coloured(void *to, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing kernel-originated lines of the wrong colour (as + above) */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_clear(coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +/*########################################################################## + EXTERNALLY CALLABLE API. + ##########################################################################*/ + +/* These functions are described in Documentation/cachetlb.txt. + Each one of these functions varies in behaviour depending on whether the + I-cache and/or D-cache are configured out. + + Note that the Linux term 'flush' corresponds to what is termed 'purge' in + the sh/sh64 jargon for the D-cache, i.e. write back dirty data then + invalidate the cache lines, and 'invalidate' for the I-cache. + */ + +#undef FLUSH_TRACE + +void flush_cache_all(void) +{ + /* Invalidate the entire contents of both caches, after writing back to + memory any dirty data from the D-cache. */ + sh64_dcache_purge_all(); + sh64_icache_inv_all(); +} + +/****************************************************************************/ + +void flush_cache_mm(struct mm_struct *mm) +{ + /* Invalidate an entire user-address space from both caches, after + writing back dirty data (e.g. for shared mmap etc). */ + + /* This could be coded selectively by inspecting all the tags then + doing 4*alloco on any set containing a match (as for + flush_cache_range), but fork/exit/execve (where this is called from) + are expensive anyway. */ + + /* Have to do a purge here, despite the comments re I-cache below. + There could be odd-coloured dirty data associated with the mm still + in the cache - if this gets written out through natural eviction + after the kernel has reused the page there will be chaos. + */ + + sh64_dcache_purge_all(); + + /* The mm being torn down won't ever be active again, so any Icache + lines tagged with its ASID won't be visible for the rest of the + lifetime of this ASID cycle. Before the ASID gets reused, there + will be a flush_cache_all. Hence we don't need to touch the + I-cache. This is similar to the lack of action needed in + flush_tlb_mm - see fault.c. */ +} + +/****************************************************************************/ + +void flush_cache_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + /* Invalidate (from both caches) the range [start,end) of virtual + addresses from the user address space specified by mm, after writing + back any dirty data. + + Note, 'end' is 1 byte beyond the end of the range to flush. */ + + sh64_dcache_purge_user_range(mm, start, end); + sh64_icache_inv_user_page_range(mm, start, end); +} + +/****************************************************************************/ + +void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn) +{ + /* Invalidate any entries in either cache for the vma within the user + address space vma->vm_mm for the page starting at virtual address + 'eaddr'. This seems to be used primarily in breaking COW. Note, + the I-cache must be searched too in case the page in question is + both writable and being executed from (e.g. stack trampolines.) + + Note, this is called with pte lock held. + */ + + sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_page(vma, eaddr); + } +} + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +void copy_user_page(void *to, void *from, unsigned long address, struct page *page) +{ + /* 'from' and 'to' are kernel virtual addresses (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the copy 'to' will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. Note however : the call to flush_page_to_ram in + (generic)/mm/memory.c:(break_cow) undoes all this good work in that one + very important case! + + TBD : can we guarantee that on every call, any cache entries for + 'from' are in the same colour sets as 'address' also? i.e. is this + always used just to deal with COW? (I suspect not). */ + + /* There are two possibilities here for when the page 'from' was last accessed: + * by the kernel : this is OK, no purge required. + * by the/a user (e.g. for break_COW) : need to purge. + + If the potential user mapping at 'address' is the same colour as + 'from' there is no need to purge any cache lines from the 'from' + page mapped into cache sets of colour 'address'. (The copy will be + accessing the page through 'from'). + */ + + if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) { + sh64_dcache_purge_coloured_phy_page(__pa(from), address); + } + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_copy(from, to); + } else { + sh64_copy_user_page_coloured(to, from, address); + } + + /* Note, don't need to flush 'from' page from the cache again - it's + done anyway by the generic code */ +} + +void clear_user_page(void *to, unsigned long address, struct page *page) +{ + /* 'to' is a kernel virtual address (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the 'to' page will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. + */ + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_clear(to); + } else { + sh64_clear_user_page_coloured(to, address); + } +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +void flush_dcache_page(struct page *page) +{ + sh64_dcache_purge_phy_page(page_to_phys(page)); + wmb(); +} + +/****************************************************************************/ + +void flush_icache_range(unsigned long start, unsigned long end) +{ + /* Flush the range [start,end] of kernel virtual adddress space from + the I-cache. The corresponding range must be purged from the + D-cache also because the SH-5 doesn't have cache snooping between + the caches. The addresses will be visible through the superpage + mapping, therefore it's guaranteed that there no cache entries for + the range in cache sets of the wrong colour. + + Primarily used for cohering the I-cache after a module has + been loaded. */ + + /* We also make sure to purge the same range from the D-cache since + flush_page_to_ram() won't be doing this for us! */ + + sh64_dcache_purge_kernel_range(start, end); + wmb(); + sh64_icache_inv_kernel_range(start, end); +} + +/****************************************************************************/ + +void flush_icache_user_range(struct vm_area_struct *vma, + struct page *page, unsigned long addr, int len) +{ + /* Flush the range of user (defined by vma->vm_mm) address space + starting at 'addr' for 'len' bytes from the cache. The range does + not straddle a page boundary, the unique physical page containing + the range is 'page'. This seems to be used mainly for invalidating + an address range following a poke into the program text through the + ptrace() call from another process (e.g. for BRK instruction + insertion). */ + + sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr); + mb(); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_small_range(vma->vm_mm, addr, len); + } +} + +/*########################################################################## + ARCH/SH64 PRIVATE CALLABLE API. + ##########################################################################*/ + +void flush_cache_sigtramp(unsigned long start, unsigned long end) +{ + /* For the address range [start,end), write back the data from the + D-cache and invalidate the corresponding region of the I-cache for + the current process. Used to flush signal trampolines on the stack + to make them executable. */ + + sh64_dcache_wback_current_user_range(start, end); + wmb(); + sh64_icache_inv_current_user_range(start, end); +} + diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c new file mode 100644 index 000000000000..d517e7d70340 --- /dev/null +++ b/arch/sh/mm/tlb-sh5.c @@ -0,0 +1,166 @@ +/* + * arch/sh64/mm/tlb.c + * + * Copyright (C) 2003 Paul Mundt + * Copyright (C) 2003 Richard Curnow + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + */ +#include +#include +#include +#include +#include + +/** + * sh64_tlb_init + * + * Perform initial setup for the DTLB and ITLB. + */ +int __init sh64_tlb_init(void) +{ + /* Assign some sane DTLB defaults */ + cpu_data->dtlb.entries = 64; + cpu_data->dtlb.step = 0x10; + + cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step; + cpu_data->dtlb.next = cpu_data->dtlb.first; + + cpu_data->dtlb.last = DTLB_FIXED | + ((cpu_data->dtlb.entries - 1) * + cpu_data->dtlb.step); + + /* And again for the ITLB */ + cpu_data->itlb.entries = 64; + cpu_data->itlb.step = 0x10; + + cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step; + cpu_data->itlb.next = cpu_data->itlb.first; + cpu_data->itlb.last = ITLB_FIXED | + ((cpu_data->itlb.entries - 1) * + cpu_data->itlb.step); + + return 0; +} + +/** + * sh64_next_free_dtlb_entry + * + * Find the next available DTLB entry + */ +unsigned long long sh64_next_free_dtlb_entry(void) +{ + return cpu_data->dtlb.next; +} + +/** + * sh64_get_wired_dtlb_entry + * + * Allocate a wired (locked-in) entry in the DTLB + */ +unsigned long long sh64_get_wired_dtlb_entry(void) +{ + unsigned long long entry = sh64_next_free_dtlb_entry(); + + cpu_data->dtlb.first += cpu_data->dtlb.step; + cpu_data->dtlb.next += cpu_data->dtlb.step; + + return entry; +} + +/** + * sh64_put_wired_dtlb_entry + * + * @entry: Address of TLB slot. + * + * Free a wired (locked-in) entry in the DTLB. + * + * Works like a stack, last one to allocate must be first one to free. + */ +int sh64_put_wired_dtlb_entry(unsigned long long entry) +{ + __flush_tlb_slot(entry); + + /* + * We don't do any particularly useful tracking of wired entries, + * so this approach works like a stack .. last one to be allocated + * has to be the first one to be freed. + * + * We could potentially load wired entries into a list and work on + * rebalancing the list periodically (which also entails moving the + * contents of a TLB entry) .. though I have a feeling that this is + * more trouble than it's worth. + */ + + /* + * Entry must be valid .. we don't want any ITLB addresses! + */ + if (entry <= DTLB_FIXED) + return -EINVAL; + + /* + * Next, check if we're within range to be freed. (ie, must be the + * entry beneath the first 'free' entry! + */ + if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step)) + return -EINVAL; + + /* If we are, then bring this entry back into the list */ + cpu_data->dtlb.first -= cpu_data->dtlb.step; + cpu_data->dtlb.next = entry; + + return 0; +} + +/** + * sh64_setup_tlb_slot + * + * @config_addr: Address of TLB slot. + * @eaddr: Virtual address. + * @asid: Address Space Identifier. + * @paddr: Physical address. + * + * Load up a virtual<->physical translation for @eaddr<->@paddr in the + * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry). + */ +inline void sh64_setup_tlb_slot(unsigned long long config_addr, + unsigned long eaddr, + unsigned long asid, + unsigned long paddr) +{ + unsigned long long pteh, ptel; + + /* Sign extension */ +#if (NEFF == 32) + pteh = (unsigned long long)(signed long long)(signed long) eaddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + pteh &= PAGE_MASK; + pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID; +#if (NEFF == 32) + ptel = (unsigned long long)(signed long long)(signed long) paddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + ptel &= PAGE_MASK; + ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE); + + asm volatile("putcfg %0, 1, %1\n\t" + "putcfg %0, 0, %2\n" + : : "r" (config_addr), "r" (ptel), "r" (pteh)); +} + +/** + * sh64_teardown_tlb_slot + * + * @config_addr: Address of TLB slot. + * + * Teardown any existing mapping in the TLB slot @config_addr. + */ +inline void sh64_teardown_tlb_slot(unsigned long long config_addr) + __attribute__ ((alias("__flush_tlb_slot"))); + diff --git a/arch/sh64/mm/cache.c b/arch/sh64/mm/cache.c deleted file mode 100644 index 421487cfff4c..000000000000 --- a/arch/sh64/mm/cache.c +++ /dev/null @@ -1,1032 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * arch/sh64/mm/cache.c - * - * Original version Copyright (C) 2000, 2001 Paolo Alberelli - * Second version Copyright (C) benedict.gaster@superh.com 2002 - * Third version Copyright Richard.Curnow@superh.com 2003 - * Hacks to third version Copyright (C) 2003 Paul Mundt - */ - -/****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* for flush_itlb_range */ - -#include - -/* This function is in entry.S */ -extern unsigned long switch_and_save_asid(unsigned long new_asid); - -/* Wired TLB entry for the D-cache */ -static unsigned long long dtlb_cache_slot; - -/** - * sh64_cache_init() - * - * This is pretty much just a straightforward clone of the SH - * detect_cpu_and_cache_system(). - * - * This function is responsible for setting up all of the cache - * info dynamically as well as taking care of CPU probing and - * setting up the relevant subtype data. - * - * FIXME: For the time being, we only really support the SH5-101 - * out of the box, and don't support dynamic probing for things - * like the SH5-103 or even cut2 of the SH5-101. Implement this - * later! - */ -int __init sh64_cache_init(void) -{ - /* - * First, setup some sane values for the I-cache. - */ - cpu_data->icache.ways = 4; - cpu_data->icache.sets = 256; - cpu_data->icache.linesz = L1_CACHE_BYTES; - - /* - * FIXME: This can probably be cleaned up a bit as well.. for example, - * do we really need the way shift _and_ the way_step_shift ?? Judging - * by the existing code, I would guess no.. is there any valid reason - * why we need to be tracking this around? - */ - cpu_data->icache.way_shift = 13; - cpu_data->icache.entry_shift = 5; - cpu_data->icache.set_shift = 4; - cpu_data->icache.way_step_shift = 16; - cpu_data->icache.asid_shift = 2; - - /* - * way offset = cache size / associativity, so just don't factor in - * associativity in the first place.. - */ - cpu_data->icache.way_ofs = cpu_data->icache.sets * - cpu_data->icache.linesz; - - cpu_data->icache.asid_mask = 0x3fc; - cpu_data->icache.idx_mask = 0x1fe0; - cpu_data->icache.epn_mask = 0xffffe000; - cpu_data->icache.flags = 0; - - /* - * Next, setup some sane values for the D-cache. - * - * On the SH5, these are pretty consistent with the I-cache settings, - * so we just copy over the existing definitions.. these can be fixed - * up later, especially if we add runtime CPU probing. - * - * Though in the meantime it saves us from having to duplicate all of - * the above definitions.. - */ - cpu_data->dcache = cpu_data->icache; - - /* - * Setup any cache-related flags here - */ -#if defined(CONFIG_DCACHE_WRITE_THROUGH) - set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)); -#elif defined(CONFIG_DCACHE_WRITE_BACK) - set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags)); -#endif - - /* - * We also need to reserve a slot for the D-cache in the DTLB, so we - * do this now .. - */ - dtlb_cache_slot = sh64_get_wired_dtlb_entry(); - - return 0; -} - -#ifdef CONFIG_DCACHE_DISABLED -#define sh64_dcache_purge_all() do { } while (0) -#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0) -#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0) -#define sh64_dcache_purge_phy_page(paddr) do { } while (0) -#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0) -#define sh64_dcache_purge_kernel_range(start, end) do { } while (0) -#define sh64_dcache_wback_current_user_range(start, end) do { } while (0) -#endif - -/*##########################################################################*/ - -/* From here onwards, a rewrite of the implementation, - by Richard.Curnow@superh.com. - - The major changes in this compared to the old version are; - 1. use more selective purging through OCBP instead of using ALLOCO to purge - by natural replacement. This avoids purging out unrelated cache lines - that happen to be in the same set. - 2. exploit the APIs copy_user_page and clear_user_page better - 3. be more selective about I-cache purging, in particular use invalidate_all - more sparingly. - - */ - -/*########################################################################## - SUPPORT FUNCTIONS - ##########################################################################*/ - -/****************************************************************************/ -/* The following group of functions deal with mapping and unmapping a temporary - page into the DTLB slot that have been set aside for our exclusive use. */ -/* In order to accomplish this, we use the generic interface for adding and - removing a wired slot entry as defined in arch/sh64/mm/tlb.c */ -/****************************************************************************/ - -static unsigned long slot_own_flags; - -static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr) -{ - local_irq_save(slot_own_flags); - sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr); -} - -static inline void sh64_teardown_dtlb_cache_slot(void) -{ - sh64_teardown_tlb_slot(dtlb_cache_slot); - local_irq_restore(slot_own_flags); -} - -/****************************************************************************/ - -#ifndef CONFIG_ICACHE_DISABLED - -static void __inline__ sh64_icache_inv_all(void) -{ - unsigned long long addr, flag, data; - unsigned int flags; - - addr=ICCR0; - flag=ICCR0_ICI; - data=0; - - /* Make this a critical section for safety (probably not strictly necessary.) */ - local_irq_save(flags); - - /* Without %1 it gets unexplicably wrong */ - asm volatile("getcfg %3, 0, %0\n\t" - "or %0, %2, %0\n\t" - "putcfg %3, 0, %0\n\t" - "synci" - : "=&r" (data) - : "0" (data), "r" (flag), "r" (addr)); - - local_irq_restore(flags); -} - -static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end) -{ - /* Invalidate range of addresses [start,end] from the I-cache, where - * the addresses lie in the kernel superpage. */ - - unsigned long long ullend, addr, aligned_start; -#if (NEFF == 32) - aligned_start = (unsigned long long)(signed long long)(signed long) start; -#else -#error "NEFF != 32" -#endif - aligned_start &= L1_CACHE_ALIGN_MASK; - addr = aligned_start; -#if (NEFF == 32) - ullend = (unsigned long long) (signed long long) (signed long) end; -#else -#error "NEFF != 32" -#endif - while (addr <= ullend) { - asm __volatile__ ("icbi %0, 0" : : "r" (addr)); - addr += L1_CACHE_BYTES; - } -} - -static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr) -{ - /* If we get called, we know that vma->vm_flags contains VM_EXEC. - Also, eaddr is page-aligned. */ - - unsigned long long addr, end_addr; - unsigned long flags = 0; - unsigned long running_asid, vma_asid; - addr = eaddr; - end_addr = addr + PAGE_SIZE; - - /* Check whether we can use the current ASID for the I-cache - invalidation. For example, if we're called via - access_process_vm->flush_cache_page->here, (e.g. when reading from - /proc), 'running_asid' will be that of the reader, not of the - victim. - - Also, note the risk that we might get pre-empted between the ASID - compare and blocking IRQs, and before we regain control, the - pid->ASID mapping changes. However, the whole cache will get - invalidated when the mapping is renewed, so the worst that can - happen is that the loop below ends up invalidating somebody else's - cache entries. - */ - - running_asid = get_asid(); - vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK); - if (running_asid != vma_asid) { - local_irq_save(flags); - switch_and_save_asid(vma_asid); - } - while (addr < end_addr) { - /* Worth unrolling a little */ - asm __volatile__("icbi %0, 0" : : "r" (addr)); - asm __volatile__("icbi %0, 32" : : "r" (addr)); - asm __volatile__("icbi %0, 64" : : "r" (addr)); - asm __volatile__("icbi %0, 96" : : "r" (addr)); - addr += 128; - } - if (running_asid != vma_asid) { - switch_and_save_asid(running_asid); - local_irq_restore(flags); - } -} - -/****************************************************************************/ - -static void sh64_icache_inv_user_page_range(struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - /* Used for invalidating big chunks of I-cache, i.e. assume the range - is whole pages. If 'start' or 'end' is not page aligned, the code - is conservative and invalidates to the ends of the enclosing pages. - This is functionally OK, just a performance loss. */ - - /* See the comments below in sh64_dcache_purge_user_range() regarding - the choice of algorithm. However, for the I-cache option (2) isn't - available because there are no physical tags so aliases can't be - resolved. The icbi instruction has to be used through the user - mapping. Because icbi is cheaper than ocbp on a cache hit, it - would be cheaper to use the selective code for a large range than is - possible with the D-cache. Just assume 64 for now as a working - figure. - */ - - int n_pages; - - if (!mm) return; - - n_pages = ((end - start) >> PAGE_SHIFT); - if (n_pages >= 64) { - sh64_icache_inv_all(); - } else { - unsigned long aligned_start; - unsigned long eaddr; - unsigned long after_last_page_start; - unsigned long mm_asid, current_asid; - unsigned long long flags = 0ULL; - - mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; - current_asid = get_asid(); - - if (mm_asid != current_asid) { - /* Switch ASID and run the invalidate loop under cli */ - local_irq_save(flags); - switch_and_save_asid(mm_asid); - } - - aligned_start = start & PAGE_MASK; - after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK); - - while (aligned_start < after_last_page_start) { - struct vm_area_struct *vma; - unsigned long vma_end; - vma = find_vma(mm, aligned_start); - if (!vma || (aligned_start <= vma->vm_end)) { - /* Avoid getting stuck in an error condition */ - aligned_start += PAGE_SIZE; - continue; - } - vma_end = vma->vm_end; - if (vma->vm_flags & VM_EXEC) { - /* Executable */ - eaddr = aligned_start; - while (eaddr < vma_end) { - sh64_icache_inv_user_page(vma, eaddr); - eaddr += PAGE_SIZE; - } - } - aligned_start = vma->vm_end; /* Skip to start of next region */ - } - if (mm_asid != current_asid) { - switch_and_save_asid(current_asid); - local_irq_restore(flags); - } - } -} - -static void sh64_icache_inv_user_small_range(struct mm_struct *mm, - unsigned long start, int len) -{ - - /* Invalidate a small range of user context I-cache, not necessarily - page (or even cache-line) aligned. */ - - unsigned long long eaddr = start; - unsigned long long eaddr_end = start + len; - unsigned long current_asid, mm_asid; - unsigned long long flags; - unsigned long long epage_start; - - /* Since this is used inside ptrace, the ASID in the mm context - typically won't match current_asid. We'll have to switch ASID to do - this. For safety, and given that the range will be small, do all - this under cli. - - Note, there is a hazard that the ASID in mm->context is no longer - actually associated with mm, i.e. if the mm->context has started a - new cycle since mm was last active. However, this is just a - performance issue: all that happens is that we invalidate lines - belonging to another mm, so the owning process has to refill them - when that mm goes live again. mm itself can't have any cache - entries because there will have been a flush_cache_all when the new - mm->context cycle started. */ - - /* Align to start of cache line. Otherwise, suppose len==8 and start - was at 32N+28 : the last 4 bytes wouldn't get invalidated. */ - eaddr = start & L1_CACHE_ALIGN_MASK; - eaddr_end = start + len; - - local_irq_save(flags); - mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; - current_asid = switch_and_save_asid(mm_asid); - - epage_start = eaddr & PAGE_MASK; - - while (eaddr < eaddr_end) - { - asm __volatile__("icbi %0, 0" : : "r" (eaddr)); - eaddr += L1_CACHE_BYTES; - } - switch_and_save_asid(current_asid); - local_irq_restore(flags); -} - -static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end) -{ - /* The icbi instruction never raises ITLBMISS. i.e. if there's not a - cache hit on the virtual tag the instruction ends there, without a - TLB lookup. */ - - unsigned long long aligned_start; - unsigned long long ull_end; - unsigned long long addr; - - ull_end = end; - - /* Just invalidate over the range using the natural addresses. TLB - miss handling will be OK (TBC). Since it's for the current process, - either we're already in the right ASID context, or the ASIDs have - been recycled since we were last active in which case we might just - invalidate another processes I-cache entries : no worries, just a - performance drop for him. */ - aligned_start = start & L1_CACHE_ALIGN_MASK; - addr = aligned_start; - while (addr < ull_end) { - asm __volatile__ ("icbi %0, 0" : : "r" (addr)); - asm __volatile__ ("nop"); - asm __volatile__ ("nop"); - addr += L1_CACHE_BYTES; - } -} - -#endif /* !CONFIG_ICACHE_DISABLED */ - -/****************************************************************************/ - -#ifndef CONFIG_DCACHE_DISABLED - -/* Buffer used as the target of alloco instructions to purge data from cache - sets by natural eviction. -- RPC */ -#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4) -static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, }; - -/****************************************************************************/ - -static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets) -{ - /* Purge all ways in a particular block of sets, specified by the base - set number and number of sets. Can handle wrap-around, if that's - needed. */ - - int dummy_buffer_base_set; - unsigned long long eaddr, eaddr0, eaddr1; - int j; - int set_offset; - - dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift; - set_offset = sets_to_purge_base - dummy_buffer_base_set; - - for (j=0; jdcache.sets - 1); - eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift); - - /* Do one alloco which hits the required set per cache way. For - write-back mode, this will purge the #ways resident lines. There's - little point unrolling this loop because the allocos stall more if - they're too close together. */ - eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; - for (eaddr=eaddr0; eaddrdcache.way_ofs) { - asm __volatile__ ("alloco %0, 0" : : "r" (eaddr)); - asm __volatile__ ("synco"); /* TAKum03020 */ - } - - eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; - for (eaddr=eaddr0; eaddrdcache.way_ofs) { - /* Load from each address. Required because alloco is a NOP if - the cache is write-through. Write-through is a config option. */ - if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags))) - *(volatile unsigned char *)(int)eaddr; - } - } - - /* Don't use OCBI to invalidate the lines. That costs cycles directly. - If the dummy block is just left resident, it will naturally get - evicted as required. */ - - return; -} - -/****************************************************************************/ - -static void sh64_dcache_purge_all(void) -{ - /* Purge the entire contents of the dcache. The most efficient way to - achieve this is to use alloco instructions on a region of unused - memory equal in size to the cache, thereby causing the current - contents to be discarded by natural eviction. The alternative, - namely reading every tag, setting up a mapping for the corresponding - page and doing an OCBP for the line, would be much more expensive. - */ - - sh64_dcache_purge_sets(0, cpu_data->dcache.sets); - - return; - -} - -/****************************************************************************/ - -static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end) -{ - /* Purge the range of addresses [start,end] from the D-cache. The - addresses lie in the superpage mapping. There's no harm if we - overpurge at either end - just a small performance loss. */ - unsigned long long ullend, addr, aligned_start; -#if (NEFF == 32) - aligned_start = (unsigned long long)(signed long long)(signed long) start; -#else -#error "NEFF != 32" -#endif - aligned_start &= L1_CACHE_ALIGN_MASK; - addr = aligned_start; -#if (NEFF == 32) - ullend = (unsigned long long) (signed long long) (signed long) end; -#else -#error "NEFF != 32" -#endif - while (addr <= ullend) { - asm __volatile__ ("ocbp %0, 0" : : "r" (addr)); - addr += L1_CACHE_BYTES; - } - return; -} - -/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for - anything else in the kernel */ -#define MAGIC_PAGE0_START 0xffffffffec000000ULL - -static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr) -{ - /* Purge the physical page 'paddr' from the cache. It's known that any - cache lines requiring attention have the same page colour as the the - address 'eaddr'. - - This relies on the fact that the D-cache matches on physical tags - when no virtual tag matches. So we create an alias for the original - page and purge through that. (Alternatively, we could have done - this by switching ASID to match the original mapping and purged - through that, but that involves ASID switching cost + probably a - TLBMISS + refill anyway.) - */ - - unsigned long long magic_page_start; - unsigned long long magic_eaddr, magic_eaddr_end; - - magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK); - - /* As long as the kernel is not pre-emptible, this doesn't need to be - under cli/sti. */ - - sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr); - - magic_eaddr = magic_page_start; - magic_eaddr_end = magic_eaddr + PAGE_SIZE; - while (magic_eaddr < magic_eaddr_end) { - /* Little point in unrolling this loop - the OCBPs are blocking - and won't go any quicker (i.e. the loop overhead is parallel - to part of the OCBP execution.) */ - asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr)); - magic_eaddr += L1_CACHE_BYTES; - } - - sh64_teardown_dtlb_cache_slot(); -} - -/****************************************************************************/ - -static void sh64_dcache_purge_phy_page(unsigned long paddr) -{ - /* Pure a page given its physical start address, by creating a - temporary 1 page mapping and purging across that. Even if we know - the virtual address (& vma or mm) of the page, the method here is - more elegant because it avoids issues of coping with page faults on - the purge instructions (i.e. no special-case code required in the - critical path in the TLB miss handling). */ - - unsigned long long eaddr_start, eaddr, eaddr_end; - int i; - - /* As long as the kernel is not pre-emptible, this doesn't need to be - under cli/sti. */ - - eaddr_start = MAGIC_PAGE0_START; - for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) { - sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr); - - eaddr = eaddr_start; - eaddr_end = eaddr + PAGE_SIZE; - while (eaddr < eaddr_end) { - asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr)); - eaddr += L1_CACHE_BYTES; - } - - sh64_teardown_dtlb_cache_slot(); - eaddr_start += PAGE_SIZE; - } -} - -static void sh64_dcache_purge_user_pages(struct mm_struct *mm, - unsigned long addr, unsigned long end) -{ - pgd_t *pgd; - pmd_t *pmd; - pte_t *pte; - pte_t entry; - spinlock_t *ptl; - unsigned long paddr; - - if (!mm) - return; /* No way to find physical address of page */ - - pgd = pgd_offset(mm, addr); - if (pgd_bad(*pgd)) - return; - - pmd = pmd_offset(pgd, addr); - if (pmd_none(*pmd) || pmd_bad(*pmd)) - return; - - pte = pte_offset_map_lock(mm, pmd, addr, &ptl); - do { - entry = *pte; - if (pte_none(entry) || !pte_present(entry)) - continue; - paddr = pte_val(entry) & PAGE_MASK; - sh64_dcache_purge_coloured_phy_page(paddr, addr); - } while (pte++, addr += PAGE_SIZE, addr != end); - pte_unmap_unlock(pte - 1, ptl); -} -/****************************************************************************/ - -static void sh64_dcache_purge_user_range(struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - /* There are at least 5 choices for the implementation of this, with - pros (+), cons(-), comments(*): - - 1. ocbp each line in the range through the original user's ASID - + no lines spuriously evicted - - tlbmiss handling (must either handle faults on demand => extra - special-case code in tlbmiss critical path), or map the page in - advance (=> flush_tlb_range in advance to avoid multiple hits) - - ASID switching - - expensive for large ranges - - 2. temporarily map each page in the range to a special effective - address and ocbp through the temporary mapping; relies on the - fact that SH-5 OCB* always do TLB lookup and match on ptags (they - never look at the etags) - + no spurious evictions - - expensive for large ranges - * surely cheaper than (1) - - 3. walk all the lines in the cache, check the tags, if a match - occurs create a page mapping to ocbp the line through - + no spurious evictions - - tag inspection overhead - - (especially for small ranges) - - potential cost of setting up/tearing down page mapping for - every line that matches the range - * cost partly independent of range size - - 4. walk all the lines in the cache, check the tags, if a match - occurs use 4 * alloco to purge the line (+3 other probably - innocent victims) by natural eviction - + no tlb mapping overheads - - spurious evictions - - tag inspection overhead - - 5. implement like flush_cache_all - + no tag inspection overhead - - spurious evictions - - bad for small ranges - - (1) can be ruled out as more expensive than (2). (2) appears best - for small ranges. The choice between (3), (4) and (5) for large - ranges and the range size for the large/small boundary need - benchmarking to determine. - - For now use approach (2) for small ranges and (5) for large ones. - - */ - - int n_pages; - - n_pages = ((end - start) >> PAGE_SHIFT); - if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) { -#if 1 - sh64_dcache_purge_all(); -#else - unsigned long long set, way; - unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; - for (set = 0; set < cpu_data->dcache.sets; set++) { - unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift); - for (way = 0; way < cpu_data->dcache.ways; way++) { - unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift); - unsigned long long tag0; - unsigned long line_valid; - - asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr)); - line_valid = tag0 & SH_CACHE_VALID; - if (line_valid) { - unsigned long cache_asid; - unsigned long epn; - - cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift; - /* The next line needs some - explanation. The virtual tags - encode bits [31:13] of the virtual - address, bit [12] of the 'tag' being - implied by the cache set index. */ - epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift); - - if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) { - /* TODO : could optimise this - call by batching multiple - adjacent sets together. */ - sh64_dcache_purge_sets(set, 1); - break; /* Don't waste time inspecting other ways for this set */ - } - } - } - } -#endif - } else { - /* Small range, covered by a single page table page */ - start &= PAGE_MASK; /* should already be so */ - end = PAGE_ALIGN(end); /* should already be so */ - sh64_dcache_purge_user_pages(mm, start, end); - } - return; -} - -static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end) -{ - unsigned long long aligned_start; - unsigned long long ull_end; - unsigned long long addr; - - ull_end = end; - - /* Just wback over the range using the natural addresses. TLB miss - handling will be OK (TBC) : the range has just been written to by - the signal frame setup code, so the PTEs must exist. - - Note, if we have CONFIG_PREEMPT and get preempted inside this loop, - it doesn't matter, even if the pid->ASID mapping changes whilst - we're away. In that case the cache will have been flushed when the - mapping was renewed. So the writebacks below will be nugatory (and - we'll doubtless have to fault the TLB entry/ies in again with the - new ASID), but it's a rare case. - */ - aligned_start = start & L1_CACHE_ALIGN_MASK; - addr = aligned_start; - while (addr < ull_end) { - asm __volatile__ ("ocbwb %0, 0" : : "r" (addr)); - addr += L1_CACHE_BYTES; - } -} - -/****************************************************************************/ - -/* These *MUST* lie in an area of virtual address space that's otherwise unused. */ -#define UNIQUE_EADDR_START 0xe0000000UL -#define UNIQUE_EADDR_END 0xe8000000UL - -static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr) -{ - /* Given a physical address paddr, and a user virtual address - user_eaddr which will eventually be mapped to it, create a one-off - kernel-private eaddr mapped to the same paddr. This is used for - creating special destination pages for copy_user_page and - clear_user_page */ - - static unsigned long current_pointer = UNIQUE_EADDR_START; - unsigned long coloured_pointer; - - if (current_pointer == UNIQUE_EADDR_END) { - sh64_dcache_purge_all(); - current_pointer = UNIQUE_EADDR_START; - } - - coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK); - sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr); - - current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS); - - return coloured_pointer; -} - -/****************************************************************************/ - -static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address) -{ - void *coloured_to; - - /* Discard any existing cache entries of the wrong colour. These are - present quite often, if the kernel has recently used the page - internally, then given it up, then it's been allocated to the user. - */ - sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); - - coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); - sh64_page_copy(from, coloured_to); - - sh64_teardown_dtlb_cache_slot(); -} - -static void sh64_clear_user_page_coloured(void *to, unsigned long address) -{ - void *coloured_to; - - /* Discard any existing kernel-originated lines of the wrong colour (as - above) */ - sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); - - coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); - sh64_page_clear(coloured_to); - - sh64_teardown_dtlb_cache_slot(); -} - -#endif /* !CONFIG_DCACHE_DISABLED */ - -/****************************************************************************/ - -/*########################################################################## - EXTERNALLY CALLABLE API. - ##########################################################################*/ - -/* These functions are described in Documentation/cachetlb.txt. - Each one of these functions varies in behaviour depending on whether the - I-cache and/or D-cache are configured out. - - Note that the Linux term 'flush' corresponds to what is termed 'purge' in - the sh/sh64 jargon for the D-cache, i.e. write back dirty data then - invalidate the cache lines, and 'invalidate' for the I-cache. - */ - -#undef FLUSH_TRACE - -void flush_cache_all(void) -{ - /* Invalidate the entire contents of both caches, after writing back to - memory any dirty data from the D-cache. */ - sh64_dcache_purge_all(); - sh64_icache_inv_all(); -} - -/****************************************************************************/ - -void flush_cache_mm(struct mm_struct *mm) -{ - /* Invalidate an entire user-address space from both caches, after - writing back dirty data (e.g. for shared mmap etc). */ - - /* This could be coded selectively by inspecting all the tags then - doing 4*alloco on any set containing a match (as for - flush_cache_range), but fork/exit/execve (where this is called from) - are expensive anyway. */ - - /* Have to do a purge here, despite the comments re I-cache below. - There could be odd-coloured dirty data associated with the mm still - in the cache - if this gets written out through natural eviction - after the kernel has reused the page there will be chaos. - */ - - sh64_dcache_purge_all(); - - /* The mm being torn down won't ever be active again, so any Icache - lines tagged with its ASID won't be visible for the rest of the - lifetime of this ASID cycle. Before the ASID gets reused, there - will be a flush_cache_all. Hence we don't need to touch the - I-cache. This is similar to the lack of action needed in - flush_tlb_mm - see fault.c. */ -} - -/****************************************************************************/ - -void flush_cache_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - - /* Invalidate (from both caches) the range [start,end) of virtual - addresses from the user address space specified by mm, after writing - back any dirty data. - - Note, 'end' is 1 byte beyond the end of the range to flush. */ - - sh64_dcache_purge_user_range(mm, start, end); - sh64_icache_inv_user_page_range(mm, start, end); -} - -/****************************************************************************/ - -void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn) -{ - /* Invalidate any entries in either cache for the vma within the user - address space vma->vm_mm for the page starting at virtual address - 'eaddr'. This seems to be used primarily in breaking COW. Note, - the I-cache must be searched too in case the page in question is - both writable and being executed from (e.g. stack trampolines.) - - Note, this is called with pte lock held. - */ - - sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT); - - if (vma->vm_flags & VM_EXEC) { - sh64_icache_inv_user_page(vma, eaddr); - } -} - -/****************************************************************************/ - -#ifndef CONFIG_DCACHE_DISABLED - -void copy_user_page(void *to, void *from, unsigned long address, struct page *page) -{ - /* 'from' and 'to' are kernel virtual addresses (within the superpage - mapping of the physical RAM). 'address' is the user virtual address - where the copy 'to' will be mapped after. This allows a custom - mapping to be used to ensure that the new copy is placed in the - right cache sets for the user to see it without having to bounce it - out via memory. Note however : the call to flush_page_to_ram in - (generic)/mm/memory.c:(break_cow) undoes all this good work in that one - very important case! - - TBD : can we guarantee that on every call, any cache entries for - 'from' are in the same colour sets as 'address' also? i.e. is this - always used just to deal with COW? (I suspect not). */ - - /* There are two possibilities here for when the page 'from' was last accessed: - * by the kernel : this is OK, no purge required. - * by the/a user (e.g. for break_COW) : need to purge. - - If the potential user mapping at 'address' is the same colour as - 'from' there is no need to purge any cache lines from the 'from' - page mapped into cache sets of colour 'address'. (The copy will be - accessing the page through 'from'). - */ - - if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) { - sh64_dcache_purge_coloured_phy_page(__pa(from), address); - } - - if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { - /* No synonym problem on destination */ - sh64_page_copy(from, to); - } else { - sh64_copy_user_page_coloured(to, from, address); - } - - /* Note, don't need to flush 'from' page from the cache again - it's - done anyway by the generic code */ -} - -void clear_user_page(void *to, unsigned long address, struct page *page) -{ - /* 'to' is a kernel virtual address (within the superpage - mapping of the physical RAM). 'address' is the user virtual address - where the 'to' page will be mapped after. This allows a custom - mapping to be used to ensure that the new copy is placed in the - right cache sets for the user to see it without having to bounce it - out via memory. - */ - - if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { - /* No synonym problem on destination */ - sh64_page_clear(to); - } else { - sh64_clear_user_page_coloured(to, address); - } -} - -#endif /* !CONFIG_DCACHE_DISABLED */ - -/****************************************************************************/ - -void flush_dcache_page(struct page *page) -{ - sh64_dcache_purge_phy_page(page_to_phys(page)); - wmb(); -} - -/****************************************************************************/ - -void flush_icache_range(unsigned long start, unsigned long end) -{ - /* Flush the range [start,end] of kernel virtual adddress space from - the I-cache. The corresponding range must be purged from the - D-cache also because the SH-5 doesn't have cache snooping between - the caches. The addresses will be visible through the superpage - mapping, therefore it's guaranteed that there no cache entries for - the range in cache sets of the wrong colour. - - Primarily used for cohering the I-cache after a module has - been loaded. */ - - /* We also make sure to purge the same range from the D-cache since - flush_page_to_ram() won't be doing this for us! */ - - sh64_dcache_purge_kernel_range(start, end); - wmb(); - sh64_icache_inv_kernel_range(start, end); -} - -/****************************************************************************/ - -void flush_icache_user_range(struct vm_area_struct *vma, - struct page *page, unsigned long addr, int len) -{ - /* Flush the range of user (defined by vma->vm_mm) address space - starting at 'addr' for 'len' bytes from the cache. The range does - not straddle a page boundary, the unique physical page containing - the range is 'page'. This seems to be used mainly for invalidating - an address range following a poke into the program text through the - ptrace() call from another process (e.g. for BRK instruction - insertion). */ - - sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr); - mb(); - - if (vma->vm_flags & VM_EXEC) { - sh64_icache_inv_user_small_range(vma->vm_mm, addr, len); - } -} - -/*########################################################################## - ARCH/SH64 PRIVATE CALLABLE API. - ##########################################################################*/ - -void flush_cache_sigtramp(unsigned long start, unsigned long end) -{ - /* For the address range [start,end), write back the data from the - D-cache and invalidate the corresponding region of the I-cache for - the current process. Used to flush signal trampolines on the stack - to make them executable. */ - - sh64_dcache_wback_current_user_range(start, end); - wmb(); - sh64_icache_inv_current_user_range(start, end); -} - diff --git a/arch/sh64/mm/tlb.c b/arch/sh64/mm/tlb.c deleted file mode 100644 index d517e7d70340..000000000000 --- a/arch/sh64/mm/tlb.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * arch/sh64/mm/tlb.c - * - * Copyright (C) 2003 Paul Mundt - * Copyright (C) 2003 Richard Curnow - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - */ -#include -#include -#include -#include -#include - -/** - * sh64_tlb_init - * - * Perform initial setup for the DTLB and ITLB. - */ -int __init sh64_tlb_init(void) -{ - /* Assign some sane DTLB defaults */ - cpu_data->dtlb.entries = 64; - cpu_data->dtlb.step = 0x10; - - cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step; - cpu_data->dtlb.next = cpu_data->dtlb.first; - - cpu_data->dtlb.last = DTLB_FIXED | - ((cpu_data->dtlb.entries - 1) * - cpu_data->dtlb.step); - - /* And again for the ITLB */ - cpu_data->itlb.entries = 64; - cpu_data->itlb.step = 0x10; - - cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step; - cpu_data->itlb.next = cpu_data->itlb.first; - cpu_data->itlb.last = ITLB_FIXED | - ((cpu_data->itlb.entries - 1) * - cpu_data->itlb.step); - - return 0; -} - -/** - * sh64_next_free_dtlb_entry - * - * Find the next available DTLB entry - */ -unsigned long long sh64_next_free_dtlb_entry(void) -{ - return cpu_data->dtlb.next; -} - -/** - * sh64_get_wired_dtlb_entry - * - * Allocate a wired (locked-in) entry in the DTLB - */ -unsigned long long sh64_get_wired_dtlb_entry(void) -{ - unsigned long long entry = sh64_next_free_dtlb_entry(); - - cpu_data->dtlb.first += cpu_data->dtlb.step; - cpu_data->dtlb.next += cpu_data->dtlb.step; - - return entry; -} - -/** - * sh64_put_wired_dtlb_entry - * - * @entry: Address of TLB slot. - * - * Free a wired (locked-in) entry in the DTLB. - * - * Works like a stack, last one to allocate must be first one to free. - */ -int sh64_put_wired_dtlb_entry(unsigned long long entry) -{ - __flush_tlb_slot(entry); - - /* - * We don't do any particularly useful tracking of wired entries, - * so this approach works like a stack .. last one to be allocated - * has to be the first one to be freed. - * - * We could potentially load wired entries into a list and work on - * rebalancing the list periodically (which also entails moving the - * contents of a TLB entry) .. though I have a feeling that this is - * more trouble than it's worth. - */ - - /* - * Entry must be valid .. we don't want any ITLB addresses! - */ - if (entry <= DTLB_FIXED) - return -EINVAL; - - /* - * Next, check if we're within range to be freed. (ie, must be the - * entry beneath the first 'free' entry! - */ - if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step)) - return -EINVAL; - - /* If we are, then bring this entry back into the list */ - cpu_data->dtlb.first -= cpu_data->dtlb.step; - cpu_data->dtlb.next = entry; - - return 0; -} - -/** - * sh64_setup_tlb_slot - * - * @config_addr: Address of TLB slot. - * @eaddr: Virtual address. - * @asid: Address Space Identifier. - * @paddr: Physical address. - * - * Load up a virtual<->physical translation for @eaddr<->@paddr in the - * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry). - */ -inline void sh64_setup_tlb_slot(unsigned long long config_addr, - unsigned long eaddr, - unsigned long asid, - unsigned long paddr) -{ - unsigned long long pteh, ptel; - - /* Sign extension */ -#if (NEFF == 32) - pteh = (unsigned long long)(signed long long)(signed long) eaddr; -#else -#error "Can't sign extend more than 32 bits yet" -#endif - pteh &= PAGE_MASK; - pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID; -#if (NEFF == 32) - ptel = (unsigned long long)(signed long long)(signed long) paddr; -#else -#error "Can't sign extend more than 32 bits yet" -#endif - ptel &= PAGE_MASK; - ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE); - - asm volatile("putcfg %0, 1, %1\n\t" - "putcfg %0, 0, %2\n" - : : "r" (config_addr), "r" (ptel), "r" (pteh)); -} - -/** - * sh64_teardown_tlb_slot - * - * @config_addr: Address of TLB slot. - * - * Teardown any existing mapping in the TLB slot @config_addr. - */ -inline void sh64_teardown_tlb_slot(unsigned long long config_addr) - __attribute__ ((alias("__flush_tlb_slot"))); - -- cgit v1.2.3-59-g8ed1b