diff options
Diffstat (limited to 'arch/sh/mm')
25 files changed, 2774 insertions, 863 deletions
diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 1265f204f7d1..f549b8cd2501 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -1,193 +1,3 @@ -# -# Processor families -# -config CPU_SH2 - bool - -config CPU_SH2A - bool - select CPU_SH2 - -config CPU_SH3 - bool - select CPU_HAS_INTEVT - select CPU_HAS_SR_RB - -config CPU_SH4 - bool - select CPU_HAS_INTEVT - select CPU_HAS_SR_RB - select CPU_HAS_PTEA if !CPU_SH4A || CPU_SHX2 - select CPU_HAS_FPU if !CPU_SH4AL_DSP - -config CPU_SH4A - bool - select CPU_SH4 - -config CPU_SH4AL_DSP - bool - select CPU_SH4A - select CPU_HAS_DSP - -config CPU_SHX2 - bool - -config CPU_SHX3 - bool - -choice - prompt "Processor sub-type selection" - -# -# Processor subtypes -# - -# SH-2 Processor Support - -config CPU_SUBTYPE_SH7619 - bool "Support SH7619 processor" - select CPU_SH2 - -# SH-2A Processor Support - -config CPU_SUBTYPE_SH7206 - bool "Support SH7206 processor" - select CPU_SH2A - -# SH-3 Processor Support - -config CPU_SUBTYPE_SH7705 - bool "Support SH7705 processor" - select CPU_SH3 - -config CPU_SUBTYPE_SH7706 - bool "Support SH7706 processor" - select CPU_SH3 - help - Select SH7706 if you have a 133 Mhz SH-3 HD6417706 CPU. - -config CPU_SUBTYPE_SH7707 - bool "Support SH7707 processor" - select CPU_SH3 - help - Select SH7707 if you have a 60 Mhz SH-3 HD6417707 CPU. - -config CPU_SUBTYPE_SH7708 - bool "Support SH7708 processor" - select CPU_SH3 - help - Select SH7708 if you have a 60 Mhz SH-3 HD6417708S or - if you have a 100 Mhz SH-3 HD6417708R CPU. - -config CPU_SUBTYPE_SH7709 - bool "Support SH7709 processor" - select CPU_SH3 - help - Select SH7709 if you have a 80 Mhz SH-3 HD6417709 CPU. - -config CPU_SUBTYPE_SH7710 - bool "Support SH7710 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7710 if you have a SH3-DSP SH7710 CPU. - -config CPU_SUBTYPE_SH7712 - bool "Support SH7712 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7712 if you have a SH3-DSP SH7712 CPU. - -config CPU_SUBTYPE_SH7720 - bool "Support SH7720 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7720 if you have a SH3-DSP SH7720 CPU. - -# SH-4 Processor Support - -config CPU_SUBTYPE_SH7750 - bool "Support SH7750 processor" - select CPU_SH4 - help - Select SH7750 if you have a 200 Mhz SH-4 HD6417750 CPU. - -config CPU_SUBTYPE_SH7091 - bool "Support SH7091 processor" - select CPU_SH4 - help - Select SH7091 if you have an SH-4 based Sega device (such as - the Dreamcast, Naomi, and Naomi 2). - -config CPU_SUBTYPE_SH7750R - bool "Support SH7750R processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7750S - bool "Support SH7750S processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7751 - bool "Support SH7751 processor" - select CPU_SH4 - help - Select SH7751 if you have a 166 Mhz SH-4 HD6417751 CPU, - or if you have a HD6417751R CPU. - -config CPU_SUBTYPE_SH7751R - bool "Support SH7751R processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7760 - bool "Support SH7760 processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH4_202 - bool "Support SH4-202 processor" - select CPU_SH4 - -# SH-4A Processor Support - -config CPU_SUBTYPE_SH7770 - bool "Support SH7770 processor" - select CPU_SH4A - -config CPU_SUBTYPE_SH7780 - bool "Support SH7780 processor" - select CPU_SH4A - -config CPU_SUBTYPE_SH7785 - bool "Support SH7785 processor" - select CPU_SH4A - select CPU_SHX2 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - -config CPU_SUBTYPE_SHX3 - bool "Support SH-X3 processor" - select CPU_SH4A - select CPU_SHX3 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - select SYS_SUPPORTS_SMP - -# SH4AL-DSP Processor Support - -config CPU_SUBTYPE_SH7343 - bool "Support SH7343 processor" - select CPU_SH4AL_DSP - -config CPU_SUBTYPE_SH7722 - bool "Support SH7722 processor" - select CPU_SH4AL_DSP - select CPU_SHX2 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - -endchoice - menu "Memory management options" config QUICKLIST @@ -207,7 +17,8 @@ config MMU config PAGE_OFFSET hex - default "0x80000000" if MMU + default "0x80000000" if MMU && SUPERH32 + default "0x20000000" if MMU && SUPERH64 default "0x00000000" config MEMORY_START @@ -228,17 +39,28 @@ config MEMORY_START config MEMORY_SIZE hex "Physical memory size" - default "0x00400000" + default "0x04000000" help This sets the default memory size assumed by your SH kernel. It can be overridden as normal by the 'mem=' argument on the kernel command line. If unsure, consult your board specifications or just leave it - as 0x00400000 which was the default value before this became + as 0x04000000 which was the default value before this became configurable. +# Physical addressing modes + +config 29BIT + def_bool !32BIT + depends on SUPERH32 + config 32BIT + bool + default y if CPU_SH5 + +config PMB bool "Support 32-bit physical addressing through PMB" depends on MMU && (CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785) + select 32BIT default y help If you say Y here, physical addressing will be extended to @@ -256,7 +78,7 @@ config X2TLB config VSYSCALL bool "Support vsyscall page" - depends on MMU + depends on MMU && (CPU_SH3 || CPU_SH4) default y help This will enable support for the kernel mapping a vDSO page @@ -335,7 +157,7 @@ config PAGE_SIZE_8KB config PAGE_SIZE_64KB bool "64kB" - depends on CPU_SH4 + depends on CPU_SH4 || CPU_SH5 help This enables support for 64kB pages, possible on all SH-4 CPUs and later. @@ -344,7 +166,7 @@ endchoice choice prompt "HugeTLB page size" - depends on HUGETLB_PAGE && CPU_SH4 && MMU + depends on HUGETLB_PAGE && (CPU_SH4 || CPU_SH5) && MMU default HUGETLB_PAGE_SIZE_64K config HUGETLB_PAGE_SIZE_64K @@ -365,6 +187,10 @@ config HUGETLB_PAGE_SIZE_64MB bool "64MB" depends on X2TLB +config HUGETLB_PAGE_SIZE_512MB + bool "512MB" + depends on CPU_SH5 + endchoice source "mm/Kconfig" @@ -392,12 +218,12 @@ config SH_DIRECT_MAPPED choice prompt "Cache mode" - default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 + default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5 default CACHE_WRITETHROUGH if (CPU_SH2 && !CPU_SH2A) config CACHE_WRITEBACK bool "Write-back" - depends on CPU_SH2A || CPU_SH3 || CPU_SH4 + depends on CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5 config CACHE_WRITETHROUGH bool "Write-through" diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile index aa44607f072d..9f4bc3d90b1e 100644 --- a/arch/sh/mm/Makefile +++ b/arch/sh/mm/Makefile @@ -1,37 +1,5 @@ -# -# Makefile for the Linux SuperH-specific parts of the memory manager. -# - -obj-y := init.o extable.o consistent.o - -ifndef CONFIG_CACHE_OFF -obj-$(CONFIG_CPU_SH2) += cache-sh2.o -obj-$(CONFIG_CPU_SH3) += cache-sh3.o -obj-$(CONFIG_CPU_SH4) += cache-sh4.o -obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o +ifeq ($(CONFIG_SUPERH32),y) +include ${srctree}/arch/sh/mm/Makefile_32 +else +include ${srctree}/arch/sh/mm/Makefile_64 endif - -mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \ - ioremap.o - -obj-y += $(mmu-y) - -ifdef CONFIG_DEBUG_FS -obj-$(CONFIG_CPU_SH4) += cache-debugfs.o -endif - -ifdef CONFIG_MMU -obj-$(CONFIG_CPU_SH3) += tlb-sh3.o -obj-$(CONFIG_CPU_SH4) += tlb-sh4.o -ifndef CONFIG_CACHE_OFF -obj-$(CONFIG_CPU_SH4) += pg-sh4.o -obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o -endif -endif - -obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_32BIT) += pmb.o -obj-$(CONFIG_NUMA) += numa.o - -EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 new file mode 100644 index 000000000000..e295db60b91b --- /dev/null +++ b/arch/sh/mm/Makefile_32 @@ -0,0 +1,36 @@ +# +# Makefile for the Linux SuperH-specific parts of the memory manager. +# + +obj-y := init.o extable_32.o consistent.o + +ifndef CONFIG_CACHE_OFF +obj-$(CONFIG_CPU_SH2) += cache-sh2.o +obj-$(CONFIG_CPU_SH3) += cache-sh3.o +obj-$(CONFIG_CPU_SH4) += cache-sh4.o +obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o +endif + +mmu-y := tlb-nommu.o pg-nommu.o +mmu-$(CONFIG_MMU) := fault_32.o tlbflush_32.o ioremap_32.o + +obj-y += $(mmu-y) + +ifdef CONFIG_DEBUG_FS +obj-$(CONFIG_CPU_SH4) += cache-debugfs.o +endif + +ifdef CONFIG_MMU +obj-$(CONFIG_CPU_SH3) += tlb-sh3.o +obj-$(CONFIG_CPU_SH4) += tlb-sh4.o +ifndef CONFIG_CACHE_OFF +obj-$(CONFIG_CPU_SH4) += pg-sh4.o +obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o +endif +endif + +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_PMB) += pmb.o +obj-$(CONFIG_NUMA) += numa.o + +EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 new file mode 100644 index 000000000000..cbd6aa33c5ac --- /dev/null +++ b/arch/sh/mm/Makefile_64 @@ -0,0 +1,44 @@ +# +# Makefile for the Linux SuperH-specific parts of the memory manager. +# + +obj-y := init.o extable_64.o consistent.o + +mmu-y := tlb-nommu.o pg-nommu.o +mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlbflush_64.o tlb-sh5.o + +ifndef CONFIG_CACHE_OFF +obj-y += cache-sh5.o +endif + +obj-y += $(mmu-y) + +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_NUMA) += numa.o + +EXTRA_CFLAGS += -Werror + +# Special flags for fault_64.o. This puts restrictions on the number of +# caller-save registers that the compiler can target when building this file. +# This is required because the code is called from a context in entry.S where +# very few registers have been saved in the exception handler (for speed +# reasons). +# The caller save registers that have been saved and which can be used are +# r2,r3,r4,r5 : argument passing +# r15, r18 : SP and LINK +# tr0-4 : allow all caller-save TR's. The compiler seems to be able to make +# use of them, so it's probably beneficial to performance to save them +# and have them available for it. +# +# The resources not listed below are callee save, i.e. the compiler is free to +# use any of them and will spill them to the stack itself. + +CFLAGS_fault_64.o += -ffixed-r7 \ + -ffixed-r8 -ffixed-r9 -ffixed-r10 -ffixed-r11 -ffixed-r12 \ + -ffixed-r13 -ffixed-r14 -ffixed-r16 -ffixed-r17 -ffixed-r19 \ + -ffixed-r20 -ffixed-r21 -ffixed-r22 -ffixed-r23 \ + -ffixed-r24 -ffixed-r25 -ffixed-r26 -ffixed-r27 \ + -ffixed-r36 -ffixed-r37 -ffixed-r38 -ffixed-r39 -ffixed-r40 \ + -ffixed-r41 -ffixed-r42 -ffixed-r43 \ + -ffixed-r60 -ffixed-r61 -ffixed-r62 \ + -fomit-frame-pointer diff --git a/arch/sh/mm/cache-debugfs.c b/arch/sh/mm/cache-debugfs.c index de6d2c9aa477..db6d950b6f5e 100644 --- a/arch/sh/mm/cache-debugfs.c +++ b/arch/sh/mm/cache-debugfs.c @@ -22,7 +22,8 @@ enum cache_type { CACHE_TYPE_UNIFIED, }; -static int cache_seq_show(struct seq_file *file, void *iter) +static int __uses_jump_to_uncached cache_seq_show(struct seq_file *file, + void *iter) { unsigned int cache_type = (unsigned int)file->private; struct cache_info *cache; @@ -34,11 +35,11 @@ static int cache_seq_show(struct seq_file *file, void *iter) * Go uncached immediately so we don't skew the results any * more than we already are.. */ - jump_to_P2(); + jump_to_uncached(); ccr = ctrl_inl(CCR); if ((ccr & CCR_CACHE_ENABLE) == 0) { - back_to_P1(); + back_to_cached(); seq_printf(file, "disabled\n"); return 0; @@ -104,7 +105,7 @@ static int cache_seq_show(struct seq_file *file, void *iter) addrstart += cache->way_incr; } - back_to_P1(); + back_to_cached(); return 0; } diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index 226b190c5b9c..43d7ff6b6ec7 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -190,7 +190,7 @@ void flush_icache_range(unsigned long start, unsigned long end) * .. which happens to be the same behavior as flush_icache_range(). * So, we simply flush out a line. */ -void flush_cache_sigtramp(unsigned long addr) +void __uses_jump_to_uncached flush_cache_sigtramp(unsigned long addr) { unsigned long v, index; unsigned long flags; @@ -205,13 +205,13 @@ void flush_cache_sigtramp(unsigned long addr) (v & boot_cpu_data.icache.entry_mask); local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); for (i = 0; i < boot_cpu_data.icache.ways; i++, index += boot_cpu_data.icache.way_incr) ctrl_outl(0, index); /* Clear out Valid-bit */ - back_to_P1(); + back_to_cached(); wmb(); local_irq_restore(flags); } @@ -256,12 +256,12 @@ void flush_dcache_page(struct page *page) } /* TODO: Selective icache invalidation through IC address array.. */ -static inline void flush_icache_all(void) +static inline void __uses_jump_to_uncached flush_icache_all(void) { unsigned long flags, ccr; local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); /* Flush I-cache */ ccr = ctrl_inl(CCR); @@ -269,11 +269,11 @@ static inline void flush_icache_all(void) ctrl_outl(ccr, CCR); /* - * back_to_P1() will take care of the barrier for us, don't add + * back_to_cached() will take care of the barrier for us, don't add * another one! */ - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c new file mode 100644 index 000000000000..4617e3aeee73 --- /dev/null +++ b/arch/sh/mm/cache-sh5.c @@ -0,0 +1,1029 @@ +/* + * arch/sh/mm/cache-sh5.c + * + * Original version Copyright (C) 2000, 2001 Paolo Alberelli + * Second version Copyright (C) benedict.gaster@superh.com 2002 + * Third version Copyright Richard.Curnow@superh.com 2003 + * Hacks to third version Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/mman.h> +#include <linux/mm.h> +#include <linux/threads.h> +#include <asm/page.h> +#include <asm/pgtable.h> +#include <asm/processor.h> +#include <asm/cache.h> +#include <asm/tlb.h> +#include <asm/io.h> +#include <asm/uaccess.h> +#include <asm/mmu_context.h> +#include <asm/pgalloc.h> /* for flush_itlb_range */ + +#include <linux/proc_fs.h> + +/* This function is in entry.S */ +extern unsigned long switch_and_save_asid(unsigned long new_asid); + +/* Wired TLB entry for the D-cache */ +static unsigned long long dtlb_cache_slot; + +/** + * sh64_cache_init() + * + * This is pretty much just a straightforward clone of the SH + * detect_cpu_and_cache_system(). + * + * This function is responsible for setting up all of the cache + * info dynamically as well as taking care of CPU probing and + * setting up the relevant subtype data. + * + * FIXME: For the time being, we only really support the SH5-101 + * out of the box, and don't support dynamic probing for things + * like the SH5-103 or even cut2 of the SH5-101. Implement this + * later! + */ +int __init sh64_cache_init(void) +{ + /* + * First, setup some sane values for the I-cache. + */ + cpu_data->icache.ways = 4; + cpu_data->icache.sets = 256; + cpu_data->icache.linesz = L1_CACHE_BYTES; + + /* + * FIXME: This can probably be cleaned up a bit as well.. for example, + * do we really need the way shift _and_ the way_step_shift ?? Judging + * by the existing code, I would guess no.. is there any valid reason + * why we need to be tracking this around? + */ + cpu_data->icache.way_shift = 13; + cpu_data->icache.entry_shift = 5; + cpu_data->icache.set_shift = 4; + cpu_data->icache.way_step_shift = 16; + cpu_data->icache.asid_shift = 2; + + /* + * way offset = cache size / associativity, so just don't factor in + * associativity in the first place.. + */ + cpu_data->icache.way_ofs = cpu_data->icache.sets * + cpu_data->icache.linesz; + + cpu_data->icache.asid_mask = 0x3fc; + cpu_data->icache.idx_mask = 0x1fe0; + cpu_data->icache.epn_mask = 0xffffe000; + cpu_data->icache.flags = 0; + + /* + * Next, setup some sane values for the D-cache. + * + * On the SH5, these are pretty consistent with the I-cache settings, + * so we just copy over the existing definitions.. these can be fixed + * up later, especially if we add runtime CPU probing. + * + * Though in the meantime it saves us from having to duplicate all of + * the above definitions.. + */ + cpu_data->dcache = cpu_data->icache; + + /* + * Setup any cache-related flags here + */ +#if defined(CONFIG_DCACHE_WRITE_THROUGH) + set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)); +#elif defined(CONFIG_DCACHE_WRITE_BACK) + set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags)); +#endif + + /* + * We also need to reserve a slot for the D-cache in the DTLB, so we + * do this now .. + */ + dtlb_cache_slot = sh64_get_wired_dtlb_entry(); + + return 0; +} + +#ifdef CONFIG_DCACHE_DISABLED +#define sh64_dcache_purge_all() do { } while (0) +#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0) +#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0) +#define sh64_dcache_purge_phy_page(paddr) do { } while (0) +#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0) +#define sh64_dcache_purge_kernel_range(start, end) do { } while (0) +#define sh64_dcache_wback_current_user_range(start, end) do { } while (0) +#endif + +/*##########################################################################*/ + +/* From here onwards, a rewrite of the implementation, + by Richard.Curnow@superh.com. + + The major changes in this compared to the old version are; + 1. use more selective purging through OCBP instead of using ALLOCO to purge + by natural replacement. This avoids purging out unrelated cache lines + that happen to be in the same set. + 2. exploit the APIs copy_user_page and clear_user_page better + 3. be more selective about I-cache purging, in particular use invalidate_all + more sparingly. + + */ + +/*########################################################################## + SUPPORT FUNCTIONS + ##########################################################################*/ + +/****************************************************************************/ +/* The following group of functions deal with mapping and unmapping a temporary + page into the DTLB slot that have been set aside for our exclusive use. */ +/* In order to accomplish this, we use the generic interface for adding and + removing a wired slot entry as defined in arch/sh/mm/tlb-sh5.c */ +/****************************************************************************/ + +static unsigned long slot_own_flags; + +static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr) +{ + local_irq_save(slot_own_flags); + sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr); +} + +static inline void sh64_teardown_dtlb_cache_slot(void) +{ + sh64_teardown_tlb_slot(dtlb_cache_slot); + local_irq_restore(slot_own_flags); +} + +/****************************************************************************/ + +#ifndef CONFIG_ICACHE_DISABLED + +static void __inline__ sh64_icache_inv_all(void) +{ + unsigned long long addr, flag, data; + unsigned int flags; + + addr=ICCR0; + flag=ICCR0_ICI; + data=0; + + /* Make this a critical section for safety (probably not strictly necessary.) */ + local_irq_save(flags); + + /* Without %1 it gets unexplicably wrong */ + asm volatile("getcfg %3, 0, %0\n\t" + "or %0, %2, %0\n\t" + "putcfg %3, 0, %0\n\t" + "synci" + : "=&r" (data) + : "0" (data), "r" (flag), "r" (addr)); + + local_irq_restore(flags); +} + +static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end) +{ + /* Invalidate range of addresses [start,end] from the I-cache, where + * the addresses lie in the kernel superpage. */ + + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr) +{ + /* If we get called, we know that vma->vm_flags contains VM_EXEC. + Also, eaddr is page-aligned. */ + + unsigned long long addr, end_addr; + unsigned long flags = 0; + unsigned long running_asid, vma_asid; + addr = eaddr; + end_addr = addr + PAGE_SIZE; + + /* Check whether we can use the current ASID for the I-cache + invalidation. For example, if we're called via + access_process_vm->flush_cache_page->here, (e.g. when reading from + /proc), 'running_asid' will be that of the reader, not of the + victim. + + Also, note the risk that we might get pre-empted between the ASID + compare and blocking IRQs, and before we regain control, the + pid->ASID mapping changes. However, the whole cache will get + invalidated when the mapping is renewed, so the worst that can + happen is that the loop below ends up invalidating somebody else's + cache entries. + */ + + running_asid = get_asid(); + vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK); + if (running_asid != vma_asid) { + local_irq_save(flags); + switch_and_save_asid(vma_asid); + } + while (addr < end_addr) { + /* Worth unrolling a little */ + asm __volatile__("icbi %0, 0" : : "r" (addr)); + asm __volatile__("icbi %0, 32" : : "r" (addr)); + asm __volatile__("icbi %0, 64" : : "r" (addr)); + asm __volatile__("icbi %0, 96" : : "r" (addr)); + addr += 128; + } + if (running_asid != vma_asid) { + switch_and_save_asid(running_asid); + local_irq_restore(flags); + } +} + +/****************************************************************************/ + +static void sh64_icache_inv_user_page_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* Used for invalidating big chunks of I-cache, i.e. assume the range + is whole pages. If 'start' or 'end' is not page aligned, the code + is conservative and invalidates to the ends of the enclosing pages. + This is functionally OK, just a performance loss. */ + + /* See the comments below in sh64_dcache_purge_user_range() regarding + the choice of algorithm. However, for the I-cache option (2) isn't + available because there are no physical tags so aliases can't be + resolved. The icbi instruction has to be used through the user + mapping. Because icbi is cheaper than ocbp on a cache hit, it + would be cheaper to use the selective code for a large range than is + possible with the D-cache. Just assume 64 for now as a working + figure. + */ + + int n_pages; + + if (!mm) return; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64) { + sh64_icache_inv_all(); + } else { + unsigned long aligned_start; + unsigned long eaddr; + unsigned long after_last_page_start; + unsigned long mm_asid, current_asid; + unsigned long long flags = 0ULL; + + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = get_asid(); + + if (mm_asid != current_asid) { + /* Switch ASID and run the invalidate loop under cli */ + local_irq_save(flags); + switch_and_save_asid(mm_asid); + } + + aligned_start = start & PAGE_MASK; + after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK); + + while (aligned_start < after_last_page_start) { + struct vm_area_struct *vma; + unsigned long vma_end; + vma = find_vma(mm, aligned_start); + if (!vma || (aligned_start <= vma->vm_end)) { + /* Avoid getting stuck in an error condition */ + aligned_start += PAGE_SIZE; + continue; + } + vma_end = vma->vm_end; + if (vma->vm_flags & VM_EXEC) { + /* Executable */ + eaddr = aligned_start; + while (eaddr < vma_end) { + sh64_icache_inv_user_page(vma, eaddr); + eaddr += PAGE_SIZE; + } + } + aligned_start = vma->vm_end; /* Skip to start of next region */ + } + if (mm_asid != current_asid) { + switch_and_save_asid(current_asid); + local_irq_restore(flags); + } + } +} + +static void sh64_icache_inv_user_small_range(struct mm_struct *mm, + unsigned long start, int len) +{ + + /* Invalidate a small range of user context I-cache, not necessarily + page (or even cache-line) aligned. */ + + unsigned long long eaddr = start; + unsigned long long eaddr_end = start + len; + unsigned long current_asid, mm_asid; + unsigned long long flags; + unsigned long long epage_start; + + /* Since this is used inside ptrace, the ASID in the mm context + typically won't match current_asid. We'll have to switch ASID to do + this. For safety, and given that the range will be small, do all + this under cli. + + Note, there is a hazard that the ASID in mm->context is no longer + actually associated with mm, i.e. if the mm->context has started a + new cycle since mm was last active. However, this is just a + performance issue: all that happens is that we invalidate lines + belonging to another mm, so the owning process has to refill them + when that mm goes live again. mm itself can't have any cache + entries because there will have been a flush_cache_all when the new + mm->context cycle started. */ + + /* Align to start of cache line. Otherwise, suppose len==8 and start + was at 32N+28 : the last 4 bytes wouldn't get invalidated. */ + eaddr = start & L1_CACHE_ALIGN_MASK; + eaddr_end = start + len; + + local_irq_save(flags); + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = switch_and_save_asid(mm_asid); + + epage_start = eaddr & PAGE_MASK; + + while (eaddr < eaddr_end) + { + asm __volatile__("icbi %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + switch_and_save_asid(current_asid); + local_irq_restore(flags); +} + +static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end) +{ + /* The icbi instruction never raises ITLBMISS. i.e. if there's not a + cache hit on the virtual tag the instruction ends there, without a + TLB lookup. */ + + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just invalidate over the range using the natural addresses. TLB + miss handling will be OK (TBC). Since it's for the current process, + either we're already in the right ASID context, or the ASIDs have + been recycled since we were last active in which case we might just + invalidate another processes I-cache entries : no worries, just a + performance drop for him. */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + asm __volatile__ ("nop"); + asm __volatile__ ("nop"); + addr += L1_CACHE_BYTES; + } +} + +#endif /* !CONFIG_ICACHE_DISABLED */ + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +/* Buffer used as the target of alloco instructions to purge data from cache + sets by natural eviction. -- RPC */ +#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4) +static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, }; + +/****************************************************************************/ + +static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets) +{ + /* Purge all ways in a particular block of sets, specified by the base + set number and number of sets. Can handle wrap-around, if that's + needed. */ + + int dummy_buffer_base_set; + unsigned long long eaddr, eaddr0, eaddr1; + int j; + int set_offset; + + dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift; + set_offset = sets_to_purge_base - dummy_buffer_base_set; + + for (j=0; j<n_sets; j++, set_offset++) { + set_offset &= (cpu_data->dcache.sets - 1); + eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift); + + /* Do one alloco which hits the required set per cache way. For + write-back mode, this will purge the #ways resident lines. There's + little point unrolling this loop because the allocos stall more if + they're too close together. */ + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) { + asm __volatile__ ("alloco %0, 0" : : "r" (eaddr)); + asm __volatile__ ("synco"); /* TAKum03020 */ + } + + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) { + /* Load from each address. Required because alloco is a NOP if + the cache is write-through. Write-through is a config option. */ + if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags))) + *(volatile unsigned char *)(int)eaddr; + } + } + + /* Don't use OCBI to invalidate the lines. That costs cycles directly. + If the dummy block is just left resident, it will naturally get + evicted as required. */ + + return; +} + +/****************************************************************************/ + +static void sh64_dcache_purge_all(void) +{ + /* Purge the entire contents of the dcache. The most efficient way to + achieve this is to use alloco instructions on a region of unused + memory equal in size to the cache, thereby causing the current + contents to be discarded by natural eviction. The alternative, + namely reading every tag, setting up a mapping for the corresponding + page and doing an OCBP for the line, would be much more expensive. + */ + + sh64_dcache_purge_sets(0, cpu_data->dcache.sets); + + return; + +} + +/****************************************************************************/ + +static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end) +{ + /* Purge the range of addresses [start,end] from the D-cache. The + addresses lie in the superpage mapping. There's no harm if we + overpurge at either end - just a small performance loss. */ + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("ocbp %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } + return; +} + +/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for + anything else in the kernel */ +#define MAGIC_PAGE0_START 0xffffffffec000000ULL + +static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr) +{ + /* Purge the physical page 'paddr' from the cache. It's known that any + cache lines requiring attention have the same page colour as the the + address 'eaddr'. + + This relies on the fact that the D-cache matches on physical tags + when no virtual tag matches. So we create an alias for the original + page and purge through that. (Alternatively, we could have done + this by switching ASID to match the original mapping and purged + through that, but that involves ASID switching cost + probably a + TLBMISS + refill anyway.) + */ + + unsigned long long magic_page_start; + unsigned long long magic_eaddr, magic_eaddr_end; + + magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK); + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr); + + magic_eaddr = magic_page_start; + magic_eaddr_end = magic_eaddr + PAGE_SIZE; + while (magic_eaddr < magic_eaddr_end) { + /* Little point in unrolling this loop - the OCBPs are blocking + and won't go any quicker (i.e. the loop overhead is parallel + to part of the OCBP execution.) */ + asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr)); + magic_eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); +} + +/****************************************************************************/ + +static void sh64_dcache_purge_phy_page(unsigned long paddr) +{ + /* Pure a page given its physical start address, by creating a + temporary 1 page mapping and purging across that. Even if we know + the virtual address (& vma or mm) of the page, the method here is + more elegant because it avoids issues of coping with page faults on + the purge instructions (i.e. no special-case code required in the + critical path in the TLB miss handling). */ + + unsigned long long eaddr_start, eaddr, eaddr_end; + int i; + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + eaddr_start = MAGIC_PAGE0_START; + for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) { + sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr); + + eaddr = eaddr_start; + eaddr_end = eaddr + PAGE_SIZE; + while (eaddr < eaddr_end) { + asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); + eaddr_start += PAGE_SIZE; + } +} + +static void sh64_dcache_purge_user_pages(struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + spinlock_t *ptl; + unsigned long paddr; + + if (!mm) + return; /* No way to find physical address of page */ + + pgd = pgd_offset(mm, addr); + if (pgd_bad(*pgd)) + return; + + pmd = pmd_offset(pgd, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return; + + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + do { + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + continue; + paddr = pte_val(entry) & PAGE_MASK; + sh64_dcache_purge_coloured_phy_page(paddr, addr); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(pte - 1, ptl); +} +/****************************************************************************/ + +static void sh64_dcache_purge_user_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* There are at least 5 choices for the implementation of this, with + pros (+), cons(-), comments(*): + + 1. ocbp each line in the range through the original user's ASID + + no lines spuriously evicted + - tlbmiss handling (must either handle faults on demand => extra + special-case code in tlbmiss critical path), or map the page in + advance (=> flush_tlb_range in advance to avoid multiple hits) + - ASID switching + - expensive for large ranges + + 2. temporarily map each page in the range to a special effective + address and ocbp through the temporary mapping; relies on the + fact that SH-5 OCB* always do TLB lookup and match on ptags (they + never look at the etags) + + no spurious evictions + - expensive for large ranges + * surely cheaper than (1) + + 3. walk all the lines in the cache, check the tags, if a match + occurs create a page mapping to ocbp the line through + + no spurious evictions + - tag inspection overhead + - (especially for small ranges) + - potential cost of setting up/tearing down page mapping for + every line that matches the range + * cost partly independent of range size + + 4. walk all the lines in the cache, check the tags, if a match + occurs use 4 * alloco to purge the line (+3 other probably + innocent victims) by natural eviction + + no tlb mapping overheads + - spurious evictions + - tag inspection overhead + + 5. implement like flush_cache_all + + no tag inspection overhead + - spurious evictions + - bad for small ranges + + (1) can be ruled out as more expensive than (2). (2) appears best + for small ranges. The choice between (3), (4) and (5) for large + ranges and the range size for the large/small boundary need + benchmarking to determine. + + For now use approach (2) for small ranges and (5) for large ones. + + */ + + int n_pages; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) { +#if 1 + sh64_dcache_purge_all(); +#else + unsigned long long set, way; + unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + for (set = 0; set < cpu_data->dcache.sets; set++) { + unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift); + for (way = 0; way < cpu_data->dcache.ways; way++) { + unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift); + unsigned long long tag0; + unsigned long line_valid; + + asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr)); + line_valid = tag0 & SH_CACHE_VALID; + if (line_valid) { + unsigned long cache_asid; + unsigned long epn; + + cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift; + /* The next line needs some + explanation. The virtual tags + encode bits [31:13] of the virtual + address, bit [12] of the 'tag' being + implied by the cache set index. */ + epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift); + + if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) { + /* TODO : could optimise this + call by batching multiple + adjacent sets together. */ + sh64_dcache_purge_sets(set, 1); + break; /* Don't waste time inspecting other ways for this set */ + } + } + } + } +#endif + } else { + /* Small range, covered by a single page table page */ + start &= PAGE_MASK; /* should already be so */ + end = PAGE_ALIGN(end); /* should already be so */ + sh64_dcache_purge_user_pages(mm, start, end); + } + return; +} + +static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end) +{ + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just wback over the range using the natural addresses. TLB miss + handling will be OK (TBC) : the range has just been written to by + the signal frame setup code, so the PTEs must exist. + + Note, if we have CONFIG_PREEMPT and get preempted inside this loop, + it doesn't matter, even if the pid->ASID mapping changes whilst + we're away. In that case the cache will have been flushed when the + mapping was renewed. So the writebacks below will be nugatory (and + we'll doubtless have to fault the TLB entry/ies in again with the + new ASID), but it's a rare case. + */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("ocbwb %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +/****************************************************************************/ + +/* These *MUST* lie in an area of virtual address space that's otherwise unused. */ +#define UNIQUE_EADDR_START 0xe0000000UL +#define UNIQUE_EADDR_END 0xe8000000UL + +static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr) +{ + /* Given a physical address paddr, and a user virtual address + user_eaddr which will eventually be mapped to it, create a one-off + kernel-private eaddr mapped to the same paddr. This is used for + creating special destination pages for copy_user_page and + clear_user_page */ + + static unsigned long current_pointer = UNIQUE_EADDR_START; + unsigned long coloured_pointer; + + if (current_pointer == UNIQUE_EADDR_END) { + sh64_dcache_purge_all(); + current_pointer = UNIQUE_EADDR_START; + } + + coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK); + sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr); + + current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS); + + return coloured_pointer; +} + +/****************************************************************************/ + +static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing cache entries of the wrong colour. These are + present quite often, if the kernel has recently used the page + internally, then given it up, then it's been allocated to the user. + */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_copy(from, coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +static void sh64_clear_user_page_coloured(void *to, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing kernel-originated lines of the wrong colour (as + above) */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_clear(coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +/*########################################################################## + EXTERNALLY CALLABLE API. + ##########################################################################*/ + +/* These functions are described in Documentation/cachetlb.txt. + Each one of these functions varies in behaviour depending on whether the + I-cache and/or D-cache are configured out. + + Note that the Linux term 'flush' corresponds to what is termed 'purge' in + the sh/sh64 jargon for the D-cache, i.e. write back dirty data then + invalidate the cache lines, and 'invalidate' for the I-cache. + */ + +#undef FLUSH_TRACE + +void flush_cache_all(void) +{ + /* Invalidate the entire contents of both caches, after writing back to + memory any dirty data from the D-cache. */ + sh64_dcache_purge_all(); + sh64_icache_inv_all(); +} + +/****************************************************************************/ + +void flush_cache_mm(struct mm_struct *mm) +{ + /* Invalidate an entire user-address space from both caches, after + writing back dirty data (e.g. for shared mmap etc). */ + + /* This could be coded selectively by inspecting all the tags then + doing 4*alloco on any set containing a match (as for + flush_cache_range), but fork/exit/execve (where this is called from) + are expensive anyway. */ + + /* Have to do a purge here, despite the comments re I-cache below. + There could be odd-coloured dirty data associated with the mm still + in the cache - if this gets written out through natural eviction + after the kernel has reused the page there will be chaos. + */ + + sh64_dcache_purge_all(); + + /* The mm being torn down won't ever be active again, so any Icache + lines tagged with its ASID won't be visible for the rest of the + lifetime of this ASID cycle. Before the ASID gets reused, there + will be a flush_cache_all. Hence we don't need to touch the + I-cache. This is similar to the lack of action needed in + flush_tlb_mm - see fault.c. */ +} + +/****************************************************************************/ + +void flush_cache_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + /* Invalidate (from both caches) the range [start,end) of virtual + addresses from the user address space specified by mm, after writing + back any dirty data. + + Note, 'end' is 1 byte beyond the end of the range to flush. */ + + sh64_dcache_purge_user_range(mm, start, end); + sh64_icache_inv_user_page_range(mm, start, end); +} + +/****************************************************************************/ + +void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn) +{ + /* Invalidate any entries in either cache for the vma within the user + address space vma->vm_mm for the page starting at virtual address + 'eaddr'. This seems to be used primarily in breaking COW. Note, + the I-cache must be searched too in case the page in question is + both writable and being executed from (e.g. stack trampolines.) + + Note, this is called with pte lock held. + */ + + sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_page(vma, eaddr); + } +} + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +void copy_user_page(void *to, void *from, unsigned long address, struct page *page) +{ + /* 'from' and 'to' are kernel virtual addresses (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the copy 'to' will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. Note however : the call to flush_page_to_ram in + (generic)/mm/memory.c:(break_cow) undoes all this good work in that one + very important case! + + TBD : can we guarantee that on every call, any cache entries for + 'from' are in the same colour sets as 'address' also? i.e. is this + always used just to deal with COW? (I suspect not). */ + + /* There are two possibilities here for when the page 'from' was last accessed: + * by the kernel : this is OK, no purge required. + * by the/a user (e.g. for break_COW) : need to purge. + + If the potential user mapping at 'address' is the same colour as + 'from' there is no need to purge any cache lines from the 'from' + page mapped into cache sets of colour 'address'. (The copy will be + accessing the page through 'from'). + */ + + if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) { + sh64_dcache_purge_coloured_phy_page(__pa(from), address); + } + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_copy(from, to); + } else { + sh64_copy_user_page_coloured(to, from, address); + } + + /* Note, don't need to flush 'from' page from the cache again - it's + done anyway by the generic code */ +} + +void clear_user_page(void *to, unsigned long address, struct page *page) +{ + /* 'to' is a kernel virtual address (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the 'to' page will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. + */ + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_clear(to); + } else { + sh64_clear_user_page_coloured(to, address); + } +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +void flush_dcache_page(struct page *page) +{ + sh64_dcache_purge_phy_page(page_to_phys(page)); + wmb(); +} + +/****************************************************************************/ + +void flush_icache_range(unsigned long start, unsigned long end) +{ + /* Flush the range [start,end] of kernel virtual adddress space from + the I-cache. The corresponding range must be purged from the + D-cache also because the SH-5 doesn't have cache snooping between + the caches. The addresses will be visible through the superpage + mapping, therefore it's guaranteed that there no cache entries for + the range in cache sets of the wrong colour. + + Primarily used for cohering the I-cache after a module has + been loaded. */ + + /* We also make sure to purge the same range from the D-cache since + flush_page_to_ram() won't be doing this for us! */ + + sh64_dcache_purge_kernel_range(start, end); + wmb(); + sh64_icache_inv_kernel_range(start, end); +} + +/****************************************************************************/ + +void flush_icache_user_range(struct vm_area_struct *vma, + struct page *page, unsigned long addr, int len) +{ + /* Flush the range of user (defined by vma->vm_mm) address space + starting at 'addr' for 'len' bytes from the cache. The range does + not straddle a page boundary, the unique physical page containing + the range is 'page'. This seems to be used mainly for invalidating + an address range following a poke into the program text through the + ptrace() call from another process (e.g. for BRK instruction + insertion). */ + + sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr); + mb(); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_small_range(vma->vm_mm, addr, len); + } +} + +/*########################################################################## + ARCH/SH64 PRIVATE CALLABLE API. + ##########################################################################*/ + +void flush_cache_sigtramp(unsigned long start, unsigned long end) +{ + /* For the address range [start,end), write back the data from the + D-cache and invalidate the corresponding region of the I-cache for + the current process. Used to flush signal trampolines on the stack + to make them executable. */ + + sh64_dcache_wback_current_user_range(start, end); + wmb(); + sh64_icache_inv_current_user_range(start, end); +} + diff --git a/arch/sh/mm/cache-sh7705.c b/arch/sh/mm/cache-sh7705.c index 4896d7376926..22dacc778823 100644 --- a/arch/sh/mm/cache-sh7705.c +++ b/arch/sh/mm/cache-sh7705.c @@ -71,7 +71,7 @@ void flush_icache_range(unsigned long start, unsigned long end) /* * Writeback&Invalidate the D-cache of the page */ -static void __flush_dcache_page(unsigned long phys) +static void __uses_jump_to_uncached __flush_dcache_page(unsigned long phys) { unsigned long ways, waysize, addrstart; unsigned long flags; @@ -92,7 +92,7 @@ static void __flush_dcache_page(unsigned long phys) * possible. */ local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); ways = current_cpu_data.dcache.ways; waysize = current_cpu_data.dcache.sets; @@ -118,7 +118,7 @@ static void __flush_dcache_page(unsigned long phys) addrstart += current_cpu_data.dcache.way_incr; } while (--ways); - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } @@ -132,15 +132,15 @@ void flush_dcache_page(struct page *page) __flush_dcache_page(PHYSADDR(page_address(page))); } -void flush_cache_all(void) +void __uses_jump_to_uncached flush_cache_all(void) { unsigned long flags; local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); cache_wback_all(); - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } diff --git a/arch/sh/mm/clear_page.S b/arch/sh/mm/clear_page.S deleted file mode 100644 index 7a7c81ee3f01..000000000000 --- a/arch/sh/mm/clear_page.S +++ /dev/null @@ -1,152 +0,0 @@ -/* - * __clear_user_page, __clear_user, clear_page implementation of SuperH - * - * Copyright (C) 2001 Kaz Kojima - * Copyright (C) 2001, 2002 Niibe Yutaka - * Copyright (C) 2006 Paul Mundt - */ -#include <linux/linkage.h> -#include <asm/page.h> - -/* - * clear_page_slow - * @to: P1 address - * - * void clear_page_slow(void *to) - */ - -/* - * r0 --- scratch - * r4 --- to - * r5 --- to + PAGE_SIZE - */ -ENTRY(clear_page_slow) - mov r4,r5 - mov.l .Llimit,r0 - add r0,r5 - mov #0,r0 - ! -1: -#if defined(CONFIG_CPU_SH3) - mov.l r0,@r4 -#elif defined(CONFIG_CPU_SH4) - movca.l r0,@r4 - mov r4,r1 -#endif - add #32,r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 -#if defined(CONFIG_CPU_SH4) - ocbwb @r1 -#endif - cmp/eq r5,r4 - bf/s 1b - add #28,r4 - ! - rts - nop -.Llimit: .long (PAGE_SIZE-28) - -ENTRY(__clear_user) - ! - mov #0, r0 - mov #0xe0, r1 ! 0xffffffe0 - ! - ! r4..(r4+31)&~32 -------- not aligned [ Area 0 ] - ! (r4+31)&~32..(r4+r5)&~32 -------- aligned [ Area 1 ] - ! (r4+r5)&~32..r4+r5 -------- not aligned [ Area 2 ] - ! - ! Clear area 0 - mov r4, r2 - ! - tst r1, r5 ! length < 32 - bt .Larea2 ! skip to remainder - ! - add #31, r2 - and r1, r2 - cmp/eq r4, r2 - bt .Larea1 - mov r2, r3 - sub r4, r3 - mov r3, r7 - mov r4, r2 - ! -.L0: dt r3 -0: mov.b r0, @r2 - bf/s .L0 - add #1, r2 - ! - sub r7, r5 - mov r2, r4 -.Larea1: - mov r4, r3 - add r5, r3 - and r1, r3 - cmp/hi r2, r3 - bf .Larea2 - ! - ! Clear area 1 -#if defined(CONFIG_CPU_SH4) -1: movca.l r0, @r2 -#else -1: mov.l r0, @r2 -#endif - add #4, r2 -2: mov.l r0, @r2 - add #4, r2 -3: mov.l r0, @r2 - add #4, r2 -4: mov.l r0, @r2 - add #4, r2 -5: mov.l r0, @r2 - add #4, r2 -6: mov.l r0, @r2 - add #4, r2 -7: mov.l r0, @r2 - add #4, r2 -8: mov.l r0, @r2 - add #4, r2 - cmp/hi r2, r3 - bt/s 1b - nop - ! - ! Clear area 2 -.Larea2: - mov r4, r3 - add r5, r3 - cmp/hs r3, r2 - bt/s .Ldone - sub r2, r3 -.L2: dt r3 -9: mov.b r0, @r2 - bf/s .L2 - add #1, r2 - ! -.Ldone: rts - mov #0, r0 ! return 0 as normal return - - ! return the number of bytes remained -.Lbad_clear_user: - mov r4, r0 - add r5, r0 - rts - sub r2, r0 - -.section __ex_table,"a" - .align 2 - .long 0b, .Lbad_clear_user - .long 1b, .Lbad_clear_user - .long 2b, .Lbad_clear_user - .long 3b, .Lbad_clear_user - .long 4b, .Lbad_clear_user - .long 5b, .Lbad_clear_user - .long 6b, .Lbad_clear_user - .long 7b, .Lbad_clear_user - .long 8b, .Lbad_clear_user - .long 9b, .Lbad_clear_user -.previous diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c index e220c29a3c00..7b2131c9eeda 100644 --- a/arch/sh/mm/consistent.c +++ b/arch/sh/mm/consistent.c @@ -1,7 +1,9 @@ /* * arch/sh/mm/consistent.c * - * Copyright (C) 2004 Paul Mundt + * Copyright (C) 2004 - 2007 Paul Mundt + * + * Declared coherent memory functions based on arch/x86/kernel/pci-dma_32.c * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive @@ -13,58 +15,152 @@ #include <asm/addrspace.h> #include <asm/io.h> -void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) +struct dma_coherent_mem { + void *virt_base; + u32 device_base; + int size; + int flags; + unsigned long *bitmap; +}; + +void *dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp) { - struct page *page, *end, *free; void *ret; - int order; + struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; + int order = get_order(size); - size = PAGE_ALIGN(size); - order = get_order(size); + if (mem) { + int page = bitmap_find_free_region(mem->bitmap, mem->size, + order); + if (page >= 0) { + *dma_handle = mem->device_base + (page << PAGE_SHIFT); + ret = mem->virt_base + (page << PAGE_SHIFT); + memset(ret, 0, size); + return ret; + } + if (mem->flags & DMA_MEMORY_EXCLUSIVE) + return NULL; + } - page = alloc_pages(gfp, order); - if (!page) - return NULL; - split_page(page, order); + ret = (void *)__get_free_pages(gfp, order); - ret = page_address(page); - memset(ret, 0, size); - *handle = virt_to_phys(ret); + if (ret != NULL) { + memset(ret, 0, size); + /* + * Pages from the page allocator may have data present in + * cache. So flush the cache before using uncached memory. + */ + dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL); + *dma_handle = virt_to_phys(ret); + } + return ret; +} +EXPORT_SYMBOL(dma_alloc_coherent); - /* - * We must flush the cache before we pass it on to the device - */ - __flush_purge_region(ret, size); +void dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) +{ + struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; + int order = get_order(size); - page = virt_to_page(ret); - free = page + (size >> PAGE_SHIFT); - end = page + (1 << order); + if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) { + int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; - while (++page < end) { - /* Free any unused pages */ - if (page >= free) { - __free_page(page); - } + bitmap_release_region(mem->bitmap, page, order); + } else { + WARN_ON(irqs_disabled()); /* for portability */ + BUG_ON(mem && mem->flags & DMA_MEMORY_EXCLUSIVE); + free_pages((unsigned long)vaddr, order); } +} +EXPORT_SYMBOL(dma_free_coherent); + +int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags) +{ + void __iomem *mem_base = NULL; + int pages = size >> PAGE_SHIFT; + int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + + if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) + goto out; + if (!size) + goto out; + if (dev->dma_mem) + goto out; + + /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ - return P2SEGADDR(ret); + mem_base = ioremap_nocache(bus_addr, size); + if (!mem_base) + goto out; + + dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); + if (!dev->dma_mem) + goto out; + dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); + if (!dev->dma_mem->bitmap) + goto free1_out; + + dev->dma_mem->virt_base = mem_base; + dev->dma_mem->device_base = device_addr; + dev->dma_mem->size = pages; + dev->dma_mem->flags = flags; + + if (flags & DMA_MEMORY_MAP) + return DMA_MEMORY_MAP; + + return DMA_MEMORY_IO; + + free1_out: + kfree(dev->dma_mem); + out: + if (mem_base) + iounmap(mem_base); + return 0; } +EXPORT_SYMBOL(dma_declare_coherent_memory); -void consistent_free(void *vaddr, size_t size) +void dma_release_declared_memory(struct device *dev) { - unsigned long addr = P1SEGADDR((unsigned long)vaddr); - struct page *page=virt_to_page(addr); - int num_pages=(size+PAGE_SIZE-1) >> PAGE_SHIFT; - int i; + struct dma_coherent_mem *mem = dev->dma_mem; - for(i=0;i<num_pages;i++) { - __free_page((page+i)); - } + if (!mem) + return; + dev->dma_mem = NULL; + iounmap(mem->virt_base); + kfree(mem->bitmap); + kfree(mem); } +EXPORT_SYMBOL(dma_release_declared_memory); -void consistent_sync(void *vaddr, size_t size, int direction) +void *dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) { - void * p1addr = (void*) P1SEGADDR((unsigned long)vaddr); + struct dma_coherent_mem *mem = dev->dma_mem; + int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT; + int pos, err; + + if (!mem) + return ERR_PTR(-EINVAL); + + pos = (device_addr - mem->device_base) >> PAGE_SHIFT; + err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages)); + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); +} +EXPORT_SYMBOL(dma_mark_declared_memory_occupied); + +void dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction direction) +{ +#ifdef CONFIG_CPU_SH5 + void *p1addr = vaddr; +#else + void *p1addr = (void*) P1SEGADDR((unsigned long)vaddr); +#endif switch (direction) { case DMA_FROM_DEVICE: /* invalidate only */ @@ -80,8 +176,4 @@ void consistent_sync(void *vaddr, size_t size, int direction) BUG(); } } - -EXPORT_SYMBOL(consistent_alloc); -EXPORT_SYMBOL(consistent_free); -EXPORT_SYMBOL(consistent_sync); - +EXPORT_SYMBOL(dma_cache_sync); diff --git a/arch/sh/mm/copy_page.S b/arch/sh/mm/copy_page.S deleted file mode 100644 index 40685018b952..000000000000 --- a/arch/sh/mm/copy_page.S +++ /dev/null @@ -1,388 +0,0 @@ -/* - * copy_page, __copy_user_page, __copy_user implementation of SuperH - * - * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima - * Copyright (C) 2002 Toshinobu Sugioka - * Copyright (C) 2006 Paul Mundt - */ -#include <linux/linkage.h> -#include <asm/page.h> - -/* - * copy_page_slow - * @to: P1 address - * @from: P1 address - * - * void copy_page_slow(void *to, void *from) - */ - -/* - * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch - * r8 --- from + PAGE_SIZE - * r9 --- not used - * r10 --- to - * r11 --- from - */ -ENTRY(copy_page_slow) - mov.l r8,@-r15 - mov.l r10,@-r15 - mov.l r11,@-r15 - mov r4,r10 - mov r5,r11 - mov r5,r8 - mov.l .Lpsz,r0 - add r0,r8 - ! -1: mov.l @r11+,r0 - mov.l @r11+,r1 - mov.l @r11+,r2 - mov.l @r11+,r3 - mov.l @r11+,r4 - mov.l @r11+,r5 - mov.l @r11+,r6 - mov.l @r11+,r7 -#if defined(CONFIG_CPU_SH3) - mov.l r0,@r10 -#elif defined(CONFIG_CPU_SH4) - movca.l r0,@r10 - mov r10,r0 -#endif - add #32,r10 - mov.l r7,@-r10 - mov.l r6,@-r10 - mov.l r5,@-r10 - mov.l r4,@-r10 - mov.l r3,@-r10 - mov.l r2,@-r10 - mov.l r1,@-r10 -#if defined(CONFIG_CPU_SH4) - ocbwb @r0 -#endif - cmp/eq r11,r8 - bf/s 1b - add #28,r10 - ! - mov.l @r15+,r11 - mov.l @r15+,r10 - mov.l @r15+,r8 - rts - nop - - .align 2 -.Lpsz: .long PAGE_SIZE -/* - * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n); - * Return the number of bytes NOT copied - */ -#define EX(...) \ - 9999: __VA_ARGS__ ; \ - .section __ex_table, "a"; \ - .long 9999b, 6000f ; \ - .previous -ENTRY(__copy_user) - ! Check if small number of bytes - mov #11,r0 - mov r4,r3 - cmp/gt r0,r6 ! r6 (len) > r0 (11) - bf/s .L_cleanup_loop_no_pop - add r6,r3 ! last destination address - - ! Calculate bytes needed to align to src - mov.l r11,@-r15 - neg r5,r0 - mov.l r10,@-r15 - add #4,r0 - mov.l r9,@-r15 - and #3,r0 - mov.l r8,@-r15 - tst r0,r0 - bt 2f - -1: - ! Copy bytes to long word align src -EX( mov.b @r5+,r1 ) - dt r0 - add #-1,r6 -EX( mov.b r1,@r4 ) - bf/s 1b - add #1,r4 - - ! Jump to appropriate routine depending on dest -2: mov #3,r1 - mov r6, r2 - and r4,r1 - shlr2 r2 - shll2 r1 - mova .L_jump_tbl,r0 - mov.l @(r0,r1),r1 - jmp @r1 - nop - - .align 2 -.L_jump_tbl: - .long .L_dest00 - .long .L_dest01 - .long .L_dest10 - .long .L_dest11 - -/* - * Come here if there are less than 12 bytes to copy - * - * Keep the branch target close, so the bf/s callee doesn't overflow - * and result in a more expensive branch being inserted. This is the - * fast-path for small copies, the jump via the jump table will hit the - * default slow-path cleanup. -PFM. - */ -.L_cleanup_loop_no_pop: - tst r6,r6 ! Check explicitly for zero - bt 1f - -2: -EX( mov.b @r5+,r0 ) - dt r6 -EX( mov.b r0,@r4 ) - bf/s 2b - add #1,r4 - -1: mov #0,r0 ! normal return -5000: - -# Exception handler: -.section .fixup, "ax" -6000: - mov.l 8000f,r1 - mov r3,r0 - jmp @r1 - sub r4,r0 - .align 2 -8000: .long 5000b - -.previous - rts - nop - -! Destination = 00 - -.L_dest00: - ! Skip the large copy for small transfers - mov #(32+32-4), r0 - cmp/gt r6, r0 ! r0 (60) > r6 (len) - bt 1f - - ! Align dest to a 32 byte boundary - neg r4,r0 - add #0x20, r0 - and #0x1f, r0 - tst r0, r0 - bt 2f - - sub r0, r6 - shlr2 r0 -3: -EX( mov.l @r5+,r1 ) - dt r0 -EX( mov.l r1,@r4 ) - bf/s 3b - add #4,r4 - -2: -EX( mov.l @r5+,r0 ) -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r2 ) -EX( mov.l @r5+,r7 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r9 ) -EX( mov.l @r5+,r10 ) -EX( mov.l @r5+,r11 ) -#ifdef CONFIG_CPU_SH4 -EX( movca.l r0,@r4 ) -#else -EX( mov.l r0,@r4 ) -#endif - add #-32, r6 -EX( mov.l r1,@(4,r4) ) - mov #32, r0 -EX( mov.l r2,@(8,r4) ) - cmp/gt r6, r0 ! r0 (32) > r6 (len) -EX( mov.l r7,@(12,r4) ) -EX( mov.l r8,@(16,r4) ) -EX( mov.l r9,@(20,r4) ) -EX( mov.l r10,@(24,r4) ) -EX( mov.l r11,@(28,r4) ) - bf/s 2b - add #32,r4 - -1: mov r6, r0 - shlr2 r0 - tst r0, r0 - bt .L_cleanup -1: -EX( mov.l @r5+,r1 ) - dt r0 -EX( mov.l r1,@r4 ) - bf/s 1b - add #4,r4 - - bra .L_cleanup - nop - -! Destination = 10 - -.L_dest10: - mov r2,r7 - shlr2 r7 - shlr r7 - tst r7,r7 - mov #7,r0 - bt/s 1f - and r0,r2 -2: - dt r7 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.l @r5+,r0 ) -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r9 ) -EX( mov.l @r5+,r10 ) -EX( mov.w r0,@r4 ) - add #2,r4 - xtrct r1,r0 - xtrct r8,r1 - xtrct r9,r8 - xtrct r10,r9 - -EX( mov.l r0,@r4 ) -EX( mov.l r1,@(4,r4) ) -EX( mov.l r8,@(8,r4) ) -EX( mov.l r9,@(12,r4) ) - -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r0 ) - xtrct r1,r10 - xtrct r8,r1 - xtrct r0,r8 - shlr16 r0 -EX( mov.l r10,@(16,r4) ) -EX( mov.l r1,@(20,r4) ) -EX( mov.l r8,@(24,r4) ) -EX( mov.w r0,@(28,r4) ) - bf/s 2b - add #30,r4 -#else -EX( mov.l @(28,r5),r0 ) -EX( mov.l @(24,r5),r8 ) -EX( mov.l @(20,r5),r9 ) -EX( mov.l @(16,r5),r10 ) -EX( mov.w r0,@(30,r4) ) - add #-2,r4 - xtrct r8,r0 - xtrct r9,r8 - xtrct r10,r9 -EX( mov.l r0,@(28,r4) ) -EX( mov.l r8,@(24,r4) ) -EX( mov.l r9,@(20,r4) ) - -EX( mov.l @(12,r5),r0 ) -EX( mov.l @(8,r5),r8 ) - xtrct r0,r10 -EX( mov.l @(4,r5),r9 ) - mov.l r10,@(16,r4) -EX( mov.l @r5,r10 ) - xtrct r8,r0 - xtrct r9,r8 - xtrct r10,r9 -EX( mov.l r0,@(12,r4) ) -EX( mov.l r8,@(8,r4) ) - swap.w r10,r0 -EX( mov.l r9,@(4,r4) ) -EX( mov.w r0,@(2,r4) ) - - add #32,r5 - bf/s 2b - add #34,r4 -#endif - tst r2,r2 - bt .L_cleanup - -1: ! Read longword, write two words per iteration -EX( mov.l @r5+,r0 ) - dt r2 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.w r0,@r4 ) - shlr16 r0 -EX( mov.w r0,@(2,r4) ) -#else -EX( mov.w r0,@(2,r4) ) - shlr16 r0 -EX( mov.w r0,@r4 ) -#endif - bf/s 1b - add #4,r4 - - bra .L_cleanup - nop - -! Destination = 01 or 11 - -.L_dest01: -.L_dest11: - ! Read longword, write byte, word, byte per iteration -EX( mov.l @r5+,r0 ) - dt r2 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.b r0,@r4 ) - shlr8 r0 - add #1,r4 -EX( mov.w r0,@r4 ) - shlr16 r0 -EX( mov.b r0,@(2,r4) ) - bf/s .L_dest01 - add #3,r4 -#else -EX( mov.b r0,@(3,r4) ) - shlr8 r0 - swap.w r0,r7 -EX( mov.b r7,@r4 ) - add #1,r4 -EX( mov.w r0,@r4 ) - bf/s .L_dest01 - add #3,r4 -#endif - -! Cleanup last few bytes -.L_cleanup: - mov r6,r0 - and #3,r0 - tst r0,r0 - bt .L_exit - mov r0,r6 - -.L_cleanup_loop: -EX( mov.b @r5+,r0 ) - dt r6 -EX( mov.b r0,@r4 ) - bf/s .L_cleanup_loop - add #1,r4 - -.L_exit: - mov #0,r0 ! normal return - -5000: - -# Exception handler: -.section .fixup, "ax" -6000: - mov.l 8000f,r1 - mov r3,r0 - jmp @r1 - sub r4,r0 - .align 2 -8000: .long 5000b - -.previous - mov.l @r15+,r8 - mov.l @r15+,r9 - mov.l @r15+,r10 - rts - mov.l @r15+,r11 diff --git a/arch/sh/mm/extable.c b/arch/sh/mm/extable_32.c index c1cf4463d09d..c1cf4463d09d 100644 --- a/arch/sh/mm/extable.c +++ b/arch/sh/mm/extable_32.c diff --git a/arch/sh/mm/extable_64.c b/arch/sh/mm/extable_64.c new file mode 100644 index 000000000000..f05499688d88 --- /dev/null +++ b/arch/sh/mm/extable_64.c @@ -0,0 +1,82 @@ +/* + * arch/sh/mm/extable_64.c + * + * Copyright (C) 2003 Richard Curnow + * Copyright (C) 2003, 2004 Paul Mundt + * + * Cloned from the 2.5 SH version.. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/rwsem.h> +#include <linux/module.h> +#include <asm/uaccess.h> + +extern unsigned long copy_user_memcpy, copy_user_memcpy_end; +extern void __copy_user_fixup(void); + +static const struct exception_table_entry __copy_user_fixup_ex = { + .fixup = (unsigned long)&__copy_user_fixup, +}; + +/* + * Some functions that may trap due to a bad user-mode address have too + * many loads and stores in them to make it at all practical to label + * each one and put them all in the main exception table. + * + * In particular, the fast memcpy routine is like this. It's fix-up is + * just to fall back to a slow byte-at-a-time copy, which is handled the + * conventional way. So it's functionally OK to just handle any trap + * occurring in the fast memcpy with that fixup. + */ +static const struct exception_table_entry *check_exception_ranges(unsigned long addr) +{ + if ((addr >= (unsigned long)©_user_memcpy) && + (addr <= (unsigned long)©_user_memcpy_end)) + return &__copy_user_fixup_ex; + + return NULL; +} + +/* Simple binary search */ +const struct exception_table_entry * +search_extable(const struct exception_table_entry *first, + const struct exception_table_entry *last, + unsigned long value) +{ + const struct exception_table_entry *mid; + + mid = check_exception_ranges(value); + if (mid) + return mid; + + while (first <= last) { + long diff; + + mid = (last - first) / 2 + first; + diff = mid->insn - value; + if (diff == 0) + return mid; + else if (diff < 0) + first = mid+1; + else + last = mid-1; + } + + return NULL; +} + +int fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return 1; + } + + return 0; +} diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault_32.c index 60d74f793a1d..33b43d20e9f6 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault_32.c @@ -172,7 +172,7 @@ no_context: bust_spinlocks(1); if (oops_may_print()) { - __typeof__(pte_val(__pte(0))) page; + unsigned long page; if (address < PAGE_SIZE) printk(KERN_ALERT "Unable to handle kernel NULL " diff --git a/arch/sh/mm/fault_64.c b/arch/sh/mm/fault_64.c new file mode 100644 index 000000000000..399d53710d2f --- /dev/null +++ b/arch/sh/mm/fault_64.c @@ -0,0 +1,275 @@ +/* + * The SH64 TLB miss. + * + * Original code from fault.c + * Copyright (C) 2000, 2001 Paolo Alberelli + * + * Fast PTE->TLB refill path + * Copyright (C) 2003 Richard.Curnow@superh.com + * + * IMPORTANT NOTES : + * The do_fast_page_fault function is called from a context in entry.S + * where very few registers have been saved. In particular, the code in + * this file must be compiled not to use ANY caller-save registers that + * are not part of the restricted save set. Also, it means that code in + * this file must not make calls to functions elsewhere in the kernel, or + * else the excepting context will see corruption in its caller-save + * registers. Plus, the entry.S save area is non-reentrant, so this code + * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic + * on any exception. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/mman.h> +#include <linux/mm.h> +#include <linux/smp.h> +#include <linux/interrupt.h> +#include <asm/system.h> +#include <asm/tlb.h> +#include <asm/io.h> +#include <asm/uaccess.h> +#include <asm/pgalloc.h> +#include <asm/mmu_context.h> +#include <asm/cpu/registers.h> + +/* Callable from fault.c, so not static */ +inline void __do_tlb_refill(unsigned long address, + unsigned long long is_text_not_data, pte_t *pte) +{ + unsigned long long ptel; + unsigned long long pteh=0; + struct tlb_info *tlbp; + unsigned long long next; + + /* Get PTEL first */ + ptel = pte_val(*pte); + + /* + * Set PTEH register + */ + pteh = address & MMU_VPN_MASK; + + /* Sign extend based on neff. */ +#if (NEFF == 32) + /* Faster sign extension */ + pteh = (unsigned long long)(signed long long)(signed long)pteh; +#else + /* General case */ + pteh = (pteh & NEFF_SIGN) ? (pteh | NEFF_MASK) : pteh; +#endif + + /* Set the ASID. */ + pteh |= get_asid() << PTEH_ASID_SHIFT; + pteh |= PTEH_VALID; + + /* Set PTEL register, set_pte has performed the sign extension */ + ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */ + + tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb); + next = tlbp->next; + __flush_tlb_slot(next); + asm volatile ("putcfg %0,1,%2\n\n\t" + "putcfg %0,0,%1\n" + : : "r" (next), "r" (pteh), "r" (ptel) ); + + next += TLB_STEP; + if (next > tlbp->last) next = tlbp->first; + tlbp->next = next; + +} + +static int handle_vmalloc_fault(struct mm_struct *mm, + unsigned long protection_flags, + unsigned long long textaccess, + unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + static pte_t *pte; + pte_t entry; + + dir = pgd_offset_k(address); + + pud = pud_offset(dir, address); + if (pud_none_or_clear_bad(pud)) + return 0; + + pmd = pmd_offset(pud, address); + if (pmd_none_or_clear_bad(pmd)) + return 0; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + + if (pte_none(entry) || !pte_present(entry)) + return 0; + if ((pte_val(entry) & protection_flags) != protection_flags) + return 0; + + __do_tlb_refill(address, textaccess, pte); + + return 1; +} + +static int handle_tlbmiss(struct mm_struct *mm, + unsigned long long protection_flags, + unsigned long long textaccess, + unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + + /* NB. The PGD currently only contains a single entry - there is no + page table tree stored for the top half of the address space since + virtual pages in that region should never be mapped in user mode. + (In kernel mode, the only things in that region are the 512Mb super + page (locked in), and vmalloc (modules) + I/O device pages (handled + by handle_vmalloc_fault), so no PGD for the upper half is required + by kernel mode either). + + See how mm->pgd is allocated and initialised in pgd_alloc to see why + the next test is necessary. - RPC */ + if (address >= (unsigned long) TASK_SIZE) + /* upper half - never has page table entries. */ + return 0; + + dir = pgd_offset(mm, address); + if (pgd_none(*dir) || !pgd_present(*dir)) + return 0; + if (!pgd_present(*dir)) + return 0; + + pud = pud_offset(dir, address); + if (pud_none(*pud) || !pud_present(*pud)) + return 0; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd) || !pmd_present(*pmd)) + return 0; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + + if (pte_none(entry) || !pte_present(entry)) + return 0; + + /* + * If the page doesn't have sufficient protection bits set to + * service the kind of fault being handled, there's not much + * point doing the TLB refill. Punt the fault to the general + * handler. + */ + if ((pte_val(entry) & protection_flags) != protection_flags) + return 0; + + __do_tlb_refill(address, textaccess, pte); + + return 1; +} + +/* + * Put all this information into one structure so that everything is just + * arithmetic relative to a single base address. This reduces the number + * of movi/shori pairs needed just to load addresses of static data. + */ +struct expevt_lookup { + unsigned short protection_flags[8]; + unsigned char is_text_access[8]; + unsigned char is_write_access[8]; +}; + +#define PRU (1<<9) +#define PRW (1<<8) +#define PRX (1<<7) +#define PRR (1<<6) + +#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED) +#define YOUNG (_PAGE_ACCESSED) + +/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether + the fault happened in user mode or privileged mode. */ +static struct expevt_lookup expevt_lookup_table = { + .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW}, + .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0} +}; + +/* + This routine handles page faults that can be serviced just by refilling a + TLB entry from an existing page table entry. (This case represents a very + large majority of page faults.) Return 1 if the fault was successfully + handled. Return 0 if the fault could not be handled. (This leads into the + general fault handling in fault.c which deals with mapping file-backed + pages, stack growth, segmentation faults, swapping etc etc) + */ +asmlinkage int do_fast_page_fault(unsigned long long ssr_md, + unsigned long long expevt, + unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + unsigned long long textaccess; + unsigned long long protection_flags; + unsigned long long index; + unsigned long long expevt4; + + /* The next few lines implement a way of hashing EXPEVT into a + * small array index which can be used to lookup parameters + * specific to the type of TLBMISS being handled. + * + * Note: + * ITLBMISS has EXPEVT==0xa40 + * RTLBMISS has EXPEVT==0x040 + * WTLBMISS has EXPEVT==0x060 + */ + expevt4 = (expevt >> 4); + /* TODO : xor ssr_md into this expression too. Then we can check + * that PRU is set when it needs to be. */ + index = expevt4 ^ (expevt4 >> 5); + index &= 7; + protection_flags = expevt_lookup_table.protection_flags[index]; + textaccess = expevt_lookup_table.is_text_access[index]; + + /* SIM + * Note this is now called with interrupts still disabled + * This is to cope with being called for a missing IO port + * address with interrupts disabled. This should be fixed as + * soon as we have a better 'fast path' miss handler. + * + * Plus take care how you try and debug this stuff. + * For example, writing debug data to a port which you + * have just faulted on is not going to work. + */ + + tsk = current; + mm = tsk->mm; + + if ((address >= VMALLOC_START && address < VMALLOC_END) || + (address >= IOBASE_VADDR && address < IOBASE_END)) { + if (ssr_md) + /* + * Process-contexts can never have this address + * range mapped + */ + if (handle_vmalloc_fault(mm, protection_flags, + textaccess, address)) + return 1; + } else if (!in_interrupt() && mm) { + if (handle_tlbmiss(mm, protection_flags, textaccess, address)) + return 1; + } + + return 0; +} diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index d5e160da64b2..2918c6b14659 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -23,9 +23,7 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); pgd_t swapper_pg_dir[PTRS_PER_PGD]; - -void (*copy_page)(void *from, void *to); -void (*clear_page)(void *to); +unsigned long cached_to_uncached = 0; void show_mem(void) { @@ -102,7 +100,8 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot) set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot)); - flush_tlb_one(get_asid(), addr); + if (cached_to_uncached) + flush_tlb_one(get_asid(), addr); } /* @@ -131,6 +130,37 @@ void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot) set_pte_phys(address, phys, prot); } + +void __init page_table_range_init(unsigned long start, unsigned long end, + pgd_t *pgd_base) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + int pgd_idx; + unsigned long vaddr; + + vaddr = start & PMD_MASK; + end = (end + PMD_SIZE - 1) & PMD_MASK; + pgd_idx = pgd_index(vaddr); + pgd = pgd_base + pgd_idx; + + for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { + BUG_ON(pgd_none(*pgd)); + pud = pud_offset(pgd, 0); + BUG_ON(pud_none(*pud)); + pmd = pmd_offset(pud, 0); + + if (!pmd_present(*pmd)) { + pte_t *pte_table; + pte_table = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); + memset(pte_table, 0, PAGE_SIZE); + pmd_populate_kernel(&init_mm, pmd, pte_table); + } + + vaddr += PMD_SIZE; + } +} #endif /* CONFIG_MMU */ /* @@ -150,6 +180,11 @@ void __init paging_init(void) * check for a null value. */ set_TTB(swapper_pg_dir); + /* Populate the relevant portions of swapper_pg_dir so that + * we can use the fixmap entries without calling kmalloc. + * pte's will be filled in by __set_fixmap(). */ + page_table_range_init(FIXADDR_START, FIXADDR_TOP, swapper_pg_dir); + memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); for_each_online_node(nid) { @@ -167,9 +202,22 @@ void __init paging_init(void) } free_area_init_nodes(max_zone_pfns); + + /* Set up the uncached fixmap */ + set_fixmap_nocache(FIX_UNCACHED, __pa(&__uncached_start)); + +#ifdef CONFIG_29BIT + /* + * Handle trivial transitions between cached and uncached + * segments, making use of the 1:1 mapping relationship in + * 512MB lowmem. + */ + cached_to_uncached = P2SEG - P1SEG; +#endif } static struct kcore_list kcore_mem, kcore_vmalloc; +int after_bootmem = 0; void __init mem_init(void) { @@ -202,17 +250,7 @@ void __init mem_init(void) memset(empty_zero_page, 0, PAGE_SIZE); __flush_wback_region(empty_zero_page, PAGE_SIZE); - /* - * Setup wrappers for copy/clear_page(), these will get overridden - * later in the boot process if a better method is available. - */ -#ifdef CONFIG_MMU - copy_page = copy_page_slow; - clear_page = clear_page_slow; -#else - copy_page = copy_page_nommu; - clear_page = clear_page_nommu; -#endif + after_bootmem = 1; codesize = (unsigned long) &_etext - (unsigned long) &_text; datasize = (unsigned long) &_edata - (unsigned long) &_etext; diff --git a/arch/sh/mm/ioremap.c b/arch/sh/mm/ioremap_32.c index 0c7b7e33abdc..0c7b7e33abdc 100644 --- a/arch/sh/mm/ioremap.c +++ b/arch/sh/mm/ioremap_32.c diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c new file mode 100644 index 000000000000..e27d16519235 --- /dev/null +++ b/arch/sh/mm/ioremap_64.c @@ -0,0 +1,404 @@ +/* + * arch/sh/mm/ioremap_64.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2003 - 2007 Paul Mundt + * + * Mostly derived from arch/sh/mm/ioremap.c which, in turn is mostly + * derived from arch/i386/mm/ioremap.c . + * + * (C) Copyright 1995 1996 Linus Torvalds + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/vmalloc.h> +#include <linux/ioport.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/io.h> +#include <linux/bootmem.h> +#include <linux/proc_fs.h> +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/addrspace.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/mmu.h> + +static void shmedia_mapioaddr(unsigned long, unsigned long); +static unsigned long shmedia_ioremap(struct resource *, u32, int); + +/* + * Generic mapping function (not visible outside): + */ + +/* + * Remap an arbitrary physical address space into the kernel virtual + * address space. Needed when the kernel wants to access high addresses + * directly. + * + * NOTE! We need to allow non-page-aligned mappings too: we will obviously + * have to convert them into an offset in a page-aligned mapping, but the + * caller shouldn't need to know that small detail. + */ +void *__ioremap(unsigned long phys_addr, unsigned long size, + unsigned long flags) +{ + void * addr; + struct vm_struct * area; + unsigned long offset, last_addr; + pgprot_t pgprot; + + /* Don't allow wraparound or zero size */ + last_addr = phys_addr + size - 1; + if (!size || last_addr < phys_addr) + return NULL; + + pgprot = __pgprot(_PAGE_PRESENT | _PAGE_READ | + _PAGE_WRITE | _PAGE_DIRTY | + _PAGE_ACCESSED | _PAGE_SHARED | flags); + + /* + * Mappings have to be page-aligned + */ + offset = phys_addr & ~PAGE_MASK; + phys_addr &= PAGE_MASK; + size = PAGE_ALIGN(last_addr + 1) - phys_addr; + + /* + * Ok, go for it.. + */ + area = get_vm_area(size, VM_IOREMAP); + pr_debug("Get vm_area returns %p addr %p\n",area,area->addr); + if (!area) + return NULL; + area->phys_addr = phys_addr; + addr = area->addr; + if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size, + phys_addr, pgprot)) { + vunmap(addr); + return NULL; + } + return (void *) (offset + (char *)addr); +} +EXPORT_SYMBOL(__ioremap); + +void __iounmap(void *addr) +{ + struct vm_struct *area; + + vfree((void *) (PAGE_MASK & (unsigned long) addr)); + area = remove_vm_area((void *) (PAGE_MASK & (unsigned long) addr)); + if (!area) { + printk(KERN_ERR "iounmap: bad address %p\n", addr); + return; + } + + kfree(area); +} +EXPORT_SYMBOL(__iounmap); + +static struct resource shmedia_iomap = { + .name = "shmedia_iomap", + .start = IOBASE_VADDR + PAGE_SIZE, + .end = IOBASE_END - 1, +}; + +static void shmedia_mapioaddr(unsigned long pa, unsigned long va); +static void shmedia_unmapioaddr(unsigned long vaddr); +static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz); + +/* + * We have the same problem as the SPARC, so lets have the same comment: + * Our mini-allocator... + * Boy this is gross! We need it because we must map I/O for + * timers and interrupt controller before the kmalloc is available. + */ + +#define XNMLN 15 +#define XNRES 10 + +struct xresource { + struct resource xres; /* Must be first */ + int xflag; /* 1 == used */ + char xname[XNMLN+1]; +}; + +static struct xresource xresv[XNRES]; + +static struct xresource *xres_alloc(void) +{ + struct xresource *xrp; + int n; + + xrp = xresv; + for (n = 0; n < XNRES; n++) { + if (xrp->xflag == 0) { + xrp->xflag = 1; + return xrp; + } + xrp++; + } + return NULL; +} + +static void xres_free(struct xresource *xrp) +{ + xrp->xflag = 0; +} + +static struct resource *shmedia_find_resource(struct resource *root, + unsigned long vaddr) +{ + struct resource *res; + + for (res = root->child; res; res = res->sibling) + if (res->start <= vaddr && res->end >= vaddr) + return res; + + return NULL; +} + +static unsigned long shmedia_alloc_io(unsigned long phys, unsigned long size, + const char *name) +{ + static int printed_full = 0; + struct xresource *xres; + struct resource *res; + char *tack; + int tlen; + + if (name == NULL) name = "???"; + + if ((xres = xres_alloc()) != 0) { + tack = xres->xname; + res = &xres->xres; + } else { + if (!printed_full) { + printk("%s: done with statics, switching to kmalloc\n", + __FUNCTION__); + printed_full = 1; + } + tlen = strlen(name); + tack = kmalloc(sizeof (struct resource) + tlen + 1, GFP_KERNEL); + if (!tack) + return -ENOMEM; + memset(tack, 0, sizeof(struct resource)); + res = (struct resource *) tack; + tack += sizeof (struct resource); + } + + strncpy(tack, name, XNMLN); + tack[XNMLN] = 0; + res->name = tack; + + return shmedia_ioremap(res, phys, size); +} + +static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz) +{ + unsigned long offset = ((unsigned long) pa) & (~PAGE_MASK); + unsigned long round_sz = (offset + sz + PAGE_SIZE-1) & PAGE_MASK; + unsigned long va; + unsigned int psz; + + if (allocate_resource(&shmedia_iomap, res, round_sz, + shmedia_iomap.start, shmedia_iomap.end, + PAGE_SIZE, NULL, NULL) != 0) { + panic("alloc_io_res(%s): cannot occupy\n", + (res->name != NULL)? res->name: "???"); + } + + va = res->start; + pa &= PAGE_MASK; + + psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE; + + /* log at boot time ... */ + printk("mapioaddr: %6s [%2d page%s] va 0x%08lx pa 0x%08x\n", + ((res->name != NULL) ? res->name : "???"), + psz, psz == 1 ? " " : "s", va, pa); + + for (psz = res->end - res->start + 1; psz != 0; psz -= PAGE_SIZE) { + shmedia_mapioaddr(pa, va); + va += PAGE_SIZE; + pa += PAGE_SIZE; + } + + res->start += offset; + res->end = res->start + sz - 1; /* not strictly necessary.. */ + + return res->start; +} + +static void shmedia_free_io(struct resource *res) +{ + unsigned long len = res->end - res->start + 1; + + BUG_ON((len & (PAGE_SIZE - 1)) != 0); + + while (len) { + len -= PAGE_SIZE; + shmedia_unmapioaddr(res->start + len); + } + + release_resource(res); +} + +static __init_refok void *sh64_get_page(void) +{ + extern int after_bootmem; + void *page; + + if (after_bootmem) { + page = (void *)get_zeroed_page(GFP_ATOMIC); + } else { + page = alloc_bootmem_pages(PAGE_SIZE); + } + + if (!page || ((unsigned long)page & ~PAGE_MASK)) + panic("sh64_get_page: Out of memory already?\n"); + + return page; +} + +static void shmedia_mapioaddr(unsigned long pa, unsigned long va) +{ + pgd_t *pgdp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep, pte; + pgprot_t prot; + unsigned long flags = 1; /* 1 = CB0-1 device */ + + pr_debug("shmedia_mapiopage pa %08lx va %08lx\n", pa, va); + + pgdp = pgd_offset_k(va); + if (pgd_none(*pgdp) || !pgd_present(*pgdp)) { + pudp = (pud_t *)sh64_get_page(); + set_pgd(pgdp, __pgd((unsigned long)pudp | _KERNPG_TABLE)); + } + + pudp = pud_offset(pgdp, va); + if (pud_none(*pudp) || !pud_present(*pudp)) { + pmdp = (pmd_t *)sh64_get_page(); + set_pud(pudp, __pud((unsigned long)pmdp | _KERNPG_TABLE)); + } + + pmdp = pmd_offset(pudp, va); + if (pmd_none(*pmdp) || !pmd_present(*pmdp) ) { + ptep = (pte_t *)sh64_get_page(); + set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE)); + } + + prot = __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | + _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SHARED | flags); + + pte = pfn_pte(pa >> PAGE_SHIFT, prot); + ptep = pte_offset_kernel(pmdp, va); + + if (!pte_none(*ptep) && + pte_val(*ptep) != pte_val(pte)) + pte_ERROR(*ptep); + + set_pte(ptep, pte); + + flush_tlb_kernel_range(va, PAGE_SIZE); +} + +static void shmedia_unmapioaddr(unsigned long vaddr) +{ + pgd_t *pgdp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + pgdp = pgd_offset_k(vaddr); + if (pgd_none(*pgdp) || pgd_bad(*pgdp)) + return; + + pudp = pud_offset(pgdp, vaddr); + if (pud_none(*pudp) || pud_bad(*pudp)) + return; + + pmdp = pmd_offset(pudp, vaddr); + if (pmd_none(*pmdp) || pmd_bad(*pmdp)) + return; + + ptep = pte_offset_kernel(pmdp, vaddr); + + if (pte_none(*ptep) || !pte_present(*ptep)) + return; + + clear_page((void *)ptep); + pte_clear(&init_mm, vaddr, ptep); +} + +unsigned long onchip_remap(unsigned long phys, unsigned long size, const char *name) +{ + if (size < PAGE_SIZE) + size = PAGE_SIZE; + + return shmedia_alloc_io(phys, size, name); +} + +void onchip_unmap(unsigned long vaddr) +{ + struct resource *res; + unsigned int psz; + + res = shmedia_find_resource(&shmedia_iomap, vaddr); + if (!res) { + printk(KERN_ERR "%s: Failed to free 0x%08lx\n", + __FUNCTION__, vaddr); + return; + } + + psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE; + + printk(KERN_DEBUG "unmapioaddr: %6s [%2d page%s] freed\n", + res->name, psz, psz == 1 ? " " : "s"); + + shmedia_free_io(res); + + if ((char *)res >= (char *)xresv && + (char *)res < (char *)&xresv[XNRES]) { + xres_free((struct xresource *)res); + } else { + kfree(res); + } +} + +#ifdef CONFIG_PROC_FS +static int +ioremap_proc_info(char *buf, char **start, off_t fpos, int length, int *eof, + void *data) +{ + char *p = buf, *e = buf + length; + struct resource *r; + const char *nm; + + for (r = ((struct resource *)data)->child; r != NULL; r = r->sibling) { + if (p + 32 >= e) /* Better than nothing */ + break; + if ((nm = r->name) == 0) nm = "???"; + p += sprintf(p, "%08lx-%08lx: %s\n", + (unsigned long)r->start, + (unsigned long)r->end, nm); + } + + return p-buf; +} +#endif /* CONFIG_PROC_FS */ + +static int __init register_proc_onchip(void) +{ +#ifdef CONFIG_PROC_FS + create_proc_read_entry("io_map",0,0, ioremap_proc_info, &shmedia_iomap); +#endif + return 0; +} + +__initcall(register_proc_onchip); diff --git a/arch/sh/mm/pg-nommu.c b/arch/sh/mm/pg-nommu.c index d15221beaa16..677dd57f0877 100644 --- a/arch/sh/mm/pg-nommu.c +++ b/arch/sh/mm/pg-nommu.c @@ -14,12 +14,12 @@ #include <linux/string.h> #include <asm/page.h> -void copy_page_nommu(void *to, void *from) +void copy_page(void *to, void *from) { memcpy(to, from, PAGE_SIZE); } -void clear_page_nommu(void *to) +void clear_page(void *to) { memset(to, 0, PAGE_SIZE); } diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index 1d45b82f0a63..ab81c602295f 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -27,6 +27,7 @@ #include <asm/pgtable.h> #include <asm/mmu.h> #include <asm/io.h> +#include <asm/mmu_context.h> #define NR_PMB_ENTRIES 16 @@ -162,18 +163,18 @@ repeat: return 0; } -int set_pmb_entry(struct pmb_entry *pmbe) +int __uses_jump_to_uncached set_pmb_entry(struct pmb_entry *pmbe) { int ret; - jump_to_P2(); + jump_to_uncached(); ret = __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, &pmbe->entry); - back_to_P1(); + back_to_cached(); return ret; } -void clear_pmb_entry(struct pmb_entry *pmbe) +void __uses_jump_to_uncached clear_pmb_entry(struct pmb_entry *pmbe) { unsigned int entry = pmbe->entry; unsigned long addr; @@ -187,7 +188,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe) entry >= NR_PMB_ENTRIES)) return; - jump_to_P2(); + jump_to_uncached(); /* Clear V-bit */ addr = mk_pmb_addr(entry); @@ -196,7 +197,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe) addr = mk_pmb_data(entry); ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr); - back_to_P1(); + back_to_cached(); clear_bit(entry, &pmb_map); } @@ -301,17 +302,17 @@ static void pmb_cache_ctor(struct kmem_cache *cachep, void *pmb) pmbe->entry = PMB_NO_ENTRY; } -static int __init pmb_init(void) +static int __uses_jump_to_uncached pmb_init(void) { unsigned int nr_entries = ARRAY_SIZE(pmb_init_map); - unsigned int entry; + unsigned int entry, i; BUG_ON(unlikely(nr_entries >= NR_PMB_ENTRIES)); pmb_cache = kmem_cache_create("pmb", sizeof(struct pmb_entry), 0, SLAB_PANIC, pmb_cache_ctor); - jump_to_P2(); + jump_to_uncached(); /* * Ordering is important, P2 must be mapped in the PMB before we @@ -329,7 +330,12 @@ static int __init pmb_init(void) /* PMB.SE and UB[7] */ ctrl_outl((1 << 31) | (1 << 7), PMB_PASCR); - back_to_P1(); + /* Flush out the TLB */ + i = ctrl_inl(MMUCR); + i |= MMUCR_TI; + ctrl_outl(i, MMUCR); + + back_to_cached(); return 0; } diff --git a/arch/sh/mm/tlb-nommu.c b/arch/sh/mm/tlb-nommu.c index 1ccca7c0532e..15111bc7ddd6 100644 --- a/arch/sh/mm/tlb-nommu.c +++ b/arch/sh/mm/tlb-nommu.c @@ -9,6 +9,7 @@ */ #include <linux/kernel.h> #include <linux/mm.h> +#include <asm/pgtable.h> /* * Nothing too terribly exciting here .. @@ -49,3 +50,12 @@ void update_mmu_cache(struct vm_area_struct * vma, { BUG(); } + +void __init page_table_range_init(unsigned long start, unsigned long end, + pgd_t *pgd_base) +{ +} + +void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot) +{ +} diff --git a/arch/sh/mm/tlb-sh4.c b/arch/sh/mm/tlb-sh4.c index 2d1dd6044307..f0c7b7397fa6 100644 --- a/arch/sh/mm/tlb-sh4.c +++ b/arch/sh/mm/tlb-sh4.c @@ -79,7 +79,8 @@ void update_mmu_cache(struct vm_area_struct * vma, local_irq_restore(flags); } -void local_flush_tlb_one(unsigned long asid, unsigned long page) +void __uses_jump_to_uncached local_flush_tlb_one(unsigned long asid, + unsigned long page) { unsigned long addr, data; @@ -91,7 +92,7 @@ void local_flush_tlb_one(unsigned long asid, unsigned long page) */ addr = MMU_UTLB_ADDRESS_ARRAY | MMU_PAGE_ASSOC_BIT; data = page | asid; /* VALID bit is off */ - jump_to_P2(); + jump_to_uncached(); ctrl_outl(data, addr); - back_to_P1(); + back_to_cached(); } diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c new file mode 100644 index 000000000000..f34274a1ded3 --- /dev/null +++ b/arch/sh/mm/tlb-sh5.c @@ -0,0 +1,164 @@ +/* + * arch/sh/mm/tlb-sh5.c + * + * Copyright (C) 2003 Paul Mundt <lethal@linux-sh.org> + * Copyright (C) 2003 Richard Curnow <richard.curnow@superh.com> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/mm.h> +#include <linux/init.h> +#include <asm/page.h> +#include <asm/tlb.h> +#include <asm/mmu_context.h> + +/** + * sh64_tlb_init + * + * Perform initial setup for the DTLB and ITLB. + */ +int __init sh64_tlb_init(void) +{ + /* Assign some sane DTLB defaults */ + cpu_data->dtlb.entries = 64; + cpu_data->dtlb.step = 0x10; + + cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step; + cpu_data->dtlb.next = cpu_data->dtlb.first; + + cpu_data->dtlb.last = DTLB_FIXED | + ((cpu_data->dtlb.entries - 1) * + cpu_data->dtlb.step); + + /* And again for the ITLB */ + cpu_data->itlb.entries = 64; + cpu_data->itlb.step = 0x10; + + cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step; + cpu_data->itlb.next = cpu_data->itlb.first; + cpu_data->itlb.last = ITLB_FIXED | + ((cpu_data->itlb.entries - 1) * + cpu_data->itlb.step); + + return 0; +} + +/** + * sh64_next_free_dtlb_entry + * + * Find the next available DTLB entry + */ +unsigned long long sh64_next_free_dtlb_entry(void) +{ + return cpu_data->dtlb.next; +} + +/** + * sh64_get_wired_dtlb_entry + * + * Allocate a wired (locked-in) entry in the DTLB + */ +unsigned long long sh64_get_wired_dtlb_entry(void) +{ + unsigned long long entry = sh64_next_free_dtlb_entry(); + + cpu_data->dtlb.first += cpu_data->dtlb.step; + cpu_data->dtlb.next += cpu_data->dtlb.step; + + return entry; +} + +/** + * sh64_put_wired_dtlb_entry + * + * @entry: Address of TLB slot. + * + * Free a wired (locked-in) entry in the DTLB. + * + * Works like a stack, last one to allocate must be first one to free. + */ +int sh64_put_wired_dtlb_entry(unsigned long long entry) +{ + __flush_tlb_slot(entry); + + /* + * We don't do any particularly useful tracking of wired entries, + * so this approach works like a stack .. last one to be allocated + * has to be the first one to be freed. + * + * We could potentially load wired entries into a list and work on + * rebalancing the list periodically (which also entails moving the + * contents of a TLB entry) .. though I have a feeling that this is + * more trouble than it's worth. + */ + + /* + * Entry must be valid .. we don't want any ITLB addresses! + */ + if (entry <= DTLB_FIXED) + return -EINVAL; + + /* + * Next, check if we're within range to be freed. (ie, must be the + * entry beneath the first 'free' entry! + */ + if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step)) + return -EINVAL; + + /* If we are, then bring this entry back into the list */ + cpu_data->dtlb.first -= cpu_data->dtlb.step; + cpu_data->dtlb.next = entry; + + return 0; +} + +/** + * sh64_setup_tlb_slot + * + * @config_addr: Address of TLB slot. + * @eaddr: Virtual address. + * @asid: Address Space Identifier. + * @paddr: Physical address. + * + * Load up a virtual<->physical translation for @eaddr<->@paddr in the + * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry). + */ +inline void sh64_setup_tlb_slot(unsigned long long config_addr, + unsigned long eaddr, + unsigned long asid, + unsigned long paddr) +{ + unsigned long long pteh, ptel; + + /* Sign extension */ +#if (NEFF == 32) + pteh = (unsigned long long)(signed long long)(signed long) eaddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + pteh &= PAGE_MASK; + pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID; +#if (NEFF == 32) + ptel = (unsigned long long)(signed long long)(signed long) paddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + ptel &= PAGE_MASK; + ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE); + + asm volatile("putcfg %0, 1, %1\n\t" + "putcfg %0, 0, %2\n" + : : "r" (config_addr), "r" (ptel), "r" (pteh)); +} + +/** + * sh64_teardown_tlb_slot + * + * @config_addr: Address of TLB slot. + * + * Teardown any existing mapping in the TLB slot @config_addr. + */ +inline void sh64_teardown_tlb_slot(unsigned long long config_addr) + __attribute__ ((alias("__flush_tlb_slot"))); diff --git a/arch/sh/mm/tlb-flush.c b/arch/sh/mm/tlbflush_32.c index 6f45c1f8a7fe..6f45c1f8a7fe 100644 --- a/arch/sh/mm/tlb-flush.c +++ b/arch/sh/mm/tlbflush_32.c diff --git a/arch/sh/mm/tlbflush_64.c b/arch/sh/mm/tlbflush_64.c new file mode 100644 index 000000000000..2a98c9ec88ff --- /dev/null +++ b/arch/sh/mm/tlbflush_64.c @@ -0,0 +1,475 @@ +/* + * arch/sh/mm/tlb-flush_64.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) + * Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/signal.h> +#include <linux/rwsem.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/mman.h> +#include <linux/mm.h> +#include <linux/smp.h> +#include <linux/interrupt.h> +#include <asm/system.h> +#include <asm/io.h> +#include <asm/tlb.h> +#include <asm/uaccess.h> +#include <asm/pgalloc.h> +#include <asm/mmu_context.h> + +extern void die(const char *,struct pt_regs *,long); + +#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) +#define PPROT(flag) PFLAG(pgprot_val(prot),flag) + +static inline void print_prots(pgprot_t prot) +{ + printk("prot is 0x%08lx\n",pgprot_val(prot)); + + printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), + PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); +} + +static inline void print_vma(struct vm_area_struct *vma) +{ + printk("vma start 0x%08lx\n", vma->vm_start); + printk("vma end 0x%08lx\n", vma->vm_end); + + print_prots(vma->vm_page_prot); + printk("vm_flags 0x%08lx\n", vma->vm_flags); +} + +static inline void print_task(struct task_struct *tsk) +{ + printk("Task pid %d\n", task_pid_nr(tsk)); +} + +static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + + dir = pgd_offset(mm, address); + if (pgd_none(*dir)) + return NULL; + + pud = pud_offset(dir, address); + if (pud_none(*pud)) + return NULL; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd)) + return NULL; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + return NULL; + + return pte; +} + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, + unsigned long textaccess, unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + struct vm_area_struct * vma; + const struct exception_table_entry *fixup; + pte_t *pte; + int fault; + + /* SIM + * Note this is now called with interrupts still disabled + * This is to cope with being called for a missing IO port + * address with interrupts disabled. This should be fixed as + * soon as we have a better 'fast path' miss handler. + * + * Plus take care how you try and debug this stuff. + * For example, writing debug data to a port which you + * have just faulted on is not going to work. + */ + + tsk = current; + mm = tsk->mm; + + /* Not an IO address, so reenable interrupts */ + local_irq_enable(); + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (in_atomic() || !mm) + goto no_context; + + /* TLB misses upon some cache flushes get done under cli() */ + down_read(&mm->mmap_sem); + + vma = find_vma(mm, address); + + if (!vma) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } + if (vma->vm_start <= address) { + goto good_area; + } + + if (!(vma->vm_flags & VM_GROWSDOWN)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); + + print_vma(vma); +#endif + goto bad_area; + } + if (expand_stack(vma, address)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + if (textaccess) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else { + if (writeaccess) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & VM_READ)) + goto bad_area; + } + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ +survive: + fault = handle_mm_fault(mm, vma, address, writeaccess); + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (fault & VM_FAULT_MAJOR) + tsk->maj_flt++; + else + tsk->min_flt++; + + /* If we get here, the page fault has been handled. Do the TLB refill + now from the newly-setup PTE, to avoid having to fault again right + away on the same instruction. */ + pte = lookup_pte (mm, address); + if (!pte) { + /* From empirical evidence, we can get here, due to + !pte_present(pte). (e.g. if a swap-in occurs, and the page + is swapped back out again before the process that wanted it + gets rescheduled?) */ + goto no_pte; + } + + __do_tlb_refill(address, textaccess, pte); + +no_pte: + + up_read(&mm->mmap_sem); + return; + +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: +#ifdef DEBUG_FAULT + printk("fault:bad area\n"); +#endif + up_read(&mm->mmap_sem); + + if (user_mode(regs)) { + static int count=0; + siginfo_t info; + if (count < 4) { + /* This is really to help debug faults when starting + * usermode, so only need a few */ + count++; + printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", + address, task_pid_nr(current), current->comm, + (unsigned long) regs->pc); +#if 0 + show_regs(regs); +#endif + } + if (is_global_init(tsk)) { + panic("INIT had user mode bad_area\n"); + } + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) address; + force_sig_info(SIGSEGV, &info, tsk); + return; + } + +no_context: +#ifdef DEBUG_FAULT + printk("fault:No context\n"); +#endif + /* Are we prepared to handle this kernel fault? */ + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return; + } + +/* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + * + */ + if (address < PAGE_SIZE) + printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); + else + printk(KERN_ALERT "Unable to handle kernel paging request"); + printk(" at virtual address %08lx\n", address); + printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); + die("Oops", regs, writeaccess); + do_exit(SIGKILL); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + if (is_global_init(current)) { + panic("INIT out of memory\n"); + yield(); + goto survive; + } + printk("fault:Out of memory\n"); + up_read(&mm->mmap_sem); + if (is_global_init(current)) { + yield(); + down_read(&mm->mmap_sem); + goto survive; + } + printk("VM: killing process %s\n", tsk->comm); + if (user_mode(regs)) + do_group_exit(SIGKILL); + goto no_context; + +do_sigbus: + printk("fault:Do sigbus\n"); + up_read(&mm->mmap_sem); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + tsk->thread.trap_no = 14; + force_sig(SIGBUS, tsk); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + goto no_context; +} + +void update_mmu_cache(struct vm_area_struct * vma, + unsigned long address, pte_t pte) +{ + /* + * This appears to get called once for every pte entry that gets + * established => I don't think it's efficient to try refilling the + * TLBs with the pages - some may not get accessed even. Also, for + * executable pages, it is impossible to determine reliably here which + * TLB they should be mapped into (or both even). + * + * So, just do nothing here and handle faults on demand. In the + * TLBMISS handling case, the refill is now done anyway after the pte + * has been fixed up, so that deals with most useful cases. + */ +} + +void local_flush_tlb_one(unsigned long asid, unsigned long page) +{ + unsigned long long match, pteh=0, lpage; + unsigned long tlb; + + /* + * Sign-extend based on neff. + */ + lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; + match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; + match |= lpage; + + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + } + + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + + } +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + unsigned long flags; + + if (vma->vm_mm) { + page &= PAGE_MASK; + local_irq_save(flags); + local_flush_tlb_one(get_asid(), page); + local_irq_restore(flags); + } +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + unsigned long flags; + unsigned long long match, pteh=0, pteh_epn, pteh_low; + unsigned long tlb; + unsigned int cpu = smp_processor_id(); + struct mm_struct *mm; + + mm = vma->vm_mm; + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + start &= PAGE_MASK; + end &= PAGE_MASK; + + match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; + + /* Flush ITLB */ + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + /* Flush DTLB */ + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + local_irq_restore(flags); +} + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned long flags; + unsigned int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + + local_irq_restore(flags); +} + +void local_flush_tlb_all(void) +{ + /* Invalidate all, including shared pages, excluding fixed TLBs */ + unsigned long flags, tlb; + + local_irq_save(flags); + + /* Flush each ITLB entry */ + for_each_itlb_entry(tlb) + __flush_tlb_slot(tlb); + + /* Flush each DTLB entry */ + for_each_dtlb_entry(tlb) + __flush_tlb_slot(tlb); + + local_irq_restore(flags); +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + /* FIXME: Optimize this later.. */ + flush_tlb_all(); +} |