aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig3
-rw-r--r--arch/arm/mm/Makefile1
-rw-r--r--arch/arm/mm/cache-l2x0.c158
-rw-r--r--arch/arm/mm/context.c64
-rw-r--r--arch/arm/mm/dma-mapping.c22
-rw-r--r--arch/arm/mm/fault.c2
-rw-r--r--arch/arm/mm/flush.c27
-rw-r--r--arch/arm/mm/fsr-3level.c4
-rw-r--r--arch/arm/mm/hugetlbpage.c101
-rw-r--r--arch/arm/mm/init.c77
-rw-r--r--arch/arm/mm/ioremap.c10
-rw-r--r--arch/arm/mm/mmu.c49
-rw-r--r--arch/arm/mm/nommu.c263
-rw-r--r--arch/arm/mm/proc-v6.S6
-rw-r--r--arch/arm/mm/proc-v7-3level.S53
-rw-r--r--arch/arm/mm/proc-v7.S27
16 files changed, 699 insertions, 168 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 9e8101ecd63e..6cacdc8dd654 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -392,7 +392,8 @@ config CPU_V7
select CPU_CACHE_V7
select CPU_CACHE_VIPT
select CPU_COPY_V6 if MMU
- select CPU_CP15_MMU
+ select CPU_CP15_MMU if MMU
+ select CPU_CP15_MPU if !MMU
select CPU_HAS_ASID if MMU
select CPU_PABRT_V7
select CPU_TLB_V7 if MMU
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index ee558a01f390..ecfe6e53f6e0 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_MODULES) += proc-syms.o
obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o
obj-$(CONFIG_HIGHMEM) += highmem.o
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_CPU_ABRT_NOMMU) += abort-nommu.o
obj-$(CONFIG_CPU_ABRT_EV4) += abort-ev4.o
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index c465faca51b0..d70e0aba0c9d 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -523,6 +523,147 @@ static void aurora_flush_range(unsigned long start, unsigned long end)
}
}
+/*
+ * For certain Broadcom SoCs, depending on the address range, different offsets
+ * need to be added to the address before passing it to L2 for
+ * invalidation/clean/flush
+ *
+ * Section Address Range Offset EMI
+ * 1 0x00000000 - 0x3FFFFFFF 0x80000000 VC
+ * 2 0x40000000 - 0xBFFFFFFF 0x40000000 SYS
+ * 3 0xC0000000 - 0xFFFFFFFF 0x80000000 VC
+ *
+ * When the start and end addresses have crossed two different sections, we
+ * need to break the L2 operation into two, each within its own section.
+ * For example, if we need to invalidate addresses starts at 0xBFFF0000 and
+ * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2)
+ * 0xC0000000 - 0xC0001000
+ *
+ * Note 1:
+ * By breaking a single L2 operation into two, we may potentially suffer some
+ * performance hit, but keep in mind the cross section case is very rare
+ *
+ * Note 2:
+ * We do not need to handle the case when the start address is in
+ * Section 1 and the end address is in Section 3, since it is not a valid use
+ * case
+ *
+ * Note 3:
+ * Section 1 in practical terms can no longer be used on rev A2. Because of
+ * that the code does not need to handle section 1 at all.
+ *
+ */
+#define BCM_SYS_EMI_START_ADDR 0x40000000UL
+#define BCM_VC_EMI_SEC3_START_ADDR 0xC0000000UL
+
+#define BCM_SYS_EMI_OFFSET 0x40000000UL
+#define BCM_VC_EMI_OFFSET 0x80000000UL
+
+static inline int bcm_addr_is_sys_emi(unsigned long addr)
+{
+ return (addr >= BCM_SYS_EMI_START_ADDR) &&
+ (addr < BCM_VC_EMI_SEC3_START_ADDR);
+}
+
+static inline unsigned long bcm_l2_phys_addr(unsigned long addr)
+{
+ if (bcm_addr_is_sys_emi(addr))
+ return addr + BCM_SYS_EMI_OFFSET;
+ else
+ return addr + BCM_VC_EMI_OFFSET;
+}
+
+static void bcm_inv_range(unsigned long start, unsigned long end)
+{
+ unsigned long new_start, new_end;
+
+ BUG_ON(start < BCM_SYS_EMI_START_ADDR);
+
+ if (unlikely(end <= start))
+ return;
+
+ new_start = bcm_l2_phys_addr(start);
+ new_end = bcm_l2_phys_addr(end);
+
+ /* normal case, no cross section between start and end */
+ if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
+ l2x0_inv_range(new_start, new_end);
+ return;
+ }
+
+ /* They cross sections, so it can only be a cross from section
+ * 2 to section 3
+ */
+ l2x0_inv_range(new_start,
+ bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
+ l2x0_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
+ new_end);
+}
+
+static void bcm_clean_range(unsigned long start, unsigned long end)
+{
+ unsigned long new_start, new_end;
+
+ BUG_ON(start < BCM_SYS_EMI_START_ADDR);
+
+ if (unlikely(end <= start))
+ return;
+
+ if ((end - start) >= l2x0_size) {
+ l2x0_clean_all();
+ return;
+ }
+
+ new_start = bcm_l2_phys_addr(start);
+ new_end = bcm_l2_phys_addr(end);
+
+ /* normal case, no cross section between start and end */
+ if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
+ l2x0_clean_range(new_start, new_end);
+ return;
+ }
+
+ /* They cross sections, so it can only be a cross from section
+ * 2 to section 3
+ */
+ l2x0_clean_range(new_start,
+ bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
+ l2x0_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
+ new_end);
+}
+
+static void bcm_flush_range(unsigned long start, unsigned long end)
+{
+ unsigned long new_start, new_end;
+
+ BUG_ON(start < BCM_SYS_EMI_START_ADDR);
+
+ if (unlikely(end <= start))
+ return;
+
+ if ((end - start) >= l2x0_size) {
+ l2x0_flush_all();
+ return;
+ }
+
+ new_start = bcm_l2_phys_addr(start);
+ new_end = bcm_l2_phys_addr(end);
+
+ /* normal case, no cross section between start and end */
+ if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
+ l2x0_flush_range(new_start, new_end);
+ return;
+ }
+
+ /* They cross sections, so it can only be a cross from section
+ * 2 to section 3
+ */
+ l2x0_flush_range(new_start,
+ bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
+ l2x0_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
+ new_end);
+}
+
static void __init l2x0_of_setup(const struct device_node *np,
u32 *aux_val, u32 *aux_mask)
{
@@ -765,6 +906,21 @@ static const struct l2x0_of_data aurora_no_outer_data = {
},
};
+static const struct l2x0_of_data bcm_l2x0_data = {
+ .setup = pl310_of_setup,
+ .save = pl310_save,
+ .outer_cache = {
+ .resume = pl310_resume,
+ .inv_range = bcm_inv_range,
+ .clean_range = bcm_clean_range,
+ .flush_range = bcm_flush_range,
+ .sync = l2x0_cache_sync,
+ .flush_all = l2x0_flush_all,
+ .inv_all = l2x0_inv_all,
+ .disable = l2x0_disable,
+ },
+};
+
static const struct of_device_id l2x0_ids[] __initconst = {
{ .compatible = "arm,pl310-cache", .data = (void *)&pl310_data },
{ .compatible = "arm,l220-cache", .data = (void *)&l2x0_data },
@@ -773,6 +929,8 @@ static const struct of_device_id l2x0_ids[] __initconst = {
.data = (void *)&aurora_no_outer_data},
{ .compatible = "marvell,aurora-outer-cache",
.data = (void *)&aurora_with_outer_data},
+ { .compatible = "bcm,bcm11351-a2-pl310-cache",
+ .data = (void *)&bcm_l2x0_data},
{}
};
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index 2ac37372ef52..b55b1015724b 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -20,6 +20,7 @@
#include <asm/smp_plat.h>
#include <asm/thread_notify.h>
#include <asm/tlbflush.h>
+#include <asm/proc-fns.h>
/*
* On ARMv6, we have the following structure in the Context ID:
@@ -39,33 +40,51 @@
* non 64-bit operations.
*/
#define ASID_FIRST_VERSION (1ULL << ASID_BITS)
-#define NUM_USER_ASIDS (ASID_FIRST_VERSION - 1)
-
-#define ASID_TO_IDX(asid) ((asid & ~ASID_MASK) - 1)
-#define IDX_TO_ASID(idx) ((idx + 1) & ~ASID_MASK)
+#define NUM_USER_ASIDS ASID_FIRST_VERSION
static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
-DEFINE_PER_CPU(atomic64_t, active_asids);
+static DEFINE_PER_CPU(atomic64_t, active_asids);
static DEFINE_PER_CPU(u64, reserved_asids);
static cpumask_t tlb_flush_pending;
+#ifdef CONFIG_ARM_ERRATA_798181
+void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
+ cpumask_t *mask)
+{
+ int cpu;
+ unsigned long flags;
+ u64 context_id, asid;
+
+ raw_spin_lock_irqsave(&cpu_asid_lock, flags);
+ context_id = mm->context.id.counter;
+ for_each_online_cpu(cpu) {
+ if (cpu == this_cpu)
+ continue;
+ /*
+ * We only need to send an IPI if the other CPUs are
+ * running the same ASID as the one being invalidated.
+ */
+ asid = per_cpu(active_asids, cpu).counter;
+ if (asid == 0)
+ asid = per_cpu(reserved_asids, cpu);
+ if (context_id == asid)
+ cpumask_set_cpu(cpu, mask);
+ }
+ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
+}
+#endif
+
#ifdef CONFIG_ARM_LPAE
static void cpu_set_reserved_ttbr0(void)
{
- unsigned long ttbl = __pa(swapper_pg_dir);
- unsigned long ttbh = 0;
-
/*
* Set TTBR0 to swapper_pg_dir which contains only global entries. The
* ASID is set to 0.
*/
- asm volatile(
- " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n"
- :
- : "r" (ttbl), "r" (ttbh));
+ cpu_set_ttbr(0, __pa(swapper_pg_dir));
isb();
}
#else
@@ -128,7 +147,16 @@ static void flush_context(unsigned int cpu)
asid = 0;
} else {
asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
- __set_bit(ASID_TO_IDX(asid), asid_map);
+ /*
+ * If this CPU has already been through a
+ * rollover, but hasn't run another task in
+ * the meantime, we must preserve its reserved
+ * ASID, as this is the only trace we have of
+ * the process it is still running.
+ */
+ if (asid == 0)
+ asid = per_cpu(reserved_asids, i);
+ __set_bit(asid & ~ASID_MASK, asid_map);
}
per_cpu(reserved_asids, i) = asid;
}
@@ -167,17 +195,19 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
/*
* Allocate a free ASID. If we can't find one, take a
* note of the currently active ASIDs and mark the TLBs
- * as requiring flushes.
+ * as requiring flushes. We always count from ASID #1,
+ * as we reserve ASID #0 to switch via TTBR0 and indicate
+ * rollover events.
*/
- asid = find_first_zero_bit(asid_map, NUM_USER_ASIDS);
+ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
if (asid == NUM_USER_ASIDS) {
generation = atomic64_add_return(ASID_FIRST_VERSION,
&asid_generation);
flush_context(cpu);
- asid = find_first_zero_bit(asid_map, NUM_USER_ASIDS);
+ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
}
__set_bit(asid, asid_map);
- asid = generation | IDX_TO_ASID(asid);
+ asid |= generation;
cpumask_clear(mm_cpumask(mm));
}
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index ef3e0f3aac96..7ec02961dfa0 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -250,7 +250,7 @@ static void __dma_free_buffer(struct page *page, size_t size)
#ifdef CONFIG_MMU
#ifdef CONFIG_HUGETLB_PAGE
-#error ARM Coherent DMA allocator does not (yet) support huge TLB
+#warning ARM Coherent DMA allocator does not (yet) support huge TLB
#endif
static void *__alloc_from_contiguous(struct device *dev, size_t size,
@@ -880,10 +880,24 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
/*
- * Mark the D-cache clean for this page to avoid extra flushing.
+ * Mark the D-cache clean for these pages to avoid extra flushing.
*/
- if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
- set_bit(PG_dcache_clean, &page->flags);
+ if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
+ unsigned long pfn;
+ size_t left = size;
+
+ pfn = page_to_pfn(page) + off / PAGE_SIZE;
+ off %= PAGE_SIZE;
+ if (off) {
+ pfn++;
+ left -= PAGE_SIZE - off;
+ }
+ while (left >= PAGE_SIZE) {
+ page = pfn_to_page(pfn++);
+ set_bit(PG_dcache_clean, &page->flags);
+ left -= PAGE_SIZE;
+ }
+ }
}
/**
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 5dbf13f954f6..c97f7940cb95 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -491,12 +491,14 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
* Some section permission faults need to be handled gracefully.
* They can happen due to a __{get,put}_user during an oops.
*/
+#ifndef CONFIG_ARM_LPAE
static int
do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
{
do_bad_area(addr, fsr, regs);
return 0;
}
+#endif /* CONFIG_ARM_LPAE */
/*
* This abort handler always returns "fault".
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 32aa5861119f..6d5ba9afb16a 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -17,6 +17,7 @@
#include <asm/highmem.h>
#include <asm/smp_plat.h>
#include <asm/tlbflush.h>
+#include <linux/hugetlb.h>
#include "mm.h"
@@ -168,19 +169,23 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
* coherent with the kernels mapping.
*/
if (!PageHighMem(page)) {
- __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
+ size_t page_size = PAGE_SIZE << compound_order(page);
+ __cpuc_flush_dcache_area(page_address(page), page_size);
} else {
- void *addr;
-
+ unsigned long i;
if (cache_is_vipt_nonaliasing()) {
- addr = kmap_atomic(page);
- __cpuc_flush_dcache_area(addr, PAGE_SIZE);
- kunmap_atomic(addr);
- } else {
- addr = kmap_high_get(page);
- if (addr) {
+ for (i = 0; i < (1 << compound_order(page)); i++) {
+ void *addr = kmap_atomic(page);
__cpuc_flush_dcache_area(addr, PAGE_SIZE);
- kunmap_high(page);
+ kunmap_atomic(addr);
+ }
+ } else {
+ for (i = 0; i < (1 << compound_order(page)); i++) {
+ void *addr = kmap_high_get(page);
+ if (addr) {
+ __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+ kunmap_high(page);
+ }
}
}
}
@@ -287,7 +292,7 @@ void flush_dcache_page(struct page *page)
mapping = page_mapping(page);
if (!cache_ops_need_broadcast() &&
- mapping && !mapping_mapped(mapping))
+ mapping && !page_mapped(page))
clear_bit(PG_dcache_clean, &page->flags);
else {
__flush_dcache_page(mapping, page);
diff --git a/arch/arm/mm/fsr-3level.c b/arch/arm/mm/fsr-3level.c
index 05a4e9431836..ab4409a2307e 100644
--- a/arch/arm/mm/fsr-3level.c
+++ b/arch/arm/mm/fsr-3level.c
@@ -9,11 +9,11 @@ static struct fsr_info fsr_info[] = {
{ do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
{ do_bad, SIGBUS, 0, "reserved access flag fault" },
{ do_bad, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
{ do_bad, SIGBUS, 0, "reserved permission fault" },
{ do_bad, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
- { do_sect_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
{ do_bad, SIGBUS, 0, "synchronous external abort" },
{ do_bad, SIGBUS, 0, "asynchronous external abort" },
diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c
new file mode 100644
index 000000000000..3d1e4a205b0b
--- /dev/null
+++ b/arch/arm/mm/hugetlbpage.c
@@ -0,0 +1,101 @@
+/*
+ * arch/arm/mm/hugetlbpage.c
+ *
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * Based on arch/x86/include/asm/hugetlb.h and Bill Carson's patches
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/pagemap.h>
+#include <linux/err.h>
+#include <linux/sysctl.h>
+#include <asm/mman.h>
+#include <asm/tlb.h>
+#include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
+
+/*
+ * On ARM, huge pages are backed by pmd's rather than pte's, so we do a lot
+ * of type casting from pmd_t * to pte_t *.
+ */
+
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd = NULL;
+
+ pgd = pgd_offset(mm, addr);
+ if (pgd_present(*pgd)) {
+ pud = pud_offset(pgd, addr);
+ if (pud_present(*pud))
+ pmd = pmd_offset(pud, addr);
+ }
+
+ return (pte_t *)pmd;
+}
+
+struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
+ int write)
+{
+ return ERR_PTR(-EINVAL);
+}
+
+int pud_huge(pud_t pud)
+{
+ return 0;
+}
+
+int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
+{
+ return 0;
+}
+
+pte_t *huge_pte_alloc(struct mm_struct *mm,
+ unsigned long addr, unsigned long sz)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pte_t *pte = NULL;
+
+ pgd = pgd_offset(mm, addr);
+ pud = pud_alloc(mm, pgd, addr);
+ if (pud)
+ pte = (pte_t *)pmd_alloc(mm, pud, addr);
+
+ return pte;
+}
+
+struct page *
+follow_huge_pmd(struct mm_struct *mm, unsigned long address,
+ pmd_t *pmd, int write)
+{
+ struct page *page;
+
+ page = pte_page(*(pte_t *)pmd);
+ if (page)
+ page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
+ return page;
+}
+
+int pmd_huge(pmd_t pmd)
+{
+ return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
+}
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 9a5cdc01fcdf..6833cbead6cc 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -36,12 +36,13 @@
#include "mm.h"
-static unsigned long phys_initrd_start __initdata = 0;
+static phys_addr_t phys_initrd_start __initdata = 0;
static unsigned long phys_initrd_size __initdata = 0;
static int __init early_initrd(char *p)
{
- unsigned long start, size;
+ phys_addr_t start;
+ unsigned long size;
char *endp;
start = memparse(p, &endp);
@@ -350,14 +351,14 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
#ifdef CONFIG_BLK_DEV_INITRD
if (phys_initrd_size &&
!memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) {
- pr_err("INITRD: 0x%08lx+0x%08lx is not a memory region - disabling initrd\n",
- phys_initrd_start, phys_initrd_size);
+ pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
+ (u64)phys_initrd_start, phys_initrd_size);
phys_initrd_start = phys_initrd_size = 0;
}
if (phys_initrd_size &&
memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) {
- pr_err("INITRD: 0x%08lx+0x%08lx overlaps in-use memory region - disabling initrd\n",
- phys_initrd_start, phys_initrd_size);
+ pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
+ (u64)phys_initrd_start, phys_initrd_size);
phys_initrd_start = phys_initrd_size = 0;
}
if (phys_initrd_size) {
@@ -442,7 +443,7 @@ static inline void
free_memmap(unsigned long start_pfn, unsigned long end_pfn)
{
struct page *start_pg, *end_pg;
- unsigned long pg, pgend;
+ phys_addr_t pg, pgend;
/*
* Convert start_pfn/end_pfn to a struct page pointer.
@@ -454,8 +455,8 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn)
* Convert to physical addresses, and
* round start upwards and end downwards.
*/
- pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
- pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;
+ pg = PAGE_ALIGN(__pa(start_pg));
+ pgend = __pa(end_pg) & PAGE_MASK;
/*
* If there are free pages between these,
@@ -582,9 +583,6 @@ static void __init free_highpages(void)
*/
void __init mem_init(void)
{
- unsigned long reserved_pages, free_pages;
- struct memblock_region *reg;
- int i;
#ifdef CONFIG_HAVE_TCM
/* These pointers are filled in on TCM detection */
extern u32 dtcm_end;
@@ -595,57 +593,16 @@ void __init mem_init(void)
/* this will put all unused low memory onto the freelists */
free_unused_memmap(&meminfo);
-
- totalram_pages += free_all_bootmem();
+ free_all_bootmem();
#ifdef CONFIG_SA1111
/* now that our DMA memory is actually so designated, we can free it */
- free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
+ free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, -1, NULL);
#endif
free_highpages();
- reserved_pages = free_pages = 0;
-
- for_each_bank(i, &meminfo) {
- struct membank *bank = &meminfo.bank[i];
- unsigned int pfn1, pfn2;
- struct page *page, *end;
-
- pfn1 = bank_pfn_start(bank);
- pfn2 = bank_pfn_end(bank);
-
- page = pfn_to_page(pfn1);
- end = pfn_to_page(pfn2 - 1) + 1;
-
- do {
- if (PageReserved(page))
- reserved_pages++;
- else if (!page_count(page))
- free_pages++;
- page++;
- } while (page < end);
- }
-
- /*
- * Since our memory may not be contiguous, calculate the
- * real number of pages we have in this system
- */
- printk(KERN_INFO "Memory:");
- num_physpages = 0;
- for_each_memblock(memory, reg) {
- unsigned long pages = memblock_region_memory_end_pfn(reg) -
- memblock_region_memory_base_pfn(reg);
- num_physpages += pages;
- printk(" %ldMB", pages >> (20 - PAGE_SHIFT));
- }
- printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT));
-
- printk(KERN_NOTICE "Memory: %luk/%luk available, %luk reserved, %luK highmem\n",
- nr_free_pages() << (PAGE_SHIFT-10),
- free_pages << (PAGE_SHIFT-10),
- reserved_pages << (PAGE_SHIFT-10),
- totalhigh_pages << (PAGE_SHIFT-10));
+ mem_init_print_info(NULL);
#define MLK(b, t) b, t, ((t) - (b)) >> 10
#define MLM(b, t) b, t, ((t) - (b)) >> 20
@@ -711,7 +668,7 @@ void __init mem_init(void)
BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
#endif
- if (PAGE_SIZE >= 16384 && num_physpages <= 128) {
+ if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
extern int sysctl_overcommit_memory;
/*
* On a machine this small we won't get
@@ -728,12 +685,12 @@ void free_initmem(void)
extern char __tcm_start, __tcm_end;
poison_init_mem(&__tcm_start, &__tcm_end - &__tcm_start);
- free_reserved_area(&__tcm_start, &__tcm_end, 0, "TCM link");
+ free_reserved_area(&__tcm_start, &__tcm_end, -1, "TCM link");
#endif
poison_init_mem(__init_begin, __init_end - __init_begin);
if (!machine_is_integrator() && !machine_is_cintegrator())
- free_initmem_default(0);
+ free_initmem_default(-1);
}
#ifdef CONFIG_BLK_DEV_INITRD
@@ -744,7 +701,7 @@ void free_initrd_mem(unsigned long start, unsigned long end)
{
if (!keep_initrd) {
poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
- free_reserved_area(start, end, 0, "initrd");
+ free_reserved_area((void *)start, (void *)end, -1, "initrd");
}
}
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 04d9006eab1f..f123d6eb074b 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -331,10 +331,10 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
return (void __iomem *) (offset + addr);
}
-void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size,
+void __iomem *__arm_ioremap_caller(phys_addr_t phys_addr, size_t size,
unsigned int mtype, void *caller)
{
- unsigned long last_addr;
+ phys_addr_t last_addr;
unsigned long offset = phys_addr & ~PAGE_MASK;
unsigned long pfn = __phys_to_pfn(phys_addr);
@@ -367,12 +367,12 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
}
EXPORT_SYMBOL(__arm_ioremap_pfn);
-void __iomem * (*arch_ioremap_caller)(unsigned long, size_t,
+void __iomem * (*arch_ioremap_caller)(phys_addr_t, size_t,
unsigned int, void *) =
__arm_ioremap_caller;
void __iomem *
-__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
+__arm_ioremap(phys_addr_t phys_addr, size_t size, unsigned int mtype)
{
return arch_ioremap_caller(phys_addr, size, mtype,
__builtin_return_address(0));
@@ -387,7 +387,7 @@ EXPORT_SYMBOL(__arm_ioremap);
* CONFIG_GENERIC_ALLOCATOR for allocating external memory.
*/
void __iomem *
-__arm_ioremap_exec(unsigned long phys_addr, size_t size, bool cached)
+__arm_ioremap_exec(phys_addr_t phys_addr, size_t size, bool cached)
{
unsigned int mtype;
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index d1d1cefa1f93..d7229d28c7f8 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -675,7 +675,8 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
}
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
- unsigned long end, unsigned long phys, const struct mem_type *type)
+ unsigned long end, phys_addr_t phys,
+ const struct mem_type *type)
{
pud_t *pud = pud_offset(pgd, addr);
unsigned long next;
@@ -989,27 +990,28 @@ phys_addr_t arm_lowmem_limit __initdata = 0;
void __init sanity_check_meminfo(void)
{
int i, j, highmem = 0;
+ phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1;
for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
struct membank *bank = &meminfo.bank[j];
- *bank = meminfo.bank[i];
+ phys_addr_t size_limit;
- if (bank->start > ULONG_MAX)
- highmem = 1;
+ *bank = meminfo.bank[i];
+ size_limit = bank->size;
-#ifdef CONFIG_HIGHMEM
- if (__va(bank->start) >= vmalloc_min ||
- __va(bank->start) < (void *)PAGE_OFFSET)
+ if (bank->start >= vmalloc_limit)
highmem = 1;
+ else
+ size_limit = vmalloc_limit - bank->start;
bank->highmem = highmem;
+#ifdef CONFIG_HIGHMEM
/*
* Split those memory banks which are partially overlapping
* the vmalloc area greatly simplifying things later.
*/
- if (!highmem && __va(bank->start) < vmalloc_min &&
- bank->size > vmalloc_min - __va(bank->start)) {
+ if (!highmem && bank->size > size_limit) {
if (meminfo.nr_banks >= NR_BANKS) {
printk(KERN_CRIT "NR_BANKS too low, "
"ignoring high memory\n");
@@ -1018,16 +1020,14 @@ void __init sanity_check_meminfo(void)
(meminfo.nr_banks - i) * sizeof(*bank));
meminfo.nr_banks++;
i++;
- bank[1].size -= vmalloc_min - __va(bank->start);
- bank[1].start = __pa(vmalloc_min - 1) + 1;
+ bank[1].size -= size_limit;
+ bank[1].start = vmalloc_limit;
bank[1].highmem = highmem = 1;
j++;
}
- bank->size = vmalloc_min - __va(bank->start);
+ bank->size = size_limit;
}
#else
- bank->highmem = highmem;
-
/*
* Highmem banks not allowed with !CONFIG_HIGHMEM.
*/
@@ -1040,31 +1040,16 @@ void __init sanity_check_meminfo(void)
}
/*
- * Check whether this memory bank would entirely overlap
- * the vmalloc area.
- */
- if (__va(bank->start) >= vmalloc_min ||
- __va(bank->start) < (void *)PAGE_OFFSET) {
- printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
- "(vmalloc region overlap).\n",
- (unsigned long long)bank->start,
- (unsigned long long)bank->start + bank->size - 1);
- continue;
- }
-
- /*
* Check whether this memory bank would partially overlap
* the vmalloc area.
*/
- if (__va(bank->start + bank->size - 1) >= vmalloc_min ||
- __va(bank->start + bank->size - 1) <= __va(bank->start)) {
- unsigned long newsize = vmalloc_min - __va(bank->start);
+ if (bank->size > size_limit) {
printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
"to -%.8llx (vmalloc region overlap).\n",
(unsigned long long)bank->start,
(unsigned long long)bank->start + bank->size - 1,
- (unsigned long long)bank->start + newsize - 1);
- bank->size = newsize;
+ (unsigned long long)bank->start + size_limit - 1);
+ bank->size = size_limit;
}
#endif
if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit)
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 5a3aba614a40..1fa50100ab6a 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -8,6 +8,7 @@
#include <linux/pagemap.h>
#include <linux/io.h>
#include <linux/memblock.h>
+#include <linux/kernel.h>
#include <asm/cacheflush.h>
#include <asm/sections.h>
@@ -15,9 +16,260 @@
#include <asm/setup.h>
#include <asm/traps.h>
#include <asm/mach/arch.h>
+#include <asm/cputype.h>
+#include <asm/mpu.h>
#include "mm.h"
+#ifdef CONFIG_ARM_MPU
+struct mpu_rgn_info mpu_rgn_info;
+
+/* Region number */
+static void rgnr_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c2, 0" : : "r" (v));
+}
+
+/* Data-side / unified region attributes */
+
+/* Region access control register */
+static void dracr_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 4" : : "r" (v));
+}
+
+/* Region size register */
+static void drsr_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 2" : : "r" (v));
+}
+
+/* Region base address register */
+static void drbar_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 0" : : "r" (v));
+}
+
+static u32 drbar_read(void)
+{
+ u32 v;
+ asm("mrc p15, 0, %0, c6, c1, 0" : "=r" (v));
+ return v;
+}
+/* Optional instruction-side region attributes */
+
+/* I-side Region access control register */
+static void iracr_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 5" : : "r" (v));
+}
+
+/* I-side Region size register */
+static void irsr_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 3" : : "r" (v));
+}
+
+/* I-side Region base address register */
+static void irbar_write(u32 v)
+{
+ asm("mcr p15, 0, %0, c6, c1, 1" : : "r" (v));
+}
+
+static unsigned long irbar_read(void)
+{
+ unsigned long v;
+ asm("mrc p15, 0, %0, c6, c1, 1" : "=r" (v));
+ return v;
+}
+
+/* MPU initialisation functions */
+void __init sanity_check_meminfo_mpu(void)
+{
+ int i;
+ struct membank *bank = meminfo.bank;
+ phys_addr_t phys_offset = PHYS_OFFSET;
+ phys_addr_t aligned_region_size, specified_mem_size, rounded_mem_size;
+
+ /* Initially only use memory continuous from PHYS_OFFSET */
+ if (bank_phys_start(&bank[0]) != phys_offset)
+ panic("First memory bank must be contiguous from PHYS_OFFSET");
+
+ /* Banks have already been sorted by start address */
+ for (i = 1; i < meminfo.nr_banks; i++) {
+ if (bank[i].start <= bank_phys_end(&bank[0]) &&
+ bank_phys_end(&bank[i]) > bank_phys_end(&bank[0])) {
+ bank[0].size = bank_phys_end(&bank[i]) - bank[0].start;
+ } else {
+ pr_notice("Ignoring RAM after 0x%.8lx. "
+ "First non-contiguous (ignored) bank start: 0x%.8lx\n",
+ (unsigned long)bank_phys_end(&bank[0]),
+ (unsigned long)bank_phys_start(&bank[i]));
+ break;
+ }
+ }
+ /* All contiguous banks are now merged in to the first bank */
+ meminfo.nr_banks = 1;
+ specified_mem_size = bank[0].size;
+
+ /*
+ * MPU has curious alignment requirements: Size must be power of 2, and
+ * region start must be aligned to the region size
+ */
+ if (phys_offset != 0)
+ pr_info("PHYS_OFFSET != 0 => MPU Region size constrained by alignment requirements\n");
+
+ /*
+ * Maximum aligned region might overflow phys_addr_t if phys_offset is
+ * 0. Hence we keep everything below 4G until we take the smaller of
+ * the aligned_region_size and rounded_mem_size, one of which is
+ * guaranteed to be smaller than the maximum physical address.
+ */
+ aligned_region_size = (phys_offset - 1) ^ (phys_offset);
+ /* Find the max power-of-two sized region that fits inside our bank */
+ rounded_mem_size = (1 << __fls(bank[0].size)) - 1;
+
+ /* The actual region size is the smaller of the two */
+ aligned_region_size = aligned_region_size < rounded_mem_size
+ ? aligned_region_size + 1
+ : rounded_mem_size + 1;
+
+ if (aligned_region_size != specified_mem_size)
+ pr_warn("Truncating memory from 0x%.8lx to 0x%.8lx (MPU region constraints)",
+ (unsigned long)specified_mem_size,
+ (unsigned long)aligned_region_size);
+
+ meminfo.bank[0].size = aligned_region_size;
+ pr_debug("MPU Region from 0x%.8lx size 0x%.8lx (end 0x%.8lx))\n",
+ (unsigned long)phys_offset,
+ (unsigned long)aligned_region_size,
+ (unsigned long)bank_phys_end(&bank[0]));
+
+}
+
+static int mpu_present(void)
+{
+ return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7);
+}
+
+static int mpu_max_regions(void)
+{
+ /*
+ * We don't support a different number of I/D side regions so if we
+ * have separate instruction and data memory maps then return
+ * whichever side has a smaller number of supported regions.
+ */
+ u32 dregions, iregions, mpuir;
+ mpuir = read_cpuid(CPUID_MPUIR);
+
+ dregions = iregions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION;
+
+ /* Check for separate d-side and i-side memory maps */
+ if (mpuir & MPUIR_nU)
+ iregions = (mpuir & MPUIR_IREGION_SZMASK) >> MPUIR_IREGION;
+
+ /* Use the smallest of the two maxima */
+ return min(dregions, iregions);
+}
+
+static int mpu_iside_independent(void)
+{
+ /* MPUIR.nU specifies whether there is *not* a unified memory map */
+ return read_cpuid(CPUID_MPUIR) & MPUIR_nU;
+}
+
+static int mpu_min_region_order(void)
+{
+ u32 drbar_result, irbar_result;
+ /* We've kept a region free for this probing */
+ rgnr_write(MPU_PROBE_REGION);
+ isb();
+ /*
+ * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum
+ * region order
+ */
+ drbar_write(0xFFFFFFFC);
+ drbar_result = irbar_result = drbar_read();
+ drbar_write(0x0);
+ /* If the MPU is non-unified, we use the larger of the two minima*/
+ if (mpu_iside_independent()) {
+ irbar_write(0xFFFFFFFC);
+ irbar_result = irbar_read();
+ irbar_write(0x0);
+ }
+ isb(); /* Ensure that MPU region operations have completed */
+ /* Return whichever result is larger */
+ return __ffs(max(drbar_result, irbar_result));
+}
+
+static int mpu_setup_region(unsigned int number, phys_addr_t start,
+ unsigned int size_order, unsigned int properties)
+{
+ u32 size_data;
+
+ /* We kept a region free for probing resolution of MPU regions*/
+ if (number > mpu_max_regions() || number == MPU_PROBE_REGION)
+ return -ENOENT;
+
+ if (size_order > 32)
+ return -ENOMEM;
+
+ if (size_order < mpu_min_region_order())
+ return -ENOMEM;
+
+ /* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */
+ size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN;
+
+ dsb(); /* Ensure all previous data accesses occur with old mappings */
+ rgnr_write(number);
+ isb();
+ drbar_write(start);
+ dracr_write(properties);
+ isb(); /* Propagate properties before enabling region */
+ drsr_write(size_data);
+
+ /* Check for independent I-side registers */
+ if (mpu_iside_independent()) {
+ irbar_write(start);
+ iracr_write(properties);
+ isb();
+ irsr_write(size_data);
+ }
+ isb();
+
+ /* Store region info (we treat i/d side the same, so only store d) */
+ mpu_rgn_info.rgns[number].dracr = properties;
+ mpu_rgn_info.rgns[number].drbar = start;
+ mpu_rgn_info.rgns[number].drsr = size_data;
+ return 0;
+}
+
+/*
+* Set up default MPU regions, doing nothing if there is no MPU
+*/
+void __init mpu_setup(void)
+{
+ int region_err;
+ if (!mpu_present())
+ return;
+
+ region_err = mpu_setup_region(MPU_RAM_REGION, PHYS_OFFSET,
+ ilog2(meminfo.bank[0].size),
+ MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL);
+ if (region_err) {
+ panic("MPU region initialization failure! %d", region_err);
+ } else {
+ pr_info("Using ARMv7 PMSA Compliant MPU. "
+ "Region independence: %s, Max regions: %d\n",
+ mpu_iside_independent() ? "Yes" : "No",
+ mpu_max_regions());
+ }
+}
+#else
+static void sanity_check_meminfo_mpu(void) {}
+static void __init mpu_setup(void) {}
+#endif /* CONFIG_ARM_MPU */
+
void __init arm_mm_memblock_reserve(void)
{
#ifndef CONFIG_CPU_V7M
@@ -37,7 +289,9 @@ void __init arm_mm_memblock_reserve(void)
void __init sanity_check_meminfo(void)
{
- phys_addr_t end = bank_phys_end(&meminfo.bank[meminfo.nr_banks - 1]);
+ phys_addr_t end;
+ sanity_check_meminfo_mpu();
+ end = bank_phys_end(&meminfo.bank[meminfo.nr_banks - 1]);
high_memory = __va(end - 1) + 1;
}
@@ -48,6 +302,7 @@ void __init sanity_check_meminfo(void)
void __init paging_init(struct machine_desc *mdesc)
{
early_trap_init((void *)CONFIG_VECTORS_BASE);
+ mpu_setup();
bootmem_init();
}
@@ -94,16 +349,16 @@ void __iomem *__arm_ioremap_pfn_caller(unsigned long pfn, unsigned long offset,
return __arm_ioremap_pfn(pfn, offset, size, mtype);
}
-void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
+void __iomem *__arm_ioremap(phys_addr_t phys_addr, size_t size,
unsigned int mtype)
{
return (void __iomem *)phys_addr;
}
EXPORT_SYMBOL(__arm_ioremap);
-void __iomem * (*arch_ioremap_caller)(unsigned long, size_t, unsigned int, void *);
+void __iomem * (*arch_ioremap_caller)(phys_addr_t, size_t, unsigned int, void *);
-void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size,
+void __iomem *__arm_ioremap_caller(phys_addr_t phys_addr, size_t size,
unsigned int mtype, void *caller)
{
return __arm_ioremap(phys_addr, size, mtype);
diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S
index 919405e20b80..2d1ef87328a1 100644
--- a/arch/arm/mm/proc-v6.S
+++ b/arch/arm/mm/proc-v6.S
@@ -140,8 +140,10 @@ ENTRY(cpu_v6_set_pte_ext)
ENTRY(cpu_v6_do_suspend)
stmfd sp!, {r4 - r9, lr}
mrc p15, 0, r4, c13, c0, 0 @ FCSE/PID
+#ifdef CONFIG_MMU
mrc p15, 0, r5, c3, c0, 0 @ Domain ID
mrc p15, 0, r6, c2, c0, 1 @ Translation table base 1
+#endif
mrc p15, 0, r7, c1, c0, 1 @ auxiliary control register
mrc p15, 0, r8, c1, c0, 2 @ co-processor access control
mrc p15, 0, r9, c1, c0, 0 @ control register
@@ -158,14 +160,16 @@ ENTRY(cpu_v6_do_resume)
mcr p15, 0, ip, c13, c0, 1 @ set reserved context ID
ldmia r0, {r4 - r9}
mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID
+#ifdef CONFIG_MMU
mcr p15, 0, r5, c3, c0, 0 @ Domain ID
ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP)
ALT_UP(orr r1, r1, #TTB_FLAGS_UP)
mcr p15, 0, r1, c2, c0, 0 @ Translation table base 0
mcr p15, 0, r6, c2, c0, 1 @ Translation table base 1
+ mcr p15, 0, ip, c2, c0, 2 @ TTB control register
+#endif
mcr p15, 0, r7, c1, c0, 1 @ auxiliary control register
mcr p15, 0, r8, c1, c0, 2 @ co-processor access control
- mcr p15, 0, ip, c2, c0, 2 @ TTB control register
mcr p15, 0, ip, c7, c5, 4 @ ISB
mov r0, r9 @ control register
b cpu_resume_mmu
diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S
index 363027e811d6..5ffe1956c6d9 100644
--- a/arch/arm/mm/proc-v7-3level.S
+++ b/arch/arm/mm/proc-v7-3level.S
@@ -39,6 +39,14 @@
#define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA)
#define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S)
+#ifndef __ARMEB__
+# define rpgdl r0
+# define rpgdh r1
+#else
+# define rpgdl r1
+# define rpgdh r0
+#endif
+
/*
* cpu_v7_switch_mm(pgd_phys, tsk)
*
@@ -47,10 +55,10 @@
*/
ENTRY(cpu_v7_switch_mm)
#ifdef CONFIG_MMU
- mmid r1, r1 @ get mm->context.id
- asid r3, r1
- mov r3, r3, lsl #(48 - 32) @ ASID
- mcrr p15, 0, r0, r3, c2 @ set TTB 0
+ mmid r2, r2
+ asid r2, r2
+ orr rpgdh, rpgdh, r2, lsl #(48 - 32) @ upper 32-bits of pgd
+ mcrr p15, 0, rpgdl, rpgdh, c2 @ set TTB 0
isb
#endif
mov pc, lr
@@ -106,7 +114,8 @@ ENDPROC(cpu_v7_set_pte_ext)
*/
.macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address
- cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? (branch below)
+ mov \tmp, \tmp, lsr #ARCH_PGD_SHIFT
+ cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET?
mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register
orr \tmp, \tmp, #TTB_EAE
ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP)
@@ -114,27 +123,21 @@ ENDPROC(cpu_v7_set_pte_ext)
ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16)
ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16)
/*
- * TTBR0/TTBR1 split (PAGE_OFFSET):
- * 0x40000000: T0SZ = 2, T1SZ = 0 (not used)
- * 0x80000000: T0SZ = 0, T1SZ = 1
- * 0xc0000000: T0SZ = 0, T1SZ = 2
- *
- * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise
- * booting secondary CPUs would end up using TTBR1 for the identity
- * mapping set up in TTBR0.
+ * Only use split TTBRs if PHYS_OFFSET <= PAGE_OFFSET (cmp above),
+ * otherwise booting secondary CPUs would end up using TTBR1 for the
+ * identity mapping set up in TTBR0.
*/
- bhi 9001f @ PHYS_OFFSET > PAGE_OFFSET?
- orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ
-#if defined CONFIG_VMSPLIT_2G
- /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */
- add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries
-#elif defined CONFIG_VMSPLIT_3G
- /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */
- add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd
-#endif
- /* CONFIG_VMSPLIT_1G does not need TTBR1 adjustment */
-9001: mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register
- mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1
+ orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ
+ mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR
+ mov \tmp, \ttbr1, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits
+ mov \ttbr1, \ttbr1, lsl #ARCH_PGD_SHIFT @ lower bits
+ addls \ttbr1, \ttbr1, #TTBR1_OFFSET
+ mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1
+ mov \tmp, \ttbr0, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits
+ mov \ttbr0, \ttbr0, lsl #ARCH_PGD_SHIFT @ lower bits
+ mcrr p15, 0, \ttbr0, \zero, c2 @ load TTBR0
+ mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1
+ mcrr p15, 0, \ttbr0, \zero, c2 @ load TTBR0
.endm
__CPUINIT
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index e35fec34453e..7ef3ad05df39 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -98,9 +98,11 @@ ENTRY(cpu_v7_do_suspend)
mrc p15, 0, r4, c13, c0, 0 @ FCSE/PID
mrc p15, 0, r5, c13, c0, 3 @ User r/o thread ID
stmia r0!, {r4 - r5}
+#ifdef CONFIG_MMU
mrc p15, 0, r6, c3, c0, 0 @ Domain ID
mrc p15, 0, r7, c2, c0, 1 @ TTB 1
mrc p15, 0, r11, c2, c0, 2 @ TTB control register
+#endif
mrc p15, 0, r8, c1, c0, 0 @ Control register
mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register
mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control
@@ -110,13 +112,14 @@ ENDPROC(cpu_v7_do_suspend)
ENTRY(cpu_v7_do_resume)
mov ip, #0
- mcr p15, 0, ip, c8, c7, 0 @ invalidate TLBs
mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache
mcr p15, 0, ip, c13, c0, 1 @ set reserved context ID
ldmia r0!, {r4 - r5}
mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID
mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID
ldmia r0, {r6 - r11}
+#ifdef CONFIG_MMU
+ mcr p15, 0, ip, c8, c7, 0 @ invalidate TLBs
mcr p15, 0, r6, c3, c0, 0 @ Domain ID
#ifndef CONFIG_ARM_LPAE
ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP)
@@ -125,14 +128,15 @@ ENTRY(cpu_v7_do_resume)
mcr p15, 0, r1, c2, c0, 0 @ TTB 0
mcr p15, 0, r7, c2, c0, 1 @ TTB 1
mcr p15, 0, r11, c2, c0, 2 @ TTB control register
- mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register
- teq r4, r9 @ Is it already set?
- mcrne p15, 0, r9, c1, c0, 1 @ No, so write it
- mcr p15, 0, r10, c1, c0, 2 @ Co-processor access control
ldr r4, =PRRR @ PRRR
ldr r5, =NMRR @ NMRR
mcr p15, 0, r4, c10, c2, 0 @ write PRRR
mcr p15, 0, r5, c10, c2, 1 @ write NMRR
+#endif /* CONFIG_MMU */
+ mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register
+ teq r4, r9 @ Is it already set?
+ mcrne p15, 0, r9, c1, c0, 1 @ No, so write it
+ mcr p15, 0, r10, c1, c0, 2 @ Co-processor access control
isb
dsb
mov r0, r8 @ control register
@@ -178,7 +182,8 @@ ENDPROC(cpu_pj4b_do_idle)
*/
__v7_ca5mp_setup:
__v7_ca9mp_setup:
- mov r10, #(1 << 0) @ TLB ops broadcasting
+__v7_cr7mp_setup:
+ mov r10, #(1 << 0) @ Cache/TLB ops broadcasting
b 1f
__v7_ca7mp_setup:
__v7_ca15mp_setup:
@@ -443,6 +448,16 @@ __v7_pj4b_proc_info:
#endif
/*
+ * ARM Ltd. Cortex R7 processor.
+ */
+ .type __v7_cr7mp_proc_info, #object
+__v7_cr7mp_proc_info:
+ .long 0x410fc170
+ .long 0xff0ffff0
+ __v7_proc __v7_cr7mp_setup
+ .size __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info
+
+ /*
* ARM Ltd. Cortex A7 processor.
*/
.type __v7_ca7mp_proc_info, #object
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.