Make return-to-user and kernel re-entry work. This adds a per-pmap SLB

cache.  We might want to turn that in a per-proc cache at some point, but
this gets us to the point where we can sucessfully have init(1) do its
first system call.

9 files changed, 105 insertions, 14 deletions

diff --git a/sys/arch/powerpc64/include/cpu.h b/sys/arch/powerpc64/include/cpu.h
index 3b27dc0815a..8ba1280d26c 100644
--- a/sys/arch/powerpc64/include/cpu.h
+++ b/sys/arch/powerpc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.15 2020/06/22 13:52:40 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.16 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -49,6 +49,7 @@ struct cpu_info {
 	struct pcb	*ci_curpcb;
 
 	struct slb	ci_kernel_slb[32];
+	paddr_t		ci_user_slb_pa;
 
 #define CPUSAVE_LEN	9
 	register_t	ci_tempsave[CPUSAVE_LEN];
diff --git a/sys/arch/powerpc64/include/pmap.h b/sys/arch/powerpc64/include/pmap.h
index f0721592a6d..0c252b06ecf 100644
--- a/sys/arch/powerpc64/include/pmap.h
+++ b/sys/arch/powerpc64/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.5 2020/06/21 13:23:59 kettenis Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.6 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -32,6 +32,7 @@ struct pmap {
 	int			pm_refs;
 	struct pmap_statistics	pm_stats;
 	struct mutex		pm_mtx;
+	struct slb		pm_slb[32];
 };
 
 typedef struct pmap *pmap_t;
diff --git a/sys/arch/powerpc64/include/proc.h b/sys/arch/powerpc64/include/proc.h
index b5e2281f0b9..c7eaefe0bb4 100644
--- a/sys/arch/powerpc64/include/proc.h
+++ b/sys/arch/powerpc64/include/proc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.h,v 1.2 2020/06/14 17:56:54 kettenis Exp $	*/
+/*	$OpenBSD: proc.h,v 1.3 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*
  * Copyright (c) 1991 Regents of the University of California.
@@ -39,7 +39,8 @@
  */
 struct mdproc {
 	struct trapframe *md_regs;
-	volatile int md_astpending;
+	volatile int	md_astpending;
+	paddr_t		md_user_slb_pa;
 };
 
 #endif /* _MACHINE_PROC_H_ */
diff --git a/sys/arch/powerpc64/powerpc64/genassym.cf b/sys/arch/powerpc64/powerpc64/genassym.cf
index a8013121bd9..5dcaa2e58d3 100644
--- a/sys/arch/powerpc64/powerpc64/genassym.cf
+++ b/sys/arch/powerpc64/powerpc64/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.7 2020/06/21 13:23:59 kettenis Exp $
+#	$OpenBSD: genassym.cf,v 1.8 2020/06/22 16:58:20 kettenis Exp $
 #
 # Copyright (c) 1982, 1990 The Regents of the University of California.
 # All rights reserved.
@@ -41,6 +41,7 @@ member	ci_curpcb
 member	ci_curproc
 member	ci_tempsave
 member	ci_kernel_slb
+member	ci_user_slb_pa
 define	CPUSAVE_SRR0	0
 define	CPUSAVE_SRR1	8
 define	CPUSAVE_R27	16
@@ -102,6 +103,7 @@ struct	proc
 member	p_addr
 member	p_stat
 member	P_MD_ASTPENDING	p_md.md_astpending
+member	P_MD_USER_SLB_PA p_md.md_user_slb_pa
 
 export	SONPROC
 
diff --git a/sys/arch/powerpc64/powerpc64/locore.S b/sys/arch/powerpc64/powerpc64/locore.S
index 4b9039c241d..f56aaf0ec15 100644
--- a/sys/arch/powerpc64/powerpc64/locore.S
+++ b/sys/arch/powerpc64/powerpc64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.16 2020/06/22 16:47:10 kettenis Exp $	*/
+/*	$OpenBSD: locore.S,v 1.17 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -92,6 +92,9 @@ cpu_switchto:
 	std	%r31, CI_CURPCB(%r13)
 	ld	%r1, PCB_SP(%r31)
 
+	ld	%r31, P_MD_USER_SLB_PA(%r4)
+	std	%r31, CI_USER_SLB_PA(%r13)
+
 	ld	%r31, SF_R31(%r1)
 	ld	%r30, SF_R30(%r1)
 	ld	%r29, SF_R29(%r1)
diff --git a/sys/arch/powerpc64/powerpc64/pmap.c b/sys/arch/powerpc64/powerpc64/pmap.c
index 7bdfe7264e4..229f9502f17 100644
--- a/sys/arch/powerpc64/powerpc64/pmap.c
+++ b/sys/arch/powerpc64/powerpc64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.14 2020/06/21 18:23:43 kettenis Exp $ */
+/*	$OpenBSD: pmap.c,v 1.15 2020/06/22 16:58:20 kettenis Exp $ */
 
 /*
  * Copyright (c) 2015 Martin Pieuchot
@@ -313,6 +313,8 @@ pmap_slbd_alloc(pmap_t pm, vaddr_t va)
 {
 	uint64_t esid = va >> ADDR_ESID_SHIFT;
 	struct slb_desc *slbd;
+	uint64_t slbe, slbv;
+	int i;
 
 	KASSERT(pm != pmap_kernel());
 
@@ -323,6 +325,20 @@ pmap_slbd_alloc(pmap_t pm, vaddr_t va)
 	slbd->slbd_esid = esid;
 	slbd->slbd_vsid = pmap_vsid++;
 	LIST_INSERT_HEAD(&pm->pm_slbd, slbd, slbd_list);
+
+	slbe = (slbd->slbd_esid << SLBE_ESID_SHIFT) | SLBE_VALID | 31;
+	slbv = slbd->slbd_vsid << SLBV_VSID_SHIFT;
+
+	for (i = 0; i < nitems(pm->pm_slb); i++) {
+		if (pm->pm_slb[i].slb_slbe == 0)
+			break;
+	}
+	if (i == nitems(pm->pm_slb))
+		i = arc4random_uniform(nitems(pm->pm_slb));
+
+	pm->pm_slb[i].slb_slbe = slbe;
+	pm->pm_slb[i].slb_slbv = slbv;
+
 	return slbd;
 }
 
@@ -438,8 +454,10 @@ pmap_vp_enter(pmap_t pm, vaddr_t va, struct pte_desc *pted, int flags)
 	struct pmapvp2 *vp2;
 
 	slbd = pmap_slbd_lookup(pm, va);
-	KASSERT(slbd);
-	/* XXX Allocate SLB descriptors. */
+	if (slbd == NULL) {
+		slbd = pmap_slbd_alloc(pm, va);
+		KASSERT(slbd);
+	}
 
 	vp1 = slbd->slbd_vp;
 	if (vp1 == NULL) {
@@ -844,7 +862,7 @@ pmap_init(void)
 	int i;
 
 	pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, IPL_NONE, 0,
-	    "pmap", NULL);
+	    "pmap", &pool_allocator_single);
 	pool_setlowat(&pmap_pmap_pool, 2);
 	pool_init(&pmap_vp_pool, sizeof(struct pmapvp1), 0, IPL_VM, 0,
 	    "vp", &pool_allocator_single);
@@ -891,8 +909,10 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 
 	PMAP_VP_LOCK(pm);
 	pted = pmap_vp_lookup(pm, va);
-	if (pted && PTED_VALID(pted))
+	if (pted && PTED_VALID(pted)) {
 		pmap_remove_pted(pm, pted);
+		pted = NULL;
+	}
 
 	pm->pm_stats.resident_count++;
 
diff --git a/sys/arch/powerpc64/powerpc64/trap.c b/sys/arch/powerpc64/powerpc64/trap.c
index ef25cf43dc1..313d9b31066 100644
--- a/sys/arch/powerpc64/powerpc64/trap.c
+++ b/sys/arch/powerpc64/powerpc64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.10 2020/06/21 13:23:59 kettenis Exp $	*/
+/*	$OpenBSD: trap.c,v 1.11 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -43,6 +43,7 @@ trap(struct trapframe *frame)
 	struct vm_map *map;
 	vaddr_t va;
 	int ftype;
+	int error;
 
 	switch (type) {
 	case EXC_DECR:
@@ -106,6 +107,18 @@ trap(struct trapframe *frame)
 		printf("dar 0x%lx dsisr 0x%lx\n", frame->dar, frame->dsisr);
 		goto fatal;
 
+	case EXC_ISI|EXC_USER:
+	case EXC_ISE|EXC_USER:
+		intr_enable();
+		map = &p->p_vmspace->vm_map;
+		va = frame->srr0;
+		ftype = PROT_READ | PROT_EXEC;
+		error = uvm_fault(map, trunc_page(va), 0, ftype);
+		KERNEL_UNLOCK();
+		if (error)
+			goto fatal;
+		break;
+
 	case EXC_SC|EXC_USER:
 		intr_enable();
 		syscall(frame);
diff --git a/sys/arch/powerpc64/powerpc64/trap_subr.S b/sys/arch/powerpc64/powerpc64/trap_subr.S
index 2520c982dd2..bedc5682e77 100644
--- a/sys/arch/powerpc64/powerpc64/trap_subr.S
+++ b/sys/arch/powerpc64/powerpc64/trap_subr.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: trap_subr.S,v 1.11 2020/06/21 13:23:59 kettenis Exp $ */
+/* $OpenBSD: trap_subr.S,v 1.12 2020/06/22 16:58:20 kettenis Exp $ */
 /* $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $	*/
 
 /*-
@@ -34,6 +34,7 @@
 
 #include "assym.h"
 
+#include <machine/param.h>
 #include <machine/psl.h>
 #include <machine/trap.h>
 
@@ -54,6 +55,29 @@
  * Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
  */
 
+restore_usersrs:
+	GET_CPUINFO(%r28)
+	ld	%r28, CI_USER_SLB_PA(%r28)
+	li	%r29, 0			/* Set the counter to zero */
+
+	/* Invalidate entire SLB */
+	slbia
+	slbmfee	%r31, %r29
+	clrrdi	%r31, %r31, 28
+	slbie	%r31
+
+1:	ld	%r31, 0(%r28)		/* Load SLBE */
+	cmpdi	%r31, 0			/* If SLBE is not valid, stop */
+	beqlr
+	ld	%r30, 8(%r28)		/* Load SLBV  */
+	slbmte	%r30, %r31		/* Install SLB entry */
+
+	addi	%r28, %r28, 16		/* Advance pointer */
+	addi	%r29, %r29, 1
+	cmpdi	%r29, 32		/* Repeat if we are not at the end */
+	blt	1b
+	blr
+
 restore_kernsrs:
 	GET_CPUINFO(%r28)
 	addi	%r28, %r28, CI_KERNEL_SLB
@@ -221,6 +245,26 @@ restore_kernsrs:
 	andi.	%r3, %r3, ~(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;		\
 	mtmsr	%r3;							\
 	isync;								\
+/* Decide whether we return to user mode: */				\
+	GET_CPUINFO(%r3);						\
+	ld	%r3, (savearea+CPUSAVE_SRR1)(%r3);			\
+	mtcr	%r3;							\
+	bf	17,1f;			/* branch if PSL_PR is false */	\
+/* Restore user SRs */							\
+	GET_CPUINFO(%r3);						\
+	std	%r27, (savearea+CPUSAVE_R27)(%r3);			\
+	std	%r28, (savearea+CPUSAVE_R28)(%r3);			\
+	std	%r29, (savearea+CPUSAVE_R29)(%r3);			\
+	std	%r30, (savearea+CPUSAVE_R30)(%r3);			\
+	std	%r31, (savearea+CPUSAVE_R31)(%r3);			\
+	mflr	%r27;			/* preserve LR */		\
+	bl	restore_usersrs;	/* uses r28-r31 */		\
+	mtlr	%r27;							\
+	ld	%r31, (savearea+CPUSAVE_R31)(%r3);			\
+	ld	%r30, (savearea+CPUSAVE_R30)(%r3);			\
+	ld	%r29, (savearea+CPUSAVE_R29)(%r3);			\
+	ld	%r28, (savearea+CPUSAVE_R28)(%r3);			\
+	ld	%r27, (savearea+CPUSAVE_R27)(%r3);			\
 1:	mfsprg2	%r3;			/* restore cr */		\
 	mtcr	%r3;							\
 	GET_CPUINFO(%r3);						\
@@ -315,6 +359,9 @@ generictrap:
 	bf	17, k_trap		/* branch if PSL_PR is false */
 
 u_trap:
+	GET_CPUINFO(%r1)
+	ld	%r1, CI_CURPCB(%r1)
+	addi	%r1, %r1, USPACE
 	mr	%r27, %r28
 	mtsprg2	%r29
 	bl	restore_kernsrs		/* enable kernel mapping */
diff --git a/sys/arch/powerpc64/powerpc64/vm_machdep.c b/sys/arch/powerpc64/powerpc64/vm_machdep.c
index 1112931c022..225283d6e09 100644
--- a/sys/arch/powerpc64/powerpc64/vm_machdep.c
+++ b/sys/arch/powerpc64/powerpc64/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.2 2020/06/14 17:56:54 kettenis Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.3 2020/06/22 16:58:20 kettenis Exp $	*/
 
 /*-
  * Copyright (c) 1995 Charles M. Hannum.  All rights reserved.
@@ -67,6 +67,7 @@ void
 cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb,
     void (*func)(void *), void *arg)
 {
+	pmap_t pm = p2->p_vmspace->vm_map.pmap;
 	struct pcb *pcb = &p2->p_addr->u_pcb;
 	struct trapframe *tf;
 	struct callframe *cf;
@@ -76,6 +77,8 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb,
 	/* Copy the pcb. */
 	*pcb = p1->p_addr->u_pcb;
 
+	pmap_extract(pmap_kernel(), (vaddr_t)&pm->pm_slb,
+	    &p2->p_md.md_user_slb_pa);
 	pmap_activate(p2);
 
 	kstack = (register_t)p2->p_addr + USPACE - FRAMELEN -