Add support for the XIVE interrupt controller found on POWER9 CPUs.

13 files changed, 752 insertions, 43 deletions

diff --git a/sys/arch/powerpc64/conf/GENERIC b/sys/arch/powerpc64/conf/GENERIC
index 1ba1d0514a1..2037f4051f8 100644
--- a/sys/arch/powerpc64/conf/GENERIC
+++ b/sys/arch/powerpc64/conf/GENERIC
@@ -1,4 +1,4 @@
-#	$OpenBSD: GENERIC,v 1.5 2020/06/10 15:10:51 kettenis Exp $
+#	$OpenBSD: GENERIC,v 1.6 2020/06/13 22:58:42 kettenis Exp $
 #
 # For further information on compiling OpenBSD kernels, see the config(8)
 # man page.
@@ -22,6 +22,8 @@ cpu0		at mainbus?
 opal0		at fdt?
 phb*		at fdt?
 pci*		at phb?
+xive*		at fdt?
+
 ppb*		at pci?
 pci*		at ppb?
 ahci*		at pci? disable
diff --git a/sys/arch/powerpc64/conf/files.powerpc64 b/sys/arch/powerpc64/conf/files.powerpc64
index 24919fd4741..e586dd0c1e2 100644
--- a/sys/arch/powerpc64/conf/files.powerpc64
+++ b/sys/arch/powerpc64/conf/files.powerpc64
@@ -1,4 +1,4 @@
-# $OpenBSD: files.powerpc64,v 1.11 2020/06/10 19:06:53 kettenis Exp $
+# $OpenBSD: files.powerpc64,v 1.12 2020/06/13 22:58:42 kettenis Exp $
 
 maxpartitions	16
 maxusers	2 8 128
@@ -15,6 +15,7 @@ file	arch/powerpc64/powerpc64/db_interface.c		ddb
 file	arch/powerpc64/powerpc64/db_memrw.c		ddb
 file	arch/powerpc64/powerpc64/db_trace.c		ddb
 file	arch/powerpc64/powerpc64/disksubr.c
+file	arch/powerpc64/powerpc64/intr.c
 file	arch/powerpc64/powerpc64/machdep.c
 file	arch/powerpc64/powerpc64/pmap.c
 file	arch/powerpc64/powerpc64/process_machdep.c
@@ -63,6 +64,10 @@ device	phb: pcibus
 attach	phb at fdt
 file	arch/powerpc64/dev/phb.c		phb
 
+device	xive
+attach	xive at fdt
+file	arch/powerpc64/dev/xive.c		xive
+
 # Machine-independent HID support
 include	"dev/hid/files.hid"
 
diff --git a/sys/arch/powerpc64/dev/phb.c b/sys/arch/powerpc64/dev/phb.c
index 2c215bc72d1..86726f19f09 100644
--- a/sys/arch/powerpc64/dev/phb.c
+++ b/sys/arch/powerpc64/dev/phb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: phb.c,v 1.5 2020/06/10 19:02:41 kettenis Exp $	*/
+/*	$OpenBSD: phb.c,v 1.6 2020/06/13 22:58:42 kettenis Exp $	*/
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -118,7 +118,8 @@ phb_attach(struct device *parent, struct device *self, void *aux)
 	uint32_t m64window[6];
 	uint32_t m64ranges[2];
 	int i, j, nranges, rangeslen;
-	int32_t window;
+	uint32_t window;
+	uint32_t chip_id;
 	int64_t error;
 
 	if (faa->fa_nreg < 1) {
@@ -132,6 +133,11 @@ phb_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_phb_id = OF_getpropint64(sc->sc_node, "ibm,opal-phbid", 0);
 	sc->sc_pe_number = 0;
 
+	if (OF_getproplen(sc->sc_node, "ibm,chip-id") == sizeof(chip_id)) {
+		chip_id = OF_getpropint(sc->sc_node, "ibm,chip-id", 0);
+		printf(": chip 0x%x", chip_id);
+	}
+
 	/*
 	 * Reset the IODA tables.  Should clear any gunk left behind
 	 * by Linux.
@@ -473,7 +479,7 @@ phb_intr_establish(void *v, pci_intr_handle_t ih, int level,
 			return NULL;
 
 		cookie = intr_establish(sc->sc_msi_ranges[0] + xive,
-		    IST_EDGE, level, func, arg);
+		    IST_EDGE, level, func, arg, name);
 		if (cookie == NULL)
 			return NULL;
 
diff --git a/sys/arch/powerpc64/dev/xive.c b/sys/arch/powerpc64/dev/xive.c
new file mode 100644
index 00000000000..77e298fc597
--- /dev/null
+++ b/sys/arch/powerpc64/dev/xive.c
@@ -0,0 +1,418 @@
+/*	$OpenBSD: xive.c,v 1.1 2020/06/13 22:58:42 kettenis Exp $	*/
+/*
+ * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/device.h>
+#include <sys/evcount.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+#include <machine/opal.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/fdt.h>
+
+#define XIVE_NUM_PRIORITIES	8
+#define XIVE_NUM_IRQS		1024
+
+#define XIVE_EQ_SIZE		PAGE_SHIFT
+#define XIVE_EQ_IDX_MASK	((1 << (PAGE_SHIFT - 2)) - 1)
+#define XIVE_EQ_GEN_MASK	0x80000000
+
+#define XIVE_TM_CPPR_HV		0x031
+
+#define XIVE_TM_SPC_ACK_HV	0x830
+#define  XIVE_TM_SPC_ACK_HE_MASK	0xc000
+#define  XIVE_TM_SPC_ACK_HE_NONE	0x0000
+#define  XIVE_TM_SPC_ACK_HE_PHYS	0x8000
+
+#define XIVE_ESB_STORE_TRIGGER	0x000
+#define XIVE_ESB_SET_PQ_00	0xc00
+#define XIVE_ESB_SET_PQ_01	0xd00
+#define XIVE_ESB_SET_PQ_10	0xe00
+#define XIVE_ESB_SET_PQ_11	0xf00
+
+#define XIVE_ESB_VAL_P	0x2
+#define XIVE_ESB_VAL_Q	0x1
+
+static inline uint8_t
+xive_prio(int ipl)
+{
+	return ((IPL_IPI - ipl) > 7 ? 0xff : IPL_IPI - ipl);
+}
+
+static inline int
+xive_ipl(uint8_t prio)
+{
+	return (IPL_IPI - prio);
+}
+
+struct intrhand {
+	TAILQ_ENTRY(intrhand)	ih_list;
+	int			(*ih_func)(void *);
+	void			*ih_arg;
+	int			ih_ipl;
+	int			ih_flags;
+	uint32_t		ih_girq;
+	struct evcount		ih_count;
+	const char		*ih_name;
+
+	bus_space_handle_t	ih_esb_eoi;
+	bus_space_handle_t	ih_esb_trig;
+	uint64_t		ih_xive_flags;
+};
+
+struct xive_eq {
+	struct xive_dmamem	*eq_queue;
+	uint32_t		eq_idx;
+	uint32_t		eq_gen;
+};
+
+struct xive_softc {
+	struct device		sc_dev;
+	bus_space_tag_t		sc_iot;
+	bus_space_handle_t	sc_ioh;
+	bus_dma_tag_t		sc_dmat;
+
+	struct intrhand		*sc_handler[XIVE_NUM_IRQS];
+	struct xive_eq		sc_eq[XIVE_NUM_PRIORITIES];
+
+	uint32_t		sc_page_size;
+	uint32_t		sc_lirq;
+};
+
+struct xive_softc *xive_sc;
+
+struct xive_dmamem {
+	bus_dmamap_t		xdm_map;
+	bus_dma_segment_t	xdm_seg;
+	size_t			xdm_size;
+	caddr_t			xdm_kva;
+};
+
+#define XIVE_DMA_MAP(_xdm)	((_xdm)->xdm_map)
+#define XIVE_DMA_LEN(_xdm)	((_xdm)->xdm_size)
+#define XIVE_DMA_DVA(_xdm)	((_xdm)->xdm_map->dm_segs[0].ds_addr)
+#define XIVE_DMA_KVA(_xdm)	((void *)(_xdm)->xdm_kva)
+
+struct xive_dmamem *xive_dmamem_alloc(bus_dma_tag_t, bus_size_t,
+	    bus_size_t);
+void	xive_dmamem_free(bus_dma_tag_t, struct xive_dmamem *);
+
+static inline void
+xive_write_1(struct xive_softc *sc, bus_size_t off, uint8_t val)
+{
+	bus_space_write_1(sc->sc_iot, sc->sc_ioh, off, val);
+}
+
+static inline uint16_t
+xive_read_2(struct xive_softc *sc, bus_size_t off)
+{
+	return bus_space_read_2(sc->sc_iot, sc->sc_ioh, off);
+}
+
+int	xive_match(struct device *, void *, void *);
+void	xive_attach(struct device *, struct device *, void *);
+
+struct cfattach	xive_ca = {
+	sizeof (struct xive_softc), xive_match, xive_attach
+};
+
+struct cfdriver xive_cd = {
+	NULL, "xive", DV_DULL
+};
+
+void	xive_hvi(struct trapframe *);
+void 	*xive_intr_establish(uint32_t, int, int,
+	    int (*)(void *), void *, const char *);
+void	xive_setipl(int);
+
+int
+xive_match(struct device *parent, void *match, void *aux)
+{
+	struct fdt_attach_args *faa = aux;
+
+	return OF_is_compatible(faa->fa_node, "ibm,opal-xive-pe");
+}
+
+void
+xive_attach(struct device *parent, struct device *self, void *aux)
+{
+	struct xive_softc *sc = (struct xive_softc *)self;
+	struct fdt_attach_args *faa = aux;
+	struct cpu_info *ci = curcpu();
+	int64_t error;
+	int i;
+
+	if (faa->fa_nreg < 2) {
+		printf(": no registers\n");
+		return;
+	}
+
+	sc->sc_iot = faa->fa_iot;
+	if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr,
+	    faa->fa_reg[1].size, 0, &sc->sc_ioh)) {
+		printf(": can't map registers\n");
+		return;
+	}
+
+	sc->sc_dmat = faa->fa_dmat;
+
+	sc->sc_page_size = OF_getpropint(faa->fa_node,
+	    "ibm,xive-provision-page-size", 0);
+
+	error = opal_xive_reset(OPAL_XIVE_MODE_EXPL);
+	if (error != OPAL_SUCCESS)
+		printf(": can't enable exploitation mode\n");
+
+	printf("\n");
+
+	for (i = 0; i < XIVE_NUM_PRIORITIES; i++) {
+		sc->sc_eq[i].eq_queue = xive_dmamem_alloc(sc->sc_dmat,
+		    1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE);
+		if (sc->sc_eq[i].eq_queue == NULL) {
+			printf("%s: can't allocate event queue\n",
+			    sc->sc_dev.dv_xname);
+			return;
+		}
+
+		error = opal_xive_set_queue_info(mfpir(), i,
+		    XIVE_DMA_DVA(sc->sc_eq[i].eq_queue), XIVE_EQ_SIZE,
+		    OPAL_XIVE_EQ_ENABLED | OPAL_XIVE_EQ_ALWAYS_NOTIFY);
+		if (error != OPAL_SUCCESS) {
+			printf("%s: can't enable event queue\n",
+			    sc->sc_dev.dv_xname);
+			return;
+		}
+
+		sc->sc_eq[i].eq_gen = XIVE_EQ_GEN_MASK;
+	}
+
+	/* There can be only one. */
+	KASSERT(xive_sc == NULL);
+	xive_sc = sc;
+
+	_hvi = xive_hvi;
+	_intr_establish = xive_intr_establish;
+	_setipl = xive_setipl;
+
+	/* Synchronize hardware state to software state. */
+	xive_write_1(sc, XIVE_TM_CPPR_HV, ci->ci_cpl);
+}
+
+void *
+xive_intr_establish(uint32_t girq, int type, int level,
+    int (*func)(void *), void *arg, const char *name)
+{
+	struct xive_softc *sc = xive_sc;
+	struct intrhand *ih;
+	bus_space_handle_t eoi, trig;
+	bus_size_t page_size;
+	uint64_t flags, eoi_page, trig_page;
+	uint32_t esb_shift, lirq;
+	int64_t error;
+
+	/* Allocate a logical IRQ. */
+	if (sc->sc_lirq >= XIVE_NUM_IRQS)
+		return NULL;
+	lirq = sc->sc_lirq++;
+
+	error = opal_xive_get_irq_info(girq, &flags, &eoi_page,
+	    &trig_page, &esb_shift, NULL);
+	if (error != OPAL_SUCCESS)
+		return NULL;
+	page_size = 1 << esb_shift;
+
+	/* Map EOI page. */
+	if (bus_space_map(sc->sc_iot, eoi_page, page_size, 0, &eoi))
+		return NULL;
+
+	/* Map trigger page. */
+	if (trig_page == eoi_page)
+		trig = eoi;
+	else if (trig_page == 0)
+		trig = 0;
+	else if (bus_space_map(sc->sc_iot, trig_page, page_size, 0, &trig)) {
+		bus_space_unmap(sc->sc_iot, trig, page_size);
+		return NULL;
+	}
+
+	error = opal_xive_set_irq_config(girq, mfpir(),
+	    xive_prio(level & IPL_IRQMASK), lirq);
+	if (error != OPAL_SUCCESS) {
+		if (trig != eoi && trig != 0)
+			bus_space_unmap(sc->sc_iot, trig, page_size);
+		bus_space_unmap(sc->sc_iot, eoi, page_size);
+		return NULL;
+	}
+
+	ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK);
+	ih->ih_func = func;
+	ih->ih_arg = arg;
+	ih->ih_ipl = level & IPL_IRQMASK;
+	ih->ih_flags = level & IPL_FLAGMASK;
+	ih->ih_girq = girq;
+	ih->ih_name = name;
+	ih->ih_esb_eoi = eoi;
+	ih->ih_esb_trig = trig;
+	ih->ih_xive_flags = flags;
+
+	sc->sc_handler[lirq] = ih;
+
+	/* XXX Trigger interupt for debugging. */
+	bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
+	    XIVE_ESB_STORE_TRIGGER, 0);
+
+	return ih;
+}
+
+void
+xive_eoi(struct xive_softc *sc, struct intrhand *ih)
+{
+	uint64_t eoi;
+
+	eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, XIVE_ESB_SET_PQ_00);
+	if ((eoi & XIVE_ESB_VAL_Q) && ih->ih_esb_trig != 0)
+		bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
+		    XIVE_ESB_STORE_TRIGGER, 0);
+}
+
+void
+xive_setipl(int new)
+{
+	struct xive_softc *sc = xive_sc;
+	struct cpu_info *ci = curcpu();
+	uint8_t oldprio = xive_prio(ci->ci_cpl);
+	uint8_t newprio = xive_prio(new);
+	u_long msr;
+
+	msr = intr_disable();
+	ci->ci_cpl = new;
+	if (newprio != oldprio)
+		xive_write_1(sc, XIVE_TM_CPPR_HV, newprio);
+	intr_restore(msr);
+}
+
+void
+xive_hvi(struct trapframe *frame)
+{
+	struct xive_softc *sc = xive_sc;
+	struct cpu_info *ci = curcpu();
+	struct intrhand *ih;
+	struct xive_eq *eq;
+	uint32_t *event;
+	uint32_t lirq;
+	int old, new;
+	int handled;
+	uint16_t ack, he;
+	uint8_t cppr;
+
+	old = ci->ci_cpl;
+
+	while (1) {
+		ack = xive_read_2(sc, XIVE_TM_SPC_ACK_HV);
+
+		/* Synchronize software state to hardware state. */
+		cppr = ack;
+		new = xive_ipl(cppr);
+		KASSERT(new >= ci->ci_cpl);
+		ci->ci_cpl = new;
+
+		he = (ack & XIVE_TM_SPC_ACK_HE_MASK);
+		if (he == XIVE_TM_SPC_ACK_HE_NONE)
+			break;
+		KASSERT(he == XIVE_TM_SPC_ACK_HE_PHYS);
+
+		eieio();
+
+		KASSERT(cppr < XIVE_NUM_PRIORITIES);
+		eq = &sc->sc_eq[cppr];
+		event = XIVE_DMA_KVA(eq->eq_queue);
+		while ((event[eq->eq_idx] & XIVE_EQ_GEN_MASK) == eq->eq_gen) {
+			lirq = event[eq->eq_idx] & ~XIVE_EQ_GEN_MASK;
+			KASSERT(lirq < XIVE_NUM_IRQS);
+			ih = sc->sc_handler[lirq];
+			if (ih != NULL) {
+				intr_enable();
+				handled = ih->ih_func(ih->ih_arg);
+				intr_disable();
+				if (handled)
+					ih->ih_count.ec_count++;
+				xive_eoi(sc, ih);
+			}
+			eq->eq_idx = (eq->eq_idx + 1) & XIVE_EQ_IDX_MASK;
+
+			/* Toggle generation on wrap around. */
+			if (eq->eq_idx == 0)
+				eq->eq_gen ^= XIVE_EQ_GEN_MASK;
+		}
+	}
+
+	ci->ci_cpl = old;
+	xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(old));
+}
+
+struct xive_dmamem *
+xive_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align)
+{
+	struct xive_dmamem *xdm;
+	int nsegs;
+
+	xdm = malloc(sizeof(*xdm), M_DEVBUF, M_WAITOK | M_ZERO);
+	xdm->xdm_size = size;
+
+	if (bus_dmamap_create(dmat, size, 1, size, 0,
+	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &xdm->xdm_map) != 0)
+		goto xdmfree;
+
+	if (bus_dmamem_alloc(dmat, size, align, 0, &xdm->xdm_seg, 1,
+	    &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
+		goto destroy;
+
+	if (bus_dmamem_map(dmat, &xdm->xdm_seg, nsegs, size,
+	    &xdm->xdm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0)
+		goto free;
+
+	if (bus_dmamap_load_raw(dmat, xdm->xdm_map, &xdm->xdm_seg,
+	    nsegs, size, BUS_DMA_WAITOK) != 0)
+		goto unmap;
+
+	return xdm;
+
+unmap:
+	bus_dmamem_unmap(dmat, xdm->xdm_kva, size);
+free:
+	bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
+destroy:
+	bus_dmamap_destroy(dmat, xdm->xdm_map);
+xdmfree:
+	free(xdm, M_DEVBUF, sizeof(*xdm));
+
+	return NULL;
+}
+
+void
+xive_dmamem_free(bus_dma_tag_t dmat, struct xive_dmamem *xdm)
+{
+	bus_dmamem_unmap(dmat, xdm->xdm_kva, xdm->xdm_size);
+	bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
+	bus_dmamap_destroy(dmat, xdm->xdm_map);
+	free(xdm, M_DEVBUF, sizeof(*xdm));
+}
diff --git a/sys/arch/powerpc64/include/cpu.h b/sys/arch/powerpc64/include/cpu.h
index 51791d5fdde..ed1e2b49322 100644
--- a/sys/arch/powerpc64/include/cpu.h
+++ b/sys/arch/powerpc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.9 2020/06/10 19:06:53 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.10 2020/06/13 22:58:42 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -56,6 +56,7 @@ struct cpu_info {
 	
 	volatile int 	ci_cpl;
 	uint32_t	ci_ipending;
+	uint32_t	ci_idepth;
 #ifdef DIAGNOSTIC
 	int		ci_mutex_level;
 #endif
diff --git a/sys/arch/powerpc64/include/intr.h b/sys/arch/powerpc64/include/intr.h
index cbeb21ce3d5..092547cd827 100644
--- a/sys/arch/powerpc64/include/intr.h
+++ b/sys/arch/powerpc64/include/intr.h
@@ -1,38 +1,94 @@
+/*	$OpenBSD: intr.h,v 1.4 2020/06/13 22:58:42 kettenis Exp $	*/
+
+/*
+ * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _MACHINE_INTR_H_
+#define _MACHINE_INTR_H_
+
 #define IPL_NONE	0
-#define IPL_SOFTCLOCK	2
-#define IPL_SOFTNET	3
-#define IPL_SOFTTTY	4
-#define IPL_BIO		5
-#define IPL_NET		6
-#define IPL_TTY		7
-#define IPL_VM		8
-#define IPL_AUDIO	9
-#define IPL_CLOCK	10
+#define IPL_SOFTCLOCK	1
+#define IPL_SOFTNET	2
+#define IPL_SOFTTTY	3
+#define IPL_BIO		4
+#define IPL_NET		5
+#define IPL_TTY		6
+#define IPL_VM		IPL_TTY
+#define IPL_AUDIO	7
+#define IPL_CLOCK	8
 #define IPL_STATCLOCK	IPL_CLOCK
-#define IPL_HIGH	15
-#define IPL_MPFLOOR	IPL_TTY
-#define IPL_MPSAFE	0
-
-#define spl0()		9
-#define splsoftclock()	0
-#define splsoftnet()	0
-#define splsofttty()	0
-#define splbio()	0
-#define splnet()	0
-#define spltty()	0
-#define splvm()		0
-#define splclock()	0
-#define splstatclock()	0
-#define splsched()	0
-#define splhigh()	0
-#define splraise(s)	0
-#define splx(s)		(void)s
-
-#define splassert(x)
-#define splsoftassert(x)
+#define IPL_SCHED	IPL_CLOCK
+#define IPL_HIGH	IPL_CLOCK
+#define IPL_IPI		9
+
+#define	IPL_MPFLOOR	IPL_TTY
+/* Interrupt priority 'flags'. */
+#define	IPL_IRQMASK	0xf	/* priority only */
+#define	IPL_FLAGMASK	0xf00	/* flags only*/
+#define	IPL_MPSAFE	0x100	/* 'mpsafe' interrupt, no kernel lock */
+
+int	splraise(int);
+int	spllower(int);
+void	splx(int);
+
+#define spl0()		spllower(IPL_NONE)
+#define splsoftclock()	splraise(IPL_SOFTCLOCK)
+#define splsoftnet()	splraise(IPL_SOFTNET)
+#define splsofttty()	splraise(IPL_SOFTTTY)
+#define splbio()	splraise(IPL_BIO)
+#define splnet()	splraise(IPL_NET)
+#define spltty()	splraise(IPL_TTY)
+#define splvm()		splraise(IPL_VM)
+#define splclock()	splraise(IPL_CLOCK)
+#define splstatclock()	splraise(IPL_STATCLOCK)
+#define splsched()	splraise(IPL_SCHED)
+#define splhigh()	splraise(IPL_HIGH)
+
+#ifdef DIAGNOSTIC
+/*
+ * Although this function is implemented in MI code, it must be in this MD
+ * header because we don't want this header to include MI includes.
+ */
+void splassert_fail(int, int, const char *);
+extern int splassert_ctl;
+void splassert_check(int, const char *);
+#define splassert(__wantipl) do {			\
+	if (splassert_ctl > 0) {			\
+		splassert_check(__wantipl, __func__);	\
+	}						\
+} while (0)
+#define	splsoftassert(wantipl)	splassert(wantipl)
+#else
+#define	splassert(wantipl)	do { /* nothing */ } while (0)
+#define	splsoftassert(wantipl)	do { /* nothing */ } while (0)
+#endif
 
 #define intr_barrier(x)
 
-#define intr_establish(girq, type, level, func, arg) (arg)
+#define IST_EDGE	0
+#define IST_LEVEL	1
+
+void	*intr_establish(uint32_t, int, int,
+	    int (*)(void *), void *, const char *);
+
+extern void (*_hvi)(struct trapframe *);
+extern void *(*_intr_establish)(uint32_t, int, int,
+	    int (*)(void *), void *, const char *);
+extern void (*_setipl)(int);
 
 #include <machine/softintr.h>
+
+#endif /* _MACHINE_INTR_H_ */
diff --git a/sys/arch/powerpc64/include/opal.h b/sys/arch/powerpc64/include/opal.h
index 1fb16fc8159..0a3f2094d88 100644
--- a/sys/arch/powerpc64/include/opal.h
+++ b/sys/arch/powerpc64/include/opal.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: opal.h,v 1.8 2020/06/10 19:00:02 kettenis Exp $	*/
+/*	$OpenBSD: opal.h,v 1.9 2020/06/13 22:58:42 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -41,6 +41,15 @@
 #define OPAL_GET_MSI_64			40
 #define OPAL_PCI_MAP_PE_DMA_WINDOW_REAL	45
 #define OPAL_PCI_RESET			49
+#define OPAL_XIVE_RESET			128
+#define OPAL_XIVE_GET_IRQ_INFO		129
+#define OPAL_XIVE_GET_IRQ_CONFIG	131
+#define OPAL_XIVE_SET_IRQ_CONFIG	131
+#define OPAL_XIVE_GET_QUEUE_INFO	132
+#define OPAL_XIVE_SET_QUEUE_INFO	133
+#define OPAL_XIVE_GET_VP_INFO		137
+#define OPAL_XIVE_SET_VP_INFO		138
+#define OPAL_XIVE_DUMP			142
 
 /* Return codes. */
 #define OPAL_SUCCESS			0
@@ -92,6 +101,35 @@
 #define OPAL_DEASSERT_RESET		0
 #define OPAL_ASSERT_RESET		1
 
+/* OPAL_XIVE_RESET */
+#define OPAL_XIVE_MODE_EMU		0
+#define OPAL_XIVE_MODE_EXPL		1
+
+/* OPAL_XIVE_GET_IRQ_INFO */
+#define OPAL_XIVE_IRQ_TRIGGER_PAGE	0x00000001
+#define OPAL_XIVE_IRQ_STORE_EOI		0x00000002
+#define OPAL_XIVE_IRQ_LSI		0x00000004
+#define OPAL_XIVE_IRQ_SHIFT_BUG		0x00000008
+#define OPAL_XIVE_IRQ_MASK_VIA_FW	0x00000010
+#define OPAL_XIVE_IRQ_EOI_VIA_FW	0x00000020
+
+/* OPAL_XIVE_GET_QUEUE_INFO */
+#define OPAL_XIVE_EQ_ENABLED		0x00000001
+#define OPAL_XIVE_EQ_ALWAYS_NOTIFY	0x00000002
+#define OPAL_XIVE_EQ_ESCALATE		0x00000004
+
+/* OPAL_XIVE_GET_VP_INFO */
+#define OPAL_XIVE_VP_ENABLED		0x00000001
+#define OPAL_XIVE_VP_SINGLE_ESCALATION	0x00000002
+
+/* OPAL_XIVE_DUMP */
+#define XIVE_DUMP_TM_HYP	0x00000000
+#define XIVE_DUMP_TM_POOL	0x00000001
+#define XIVE_DUMP_TM_OS		0x00000002
+#define XIVE_DUMP_TM_USER	0x00000003
+#define XIVE_DUMP_VP		0x00000004
+#define XIVE_DUMP_EMU_STATE	0x00000005
+
 #ifndef _LOCORE
 int64_t	opal_test(uint64_t);
 int64_t	opal_console_write(int64_t, int64_t *, const uint8_t *);
@@ -121,6 +159,19 @@ int64_t	opal_get_msi_64(uint64_t, uint32_t, uint32_t, uint8_t,
 int64_t opal_pci_map_pe_dma_window_real(uint64_t, uint64_t, uint16_t,
 	    uint64_t, uint64_t);
 int64_t opal_pci_reset(uint64_t, uint8_t, uint8_t);
+int64_t	opal_xive_reset(uint64_t);
+int64_t	opal_xive_get_irq_info(uint32_t, uint64_t *, uint64_t *,
+	    uint64_t *, uint32_t *, uint32_t *);
+int64_t	opal_xive_get_irq_config(uint32_t, uint64_t *, uint8_t *, uint32_t *);
+int64_t	opal_xive_set_irq_config(uint32_t, uint64_t, uint8_t, uint32_t);
+int64_t	opal_xive_get_queue_info(uint64_t, uint8_t, uint64_t *,
+	    uint64_t *, uint64_t *, uint32_t *, uint64_t *);
+int64_t	opal_xive_set_queue_info(uint64_t, uint8_t, uint64_t,
+	    uint64_t, uint64_t);
+int64_t	opal_xive_get_vp_info(uint64_t, uint64_t *, uint64_t *,
+	    uint64_t *, uint32_t *);
+int64_t	opal_xive_set_vp_info(uint64_t, uint64_t, uint64_t);
+int64_t	opal_xive_dump(uint32_t, uint32_t);
 
 void	opal_printf(const char *fmt, ...);
 #endif
diff --git a/sys/arch/powerpc64/include/trap.h b/sys/arch/powerpc64/include/trap.h
index 3b328a529a4..a9e18d3617e 100644
--- a/sys/arch/powerpc64/include/trap.h
+++ b/sys/arch/powerpc64/include/trap.h
@@ -144,5 +144,6 @@
 
 /* Magic pointer to store trap handler entry point */
 #define	TRAP_ENTRY	0x1f8
+#define	TRAP_HVENTRY	0x1f0
 
 #endif	/* _MACHINE_TRAP_H_ */
diff --git a/sys/arch/powerpc64/powerpc64/intr.c b/sys/arch/powerpc64/powerpc64/intr.c
new file mode 100644
index 00000000000..c7c4932b99f
--- /dev/null
+++ b/sys/arch/powerpc64/powerpc64/intr.c
@@ -0,0 +1,122 @@
+/*	$OpenBSD: intr.c,v 1.1 2020/06/13 22:58:42 kettenis Exp $	*/
+
+/*
+ * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/intr.h>
+
+/* Dummy implementations. */
+void	dummy_hvi(struct trapframe *);
+void	*dummy_intr_establish(uint32_t, int, int,
+	    int (*)(void *), void *, const char *);
+void	dummy_setipl(int);
+
+/*
+ * The function pointers are overridden when the driver for the real
+ * interrupt controller attaches.
+ */
+void	(*_hvi)(struct trapframe *) = dummy_hvi;
+void	*(*_intr_establish)(uint32_t, int, int,
+	    int (*)(void *), void *, const char *) = dummy_intr_establish;
+void	(*_setipl)(int) = dummy_setipl;
+
+void
+hvi_intr(struct trapframe *frame)
+{
+	(*_hvi)(frame);
+}
+
+void *
+intr_establish(uint32_t girq, int type, int level,
+    int (*func)(void *), void *arg, const char *name)
+{
+	return (*_intr_establish)(girq, type, level, func, arg, name);
+}
+
+int
+splraise(int new)
+{
+	struct cpu_info *ci = curcpu();
+	int old = ci->ci_cpl;
+
+	if (new > old)
+		(*_setipl)(new);
+	return old;
+}
+
+int
+spllower(int new)
+{
+	struct cpu_info *ci = curcpu();
+	int old = ci->ci_cpl;
+
+	if (new < old)
+		(*_setipl)(new);
+	return old;
+}
+
+void
+splx(int new)
+{
+	struct cpu_info *ci = curcpu();
+
+	if (ci->ci_cpl != new)
+		(*_setipl)(new);
+}
+
+#ifdef DIAGNOSTIC
+void
+splassert_check(int wantipl, const char *func)
+{
+	int oldipl = curcpu()->ci_cpl;
+
+	if (oldipl < wantipl) {
+		splassert_fail(wantipl, oldipl, func);
+		/*
+		 * If the splassert_ctl is set to not panic, raise the ipl
+		 * in a feeble attempt to reduce damage.
+		 */
+		(*_setipl)(wantipl);
+	}
+
+	if (wantipl == IPL_NONE && curcpu()->ci_idepth != 0) {
+		splassert_fail(-1, curcpu()->ci_idepth, func);
+	}
+}
+#endif
+
+void
+dummy_hvi(struct trapframe *frame)
+{
+	panic("Unhandled Hypervisor Virtualization interrupt");
+}
+
+void *
+dummy_intr_establish(uint32_t girq, int type, int level,
+	    int (*func)(void *), void *arg, const char *name)
+{
+	return NULL;
+}
+
+void
+dummy_setipl(int new)
+{
+	struct cpu_info *ci = curcpu();
+	ci->ci_cpl = new;
+}
diff --git a/sys/arch/powerpc64/powerpc64/locore.S b/sys/arch/powerpc64/powerpc64/locore.S
index 9b270504a9a..d3463a44069 100644
--- a/sys/arch/powerpc64/powerpc64/locore.S
+++ b/sys/arch/powerpc64/powerpc64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.11 2020/06/12 22:01:01 gkoehler Exp $	*/
+/*	$OpenBSD: locore.S,v 1.12 2020/06/13 22:58:42 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -99,6 +99,15 @@ OPAL_CALL(OPAL_GET_MSI_32, opal_get_msi_32)
 OPAL_CALL(OPAL_GET_MSI_64, opal_get_msi_64)
 OPAL_CALL(OPAL_PCI_MAP_PE_DMA_WINDOW_REAL, opal_pci_map_pe_dma_window_real)
 OPAL_CALL(OPAL_PCI_RESET, opal_pci_reset)
+OPAL_CALL(OPAL_XIVE_RESET, opal_xive_reset)
+OPAL_CALL(OPAL_XIVE_GET_IRQ_INFO, opal_xive_get_irq_info)
+OPAL_CALL(OPAL_XIVE_GET_IRQ_CONFIG, opal_xive_get_irq_config)
+OPAL_CALL(OPAL_XIVE_SET_IRQ_CONFIG, opal_xive_set_irq_config)
+OPAL_CALL(OPAL_XIVE_GET_QUEUE_INFO, opal_xive_get_queue_info)
+OPAL_CALL(OPAL_XIVE_SET_QUEUE_INFO, opal_xive_set_queue_info)
+OPAL_CALL(OPAL_XIVE_GET_VP_INFO, opal_xive_get_vp_info)
+OPAL_CALL(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info)
+OPAL_CALL(OPAL_XIVE_DUMP, opal_xive_dump)
 
 opal_call:
 	mflr	%r11
diff --git a/sys/arch/powerpc64/powerpc64/machdep.c b/sys/arch/powerpc64/powerpc64/machdep.c
index 3e086fc0af8..dc6b2fe4c24 100644
--- a/sys/arch/powerpc64/powerpc64/machdep.c
+++ b/sys/arch/powerpc64/powerpc64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.19 2020/06/10 19:06:53 kettenis Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.20 2020/06/13 22:58:42 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -66,7 +66,9 @@ extern uint64_t opal_base;
 extern uint64_t opal_entry;
 
 extern char trapcode[], trapcodeend[];
+extern char hvtrapcode[], hvtrapcodeend[];
 extern char generictrap[];
+extern char generichvtrap[];
 
 struct fdt_reg memreg[VM_PHYSSEG_MAX];
 int nmemreg;
@@ -125,9 +127,15 @@ init_powernv(void *fdt, void *tocbase)
 	 */
 	for (trap = EXC_RST; trap < EXC_LAST; trap += 32)
 		memcpy((void *)trap, trapcode, trapcodeend - trapcode);
-	__syncicache(EXC_RSVD, EXC_LAST - EXC_RSVD);
+
+	/* Hypervisor Virtualization interrupt needs special handling. */
+	memcpy((void *)EXC_HVI, hvtrapcode, hvtrapcodeend - hvtrapcode);
 
 	*((void **)TRAP_ENTRY) = generictrap;
+	*((void **)TRAP_HVENTRY) = generichvtrap;
+
+	/* Make the stubs visible to the CPU. */
+	__syncicache(EXC_RSVD, EXC_LAST - EXC_RSVD);
 
 	/* We're now ready to take traps. */
 	msr = mfmsr();
diff --git a/sys/arch/powerpc64/powerpc64/trap.c b/sys/arch/powerpc64/powerpc64/trap.c
index e7f4d086336..fd9edd4075b 100644
--- a/sys/arch/powerpc64/powerpc64/trap.c
+++ b/sys/arch/powerpc64/powerpc64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.5 2020/06/12 22:01:01 gkoehler Exp $	*/
+/*	$OpenBSD: trap.c,v 1.6 2020/06/13 22:58:42 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -25,6 +25,7 @@
 #include <machine/trap.h>
 
 void	decr_intr(struct trapframe *); /* clock.c */
+void	hvi_intr(struct trapframe *);
 
 void
 trap(struct trapframe *frame)
@@ -33,6 +34,9 @@ trap(struct trapframe *frame)
 	case EXC_DECR:
 		decr_intr(frame);
 		return;
+	case EXC_HVI:
+		hvi_intr(frame);
+		return;
 #ifdef DDB
 	case EXC_PGM:
 		/* At a trap instruction, enter the debugger. */
diff --git a/sys/arch/powerpc64/powerpc64/trap_subr.S b/sys/arch/powerpc64/powerpc64/trap_subr.S
index be1544fa363..0f1a633da1a 100644
--- a/sys/arch/powerpc64/powerpc64/trap_subr.S
+++ b/sys/arch/powerpc64/powerpc64/trap_subr.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: trap_subr.S,v 1.5 2020/06/09 18:57:33 kettenis Exp $ */
+/* $OpenBSD: trap_subr.S,v 1.6 2020/06/13 22:58:42 kettenis Exp $ */
 /* $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $	*/
 
 /*-
@@ -214,6 +214,32 @@ trapcode:
 	blrl
 trapcodeend:
 
+	.globl hvtrapcode, hvtrapcodeend
+hvtrapcode:
+	mtsprg1	%r1
+	mflr	%r1
+	mtsprg2	%r1
+	ld	%r1, TRAP_HVENTRY(0)
+	mtlr	%r1
+	li	%r1, 0xe0
+	blrl
+hvtrapcodeend:
+
+/*
+ * generichvtrap makes a hypervisor trap look like a normal trap.
+ */
+
+	.globl generichvtrap
+generichvtrap:
+	/* Move HSRR0/HSRR1 to SSR0/SRR1 */
+	mtsprg3	%r1
+	mfspr	%r1, 314	/* HSRR0 */
+	mtsrr0	%r1
+	mfspr	%r1, 315	/* HSRR1 */
+	mtsrr1	%r1
+	mfsprg3	%r1
+	/* FALLTHROUGH */
+
 /*
  * generictrap does some standard setup for trap handling to minimize
  * the code that need be installed in the actual vectors. It expects