Implement pci_intr_establish_cpu() for pyro(4) and vpci(4) based sparc64

systems.  MSIs on these systems are delivered to event queues, which
trigger interrupts when non-empty.  The interrupt handler dequeues the
MSIs and converts them into soft interrupts, which run on the same cpu
as the event queue interrupt.

To target pci device interrupts to different cpus, we set up an event
queue per cpu in the system, or as many as we can, if there are fewer
event queues available.  For now, we don't have a way to feed this
information back to intrmap, so instead we just map interrupts for cpus
that don't have an event queue to another cpu that does have one.

Tested on V215 (pyro), T5120, T4-1, S7-2 (vpci).
dlg@ got the pyro side of it working for me.
ok dlg@ kettenis@

10 files changed, 426 insertions, 129 deletions

diff --git a/sys/arch/sparc64/dev/msi.c b/sys/arch/sparc64/dev/msi.c
index 3bbab713808..cfc8bc75e8a 100644
--- a/sys/arch/sparc64/dev/msi.c
+++ b/sys/arch/sparc64/dev/msi.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: msi.c,v 1.4 2015/09/08 10:24:25 deraadt Exp $	*/
+/*	$OpenBSD: msi.c,v 1.5 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*
  * Copyright (c) 2011 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -28,10 +28,10 @@ struct msi_msg {
 };
 
 struct msi_eq *
-msi_eq_alloc(bus_dma_tag_t t, int msi_eq_size)
+msi_eq_alloc(bus_dma_tag_t t, int msi_eq_size, int num_eq)
 {
 	struct msi_eq *meq;
-	bus_size_t size;
+	bus_size_t eqsize, size;
 	caddr_t va;
 	int nsegs;
 
@@ -39,13 +39,15 @@ msi_eq_alloc(bus_dma_tag_t t, int msi_eq_size)
 	if (meq == NULL)
 		return NULL;
 
-	size = roundup(msi_eq_size * sizeof(struct msi_msg), PAGE_SIZE);
+	eqsize = roundup(msi_eq_size * sizeof(struct msi_msg),
+	    PAGE_SIZE);
+	size = num_eq * eqsize;
 
 	if (bus_dmamap_create(t, size, 1, size, 0,
 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &meq->meq_map) != 0)
 		return (NULL);
 
-	if (bus_dmamem_alloc(t, size, size, 0, &meq->meq_seg, 1,
+	if (bus_dmamem_alloc(t, size, eqsize, 0, &meq->meq_seg, 1,
 	    &nsegs, BUS_DMA_NOWAIT) != 0)
 		goto destroy;
 
@@ -59,6 +61,8 @@ msi_eq_alloc(bus_dma_tag_t t, int msi_eq_size)
 
 	meq->meq_va = va;
 	meq->meq_nentries = msi_eq_size;
+	meq->meq_queuesize = eqsize;
+	meq->meq_nqueues = num_eq;
 	return (meq);
 
 unmap:
@@ -74,13 +78,15 @@ destroy:
 void
 msi_eq_free(bus_dma_tag_t t, struct msi_eq *meq)
 {
-	bus_size_t size;
-
-	size = roundup(meq->meq_nentries * sizeof(struct msi_msg), PAGE_SIZE);
-
 	bus_dmamap_unload(t, meq->meq_map);
-	bus_dmamem_unmap(t, meq->meq_va, size);
+	bus_dmamem_unmap(t, meq->meq_va, meq->meq_nqueues * meq->meq_queuesize);
 	bus_dmamem_free(t, &meq->meq_seg, 1);
 	bus_dmamap_destroy(t, meq->meq_map);
 	free(meq, M_DEVBUF, sizeof *meq);
 }
+
+size_t
+msi_eq_offset(struct msi_eq *meq, int eq)
+{
+	return (meq->meq_queuesize * eq);
+}
diff --git a/sys/arch/sparc64/dev/msivar.h b/sys/arch/sparc64/dev/msivar.h
index b55552f1ad3..ef4bc4e6ded 100644
--- a/sys/arch/sparc64/dev/msivar.h
+++ b/sys/arch/sparc64/dev/msivar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: msivar.h,v 1.1 2011/07/06 05:35:53 kettenis Exp $	*/
+/*	$OpenBSD: msivar.h,v 1.2 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*
  * Copyright (c) 2011 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -15,12 +15,15 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-struct msi_eq *msi_eq_alloc(bus_dma_tag_t, int);
+struct msi_eq *msi_eq_alloc(bus_dma_tag_t, int, int);
+size_t msi_eq_offset(struct msi_eq *, int);
 void msi_eq_free(bus_dma_tag_t t, struct msi_eq *);
 
 struct msi_eq {
 	bus_dmamap_t	meq_map;
 	bus_dma_segment_t meq_seg;
 	caddr_t		meq_va;
+	size_t		meq_queuesize;
 	int		meq_nentries;
+	int		meq_nqueues;
 };
diff --git a/sys/arch/sparc64/dev/pci_machdep.c b/sys/arch/sparc64/dev/pci_machdep.c
index 71cb3abc527..9281dbd0299 100644
--- a/sys/arch/sparc64/dev/pci_machdep.c
+++ b/sys/arch/sparc64/dev/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.50 2020/06/17 01:15:32 dlg Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.51 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.22 2001/07/20 00:07:13 eeh Exp $	*/
 
 /*
@@ -523,6 +523,14 @@ pci_intr_establish(pc, ih, level, func, arg, what)
 	void *arg;
 	const char *what;
 {
+	return (pci_intr_establish_cpu(pc, ih, level, NULL, func, arg, what));
+}
+
+void *
+pci_intr_establish_cpu(pci_chipset_tag_t pc, pci_intr_handle_t ih,
+    int level, struct cpu_info *ci,
+    int (*func)(void *), void *arg, const char *what)
+{
 	void *cookie;
 	int flags = 0;
 
@@ -531,10 +539,10 @@ pci_intr_establish(pc, ih, level, func, arg, what)
 		level &= ~IPL_MPSAFE;
 	}
 
-	DPRINTF(SPDB_INTR, ("pci_intr_establish: ih %lu; level %d",
-	    (u_long)ih, level));
-	cookie = bus_intr_establish(pc->bustag, ih, level, flags,
-	    func, arg, what);
+	DPRINTF(SPDB_INTR, ("pci_intr_establish_cpu: ih %lu; level %d; ci %p",
+	    (u_long)ih, level, ci));
+	cookie = bus_intr_establish_cpu(pc->bustag, ih, level, flags,
+	    ci, func, arg, what);
 
 	DPRINTF(SPDB_INTR, ("; returning handle %p\n", cookie));
 	return (cookie);
diff --git a/sys/arch/sparc64/dev/pyro.c b/sys/arch/sparc64/dev/pyro.c
index 2769f71a536..4881f8d5514 100644
--- a/sys/arch/sparc64/dev/pyro.c
+++ b/sys/arch/sparc64/dev/pyro.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pyro.c,v 1.33 2019/06/25 22:30:56 dlg Exp $	*/
+/*	$OpenBSD: pyro.c,v 1.34 2020/06/23 01:21:29 jmatthew Exp $	*/
 
 /*
  * Copyright (c) 2002 Jason L. Wright (jason@thought.net)
@@ -62,18 +62,21 @@ int pyro_debug = ~0;
 #define DPRINTF(l, s)
 #endif
 
+#define FIRE_INTR_MAP(_n)		0x01000 + ((_n) * 8)
+#define FIRE_INTR_CLR(_n)		0x01400 + ((_n) * 8)
+
 #define FIRE_EQ_BASE_ADDR		0x10000
-#define FIRE_EQ_CNTRL_SET		0x11000
+#define FIRE_EQ_CNTRL_SET(_n)		0x11000 + ((_n) * 8)
 #define  FIRE_EQ_CTRL_SET_EN		0x0000100000000000UL
-#define FIRE_EQ_CNTRL_CLEAR		0x11200
-#define FIRE_EQ_STATE			0x11400
-#define FIRE_EQ_TAIL			0x11600
-#define FIRE_EQ_HEAD			0x11800
-#define FIRE_MSI_MAP			0x20000
+#define FIRE_EQ_CNTRL_CLEAR(_n)		0x11200 + ((_n) * 8)
+#define FIRE_EQ_STATE(_n)		0x11400 + ((_n) * 8)
+#define FIRE_EQ_TAIL(_n)		0x11600 + ((_n) * 8)
+#define FIRE_EQ_HEAD(_n)		0x11800 + ((_n) * 8)
+#define FIRE_MSI_MAP(_n)		0x20000 + ((_n) * 8)
 #define  FIRE_MSI_MAP_V			0x8000000000000000UL
 #define  FIRE_MSI_MAP_EQWR_N		0x4000000000000000UL
 #define  FIRE_MSI_MAP_EQNUM		0x000000000000003fUL
-#define FIRE_MSI_CLEAR			0x28000
+#define FIRE_MSI_CLEAR(_n)		0x28000 + ((_n) * 8)
 #define  FIRE_MSI_CLEAR_EQWR_N		0x4000000000000000UL
 #define FIRE_INTRMONDO_DATA0		0x2c000
 #define FIRE_INTRMONDO_DATA1		0x2c008
@@ -102,6 +105,9 @@ void pyro_attach(struct device *, struct device *, void *);
 void pyro_init(struct pyro_softc *, int);
 void pyro_init_iommu(struct pyro_softc *, struct pyro_pbm *);
 void pyro_init_msi(struct pyro_softc *, struct pyro_pbm *);
+#if 0
+void pyro_redistribute_msi(struct device *);
+#endif
 int pyro_print(void *, const char *);
 
 pci_chipset_tag_t pyro_alloc_chipset(struct pyro_pbm *, int,
@@ -124,6 +130,8 @@ paddr_t pyro_bus_mmap(bus_space_tag_t, bus_space_tag_t, bus_addr_t, off_t,
     int, int);
 void *pyro_intr_establish(bus_space_tag_t, bus_space_tag_t, int, int, int,
     int (*)(void *), void *, const char *);
+void *pyro_intr_establish_cpu(bus_space_tag_t, bus_space_tag_t, int, int, int,
+    struct cpu_info *, int (*)(void *), void *, const char *);
 void pyro_msi_ack(struct intrhand *);
 
 int pyro_msi_eq_intr(void *);
@@ -250,6 +258,9 @@ pyro_init(struct pyro_softc *sc, int busa)
 	pbm->pp_cfgt = pyro_alloc_config_tag(pbm);
 	pbm->pp_dmat = pyro_alloc_dma_tag(pbm);
 
+#if 0
+	sc->sc_pbm = pbm;
+#endif
 	pyro_init_msi(sc, pbm);
 
 	if (bus_space_map(pbm->pp_cfgt, 0, 0x10000000, 0, &pbm->pp_cfgh))
@@ -332,7 +343,15 @@ pyro_init_msi(struct pyro_softc *sc, struct pyro_pbm *pbm)
 	u_int32_t msi_addr_range[3];
 	u_int32_t msi_eq_devino[3] = { 0, 36, 24 };
 	int ihandle;
-	int msis, msi_eq_size;
+	int msis, msi_eq_size, num_eq;
+	struct pyro_eq *eq;
+	struct msi_eq *meq;
+	struct pyro_msi_msg *msgs;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+
+	/* One queue per cpu. */
+	num_eq = ncpus;
 
 	if (OF_getprop(sc->sc_node, "msi-address-ranges",
 	    msi_addr_range, sizeof(msi_addr_range)) <= 0)
@@ -347,12 +366,13 @@ pyro_init_msi(struct pyro_softc *sc, struct pyro_pbm *pbm)
 		return;
 
 	msi_eq_size = getpropint(sc->sc_node, "msi-eq-size", 256);
-	pbm->pp_meq = msi_eq_alloc(pbm->pp_dmat, msi_eq_size);
+	pbm->pp_meq = msi_eq_alloc(pbm->pp_dmat, msi_eq_size, num_eq);
 	if (pbm->pp_meq == NULL)
 		goto free_table;
 
 	bzero(pbm->pp_meq->meq_va,
-	    pbm->pp_meq->meq_nentries * sizeof(struct pyro_msi_msg));
+	    pbm->pp_meq->meq_nentries * sizeof(struct pyro_msi_msg) *
+	    num_eq);
 
 	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_BASE_ADDR,
 	    pbm->pp_meq->meq_map->dm_segs[0].ds_addr);
@@ -365,28 +385,127 @@ pyro_init_msi(struct pyro_softc *sc, struct pyro_pbm *pbm)
 	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_MSI32_ADDR,
 	    pbm->pp_msiaddr);
 
-	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_HEAD, 0);
-	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_TAIL, 0);
+	if (OF_getprop(sc->sc_node, "msi-eq-to-devino",
+	    msi_eq_devino, sizeof(msi_eq_devino)) == -1) {
+		OF_getprop(sc->sc_node, "msi-eq-devino",
+		    msi_eq_devino, sizeof(msi_eq_devino));
+	}
 
-	OF_getprop(sc->sc_node, "msi-eq-to-devino",
-	    msi_eq_devino, sizeof(msi_eq_devino));
+	pbm->pp_eq = mallocarray(num_eq, sizeof(*eq), M_DEVBUF, M_WAITOK);
+	pbm->pp_neq = num_eq;
 
-	ihandle = msi_eq_devino[2] | sc->sc_ign;
-	if (pyro_intr_establish(pbm->pp_memt, sc->sc_bust, ihandle,
-	    IPL_HIGH, 0, pyro_msi_eq_intr, pbm, sc->sc_dv.dv_xname) == NULL)
-		goto free_table;
+	meq = pbm->pp_meq;
+	msgs = (struct pyro_msi_msg *)meq->meq_va;
 
-	/* Enable EQ. */
-	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_CNTRL_SET,
-	    FIRE_EQ_CTRL_SET_EN);
+	CPU_INFO_FOREACH(cii, ci) {
+		int unit = CPU_INFO_UNIT(ci);
+		eq = &pbm->pp_eq[unit];
+
+		eq->eq_id = unit;
+		eq->eq_intr = msi_eq_devino[2] + unit;
+		eq->eq_pbm = pbm;
+		snprintf(eq->eq_name, sizeof(eq->eq_name), "%s:%d",
+		    sc->sc_dv.dv_xname, unit);
+		eq->eq_head = FIRE_EQ_HEAD(unit);
+		eq->eq_tail = FIRE_EQ_TAIL(unit);
+		eq->eq_ring = msgs + (unit * meq->meq_nentries);
+		eq->eq_mask = (meq->meq_nentries - 1);
+
+		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_EQ_HEAD(unit), 0);
+		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_EQ_TAIL(unit), 0);
+
+		ihandle = eq->eq_intr | sc->sc_ign;
+		eq->eq_ih = pyro_intr_establish_cpu(pbm->pp_memt, sc->sc_bust,
+		    ihandle, IPL_HIGH, BUS_INTR_ESTABLISH_MPSAFE, ci,
+		    pyro_msi_eq_intr, eq, eq->eq_name);
+		if (eq->eq_ih == NULL) {
+			/* XXX */
+			goto free_table;
+		}
+
+		/* Enable EQ. */
+		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_EQ_CNTRL_SET(unit), FIRE_EQ_CTRL_SET_EN);
+	}
 
 	pbm->pp_flags |= PCI_FLAGS_MSI_ENABLED;
+
+#if 0
+	/*
+         * XXX some devices may interrupt before a cpu has hatched,
+         * so rather than have their interrupts get dropped because
+         * the other cpu isn't running, point the interrupts at the
+         * boot cpu and redistribute them later on. this assumes that
+         * only msi and msix interrupts get targetted to other CPUs,
+         * so only the msi eqs need to be redistributed.
+	 */
+	config_mountroot(&sc->sc_dv, pyro_redistribute_msi);
+#endif
+
 	return;
 
 free_table:
 	free(pbm->pp_msi, M_DEVBUF, 0);
 }
 
+#if 0
+void
+pyro_redistribute_msi(struct device *dev)
+{
+	struct pyro_softc *sc = (struct pyro_softc *)dev;
+	struct pyro_pbm *pbm = sc->sc_pbm;
+	struct pyro_eq *eq;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+	uint64_t map, clr;
+	int i;
+
+	CPU_INFO_FOREACH(cii, ci) {
+		unsigned int unit = CPU_INFO_UNIT(ci);
+		struct intrhand *ih;
+
+		eq = &pbm->pp_eq[unit];
+		ih = eq->eq_ih;
+
+		map = bus_space_read_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_INTR_MAP(eq->eq_intr));
+		CLR(map, INTMAP_V);
+		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_INTR_MAP(eq->eq_intr), map);
+
+		/* wait for pending to clear */
+		for (i = 0; i < 10000; i++) {
+			clr = bus_space_read_8(sc->sc_bust, sc->sc_csrh,
+			    FIRE_INTR_CLR(eq->eq_intr));
+			clr &= 0x3;
+			if (clr != 0x3)
+				break;
+
+			delay(10000);
+		}
+
+		if (clr == 0x3) {
+			panic("%s: unable to clear pending state on eq %u",
+			    sc->sc_dv.dv_xname, eq->eq_id);
+		}
+
+		if (sc->sc_oberon) {
+			CLR(map, OBERON_INTRMAP_T_DESTID_MASK);
+			SET(map, ci->ci_upaid << OBERON_INTRMAP_T_DESTID_SHIFT);
+		} else {
+			CLR(map, FIRE_INTRMAP_T_JPID_MASK);
+			SET(map, ci->ci_upaid << FIRE_INTRMAP_T_JPID_SHIFT);
+		}
+		SET(map, INTMAP_V);
+
+		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
+		    FIRE_INTR_MAP(eq->eq_intr), map);
+	}
+}
+#endif
+
 int
 pyro_print(void *aux, const char *p)
 {
@@ -513,6 +632,7 @@ pyro_alloc_bus_tag(struct pyro_pbm *pbm, const char *name, int ss,
 	bt->sparc_bus_map = pyro_bus_map;
 	bt->sparc_bus_mmap = pyro_bus_mmap;
 	bt->sparc_intr_establish = pyro_intr_establish;
+	bt->sparc_intr_establish_cpu = pyro_intr_establish_cpu;
 	return (bt);
 }
 
@@ -642,6 +762,15 @@ void *
 pyro_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
     int level, int flags, int (*handler)(void *), void *arg, const char *what)
 {
+	return (pyro_intr_establish_cpu(t, t0, ihandle, level, flags, NULL,
+	    handler, arg, what));
+}
+
+void *
+pyro_intr_establish_cpu(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
+    int level, int flags, struct cpu_info *ci,
+    int (*handler)(void *), void *arg, const char *what)
+{
 	struct pyro_pbm *pbm = t->cookie;
 	struct pyro_softc *sc = pbm->pp_sc;
 	struct intrhand *ih = NULL;
@@ -661,6 +790,10 @@ pyro_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 
 		evcount_attach(&ih->ih_count, ih->ih_name, NULL);
 
+		if (ci == NULL)
+			ci = cpus; /* Default to the boot cpu. */
+
+		ih->ih_cpu = ci;
 		ih->ih_ack = pyro_msi_ack;
 
 		pbm->pp_msi[msinum] = ih;
@@ -681,19 +814,20 @@ pyro_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 
 		/* Map MSI to the right EQ and mark it as valid. */
 		reg = bus_space_read_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_MAP + msinum * 8);
-		reg &= ~FIRE_MSI_MAP_EQNUM;
+		    FIRE_MSI_MAP(msinum));
+		CLR(reg, FIRE_MSI_MAP_EQNUM);
+		SET(reg, CPU_INFO_UNIT(ci)); /* There's an eq per cpu. */
 		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_MAP + msinum * 8, reg);
+		    FIRE_MSI_MAP(msinum), reg);
 
 		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_CLEAR + msinum * 8, FIRE_MSI_CLEAR_EQWR_N);
+		    FIRE_MSI_CLEAR(msinum), FIRE_MSI_CLEAR_EQWR_N);
 
 		reg = bus_space_read_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_MAP + msinum * 8);
-		reg |= FIRE_MSI_MAP_V;
+		    FIRE_MSI_MAP(msinum));
+		SET(reg, FIRE_MSI_MAP_V);
 		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_MAP + msinum * 8, reg);
+		    FIRE_MSI_MAP(msinum), reg);
 
 		return (ih);
 	}
@@ -717,11 +851,12 @@ pyro_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 		ino |= INTVEC(ihandle);
 	}
 
-	ih = bus_intr_allocate(t0, handler, arg, ino, level, intrmapptr,
+	ih = bus_intr_allocate(t0, handler, arg, ino, level, NULL,
 	    intrclrptr, what);
 	if (ih == NULL)
 		return (NULL);
 
+	ih->ih_cpu = ci;
 	if (flags & BUS_INTR_ESTABLISH_MPSAFE)
 		ih->ih_mpsafe = 1;
 
@@ -730,18 +865,22 @@ pyro_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 	if (intrmapptr != NULL) {
 		u_int64_t intrmap;
 
+		ci = ih->ih_cpu;
+
 		intrmap = *intrmapptr;
 		intrmap &= ~FIRE_INTRMAP_INT_CNTRL_NUM_MASK;
 		intrmap |= FIRE_INTRMAP_INT_CNTRL_NUM0;
 		if (sc->sc_oberon) {
 			intrmap &= ~OBERON_INTRMAP_T_DESTID_MASK;
-			intrmap |= CPU_JUPITERID <<
+			intrmap |= ci->ci_upaid <<
 			    OBERON_INTRMAP_T_DESTID_SHIFT;
 		} else {
 			intrmap &= ~FIRE_INTRMAP_T_JPID_MASK;
-			intrmap |= CPU_UPAID << FIRE_INTRMAP_T_JPID_SHIFT;
+			intrmap |= ci->ci_upaid <<
+			    FIRE_INTRMAP_T_JPID_SHIFT;
 		}
 		intrmap |= INTMAP_V;
+		membar_producer();
 		*intrmapptr = intrmap;
 		intrmap = *intrmapptr;
 		ih->ih_number |= intrmap & INTMAP_INR;
@@ -758,39 +897,39 @@ pyro_msi_ack(struct intrhand *ih)
 int
 pyro_msi_eq_intr(void *arg)
 {
-	struct pyro_pbm *pbm = arg;
+	struct pyro_eq *eq = arg;
+	struct pyro_pbm *pbm = eq->eq_pbm;
 	struct pyro_softc *sc = pbm->pp_sc;
-	struct msi_eq *meq = pbm->pp_meq;
 	struct pyro_msi_msg *msg;
 	uint64_t head, tail;
 	struct intrhand *ih;
 	int msinum;
 
-	head = bus_space_read_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_HEAD);
-	tail = bus_space_read_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_TAIL);
+	head = bus_space_read_8(sc->sc_bust, sc->sc_csrh, eq->eq_head);
+	tail = bus_space_read_8(sc->sc_bust, sc->sc_csrh, eq->eq_tail);
 
 	if (head == tail)
 		return (0);
 
-	while (head != tail) {
-		msg = (struct pyro_msi_msg *)meq->meq_va;
-
-		if (msg[head].mm_type == 0)
+	do {
+		msg = &eq->eq_ring[head];
+		if (msg->mm_type == 0)
 			break;
-		msg[head].mm_type = 0;
 
-		msinum = msg[head].mm_data;
+ 		msg->mm_type = 0;
+
+		msinum = msg->mm_data;
 		ih = pbm->pp_msi[msinum];
 		bus_space_write_8(sc->sc_bust, sc->sc_csrh,
-		    FIRE_MSI_CLEAR + msinum * 8, FIRE_MSI_CLEAR_EQWR_N);
+		    FIRE_MSI_CLEAR(msinum), FIRE_MSI_CLEAR_EQWR_N);
 
 		send_softint(-1, ih->ih_pil, ih);
 
 		head += 1;
-		head &= (meq->meq_nentries - 1);
-	}
+		head &= eq->eq_mask;
+	} while (head != tail);
 
-	bus_space_write_8(sc->sc_bust, sc->sc_csrh, FIRE_EQ_HEAD, head);
+	bus_space_write_8(sc->sc_bust, sc->sc_csrh, eq->eq_head, head);
 
 	return (1);
 }
diff --git a/sys/arch/sparc64/dev/pyrovar.h b/sys/arch/sparc64/dev/pyrovar.h
index d68dea65476..edd5c0fe8db 100644
--- a/sys/arch/sparc64/dev/pyrovar.h
+++ b/sys/arch/sparc64/dev/pyrovar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pyrovar.h,v 1.4 2011/07/06 05:48:57 kettenis Exp $	*/
+/*	$OpenBSD: pyrovar.h,v 1.5 2020/06/23 01:21:29 jmatthew Exp $	*/
 
 /*
  * Copyright (c) 2007 Mark Kettenis
@@ -44,6 +44,19 @@ struct pyro_range {
 	u_int32_t	size_lo;
 };
 
+struct pyro_eq {
+	char			eq_name[16];
+	struct pyro_pbm		*eq_pbm;
+	void			*eq_ih;
+	bus_size_t		eq_head;
+	bus_size_t		eq_tail;
+	struct pyro_msi_msg	*eq_ring;
+	uint64_t		eq_mask;
+
+	unsigned int		eq_id;
+	unsigned int		eq_intr;
+};
+
 struct pyro_pbm {
 	struct pyro_softc *pp_sc;
 
@@ -65,6 +78,9 @@ struct pyro_pbm {
 	int pp_msinum;
 	struct intrhand **pp_msi;
 
+	unsigned int		pp_neq;
+	struct pyro_eq		*pp_eq;
+
 	int pp_flags;
 };
 
@@ -77,5 +93,9 @@ struct pyro_softc {
 	bus_addr_t sc_csr, sc_xbc;
 	bus_space_handle_t sc_csrh, sc_xbch;
 
+#if 0
+	struct pyro_pbm *sc_pbm;
+#endif
+
 	int sc_oberon;
 };
diff --git a/sys/arch/sparc64/dev/vpci.c b/sys/arch/sparc64/dev/vpci.c
index 069622c41bc..c011129f436 100644
--- a/sys/arch/sparc64/dev/vpci.c
+++ b/sys/arch/sparc64/dev/vpci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vpci.c,v 1.29 2019/07/26 11:33:05 kettenis Exp $	*/
+/*	$OpenBSD: vpci.c,v 1.30 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*
  * Copyright (c) 2008 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -35,6 +35,8 @@
 #include <sparc64/dev/viommuvar.h>
 #include <sparc64/dev/msivar.h>
 
+#define VPCI_DEFAULT_MSI_EQS	36
+
 extern struct sparc_pci_chipset _sparc_pci_chipset;
 
 struct vpci_msi_msg {
@@ -61,6 +63,16 @@ struct vpci_range {
 	u_int32_t	size_lo;
 };
 
+struct vpci_eq {
+	char 			eq_name[16];
+	int 			eq_id;
+	void 			*eq_ih;
+	struct msi_eq		*eq_meq;
+	struct vpci_pbm 	*eq_pbm;
+	uint8_t			*eq_ring;
+	uint64_t		eq_mask;
+};
+
 struct vpci_pbm {
 	struct vpci_softc *vp_sc;
 	uint64_t vp_devhandle;
@@ -74,12 +86,14 @@ struct vpci_pbm {
 	bus_dma_tag_t		vp_dmat;
 	struct iommu_state	vp_is;
 
-	struct msi_eq *vp_meq;
 	bus_addr_t vp_msiaddr;
 	int vp_msibase;
 	int vp_msinum;
 	struct intrhand **vp_msi;
 
+	unsigned int vp_neq;
+	struct vpci_eq *vp_eq;
+
 	int vp_flags;
 };
 
@@ -118,6 +132,8 @@ paddr_t vpci_bus_mmap(bus_space_tag_t, bus_space_tag_t, bus_addr_t, off_t,
     int, int);
 void *vpci_intr_establish(bus_space_tag_t, bus_space_tag_t, int, int, int,
     int (*)(void *), void *, const char *);
+void *vpci_intr_establish_cpu(bus_space_tag_t, bus_space_tag_t, int, int,
+    int, struct cpu_info *, int (*)(void *), void *, const char *);
 void vpci_intr_ack(struct intrhand *);
 void vpci_msi_ack(struct intrhand *);
 
@@ -249,8 +265,14 @@ vpci_init_msi(struct vpci_softc *sc, struct vpci_pbm *pbm)
 	u_int32_t msi_eq_devino[3] = { 0, 36, 24 };
 	u_int32_t msi_range[2];
 	uint64_t sysino;
-	int msis, msi_eq_size;
+	int msis, msi_eq_size, num_eq, unit;
+	struct vpci_eq *eq;
 	int err;
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+
+	/* One eq per cpu, limited by the number of eqs. */
+	num_eq = min(ncpus, getpropint(sc->sc_node, "#msi-eqs", 36));
 
 	if (OF_getprop(sc->sc_node, "msi-address-ranges",
 	    msi_addr_range, sizeof(msi_addr_range)) <= 0)
@@ -265,51 +287,85 @@ vpci_init_msi(struct vpci_softc *sc, struct vpci_pbm *pbm)
 		return;
 
 	msi_eq_size = getpropint(sc->sc_node, "msi-eq-size", 256);
-	pbm->vp_meq = msi_eq_alloc(sc->sc_dmat, msi_eq_size);
-	if (pbm->vp_meq == NULL)
-		goto free_table;
 
 	if (OF_getprop(sc->sc_node, "msi-ranges",
 	    msi_range, sizeof(msi_range)) <= 0)
 		goto free_table;
 	pbm->vp_msibase = msi_range[0];
 
-	err = hv_pci_msiq_conf(pbm->vp_devhandle, 0,
-	    pbm->vp_meq->meq_map->dm_segs[0].ds_addr,
-	    pbm->vp_meq->meq_nentries);
-	if (err != H_EOK)
-		goto free_queue;
+	pbm->vp_neq = num_eq;
+	pbm->vp_eq = mallocarray(num_eq, sizeof(*eq), M_DEVBUF,
+	    M_WAITOK | M_ZERO);
 
-	OF_getprop(sc->sc_node, "msi-eq-to-devino",
-	    msi_eq_devino, sizeof(msi_eq_devino));
-	err = sun4v_intr_devino_to_sysino(pbm->vp_devhandle,
-	    msi_eq_devino[2], &sysino);
-	if (err != H_EOK)
-		goto disable_queue;
+	CPU_INFO_FOREACH(cii, ci) {
+		unit = CPU_INFO_UNIT(ci);
+		eq = &pbm->vp_eq[unit];
 
-	if (vpci_intr_establish(pbm->vp_memt, pbm->vp_memt, sysino,
-	    IPL_HIGH, 0, vpci_msi_eq_intr, pbm, sc->sc_dv.dv_xname) == NULL)
-		goto disable_queue;
+		if (unit >= num_eq)
+			continue;
 
-	err = hv_pci_msiq_setvalid(pbm->vp_devhandle, 0, PCI_MSIQ_VALID);
-	if (err != H_EOK) {
-		printf("%s: pci_msiq_setvalid: err %d\n", __func__, err);
-		goto disable_queue;
-	}
+		eq->eq_id = unit;
+		eq->eq_pbm = pbm;
+		snprintf(eq->eq_name, sizeof(eq->eq_name), "%s:%d",
+		    sc->sc_dv.dv_xname, unit);
+
+		eq->eq_meq = msi_eq_alloc(sc->sc_dmat, msi_eq_size, 1);
+		if (eq->eq_meq == NULL)
+			goto free_queues;
+
+		err = hv_pci_msiq_conf(pbm->vp_devhandle, unit,
+		    eq->eq_meq->meq_map->dm_segs[0].ds_addr,
+		    eq->eq_meq->meq_nentries);
+		if (err != H_EOK)
+			goto free_queues;
+
+		eq->eq_mask = (eq->eq_meq->meq_nentries *
+		    sizeof(struct vpci_msi_msg)) - 1;
+
+		OF_getprop(sc->sc_node, "msi-eq-to-devino",
+		    msi_eq_devino, sizeof(msi_eq_devino));
+		err = sun4v_intr_devino_to_sysino(pbm->vp_devhandle,
+		    msi_eq_devino[2] + unit, &sysino);
+		if (err != H_EOK)
+			goto free_queues;
 
-	err = hv_pci_msiq_setstate(pbm->vp_devhandle, 0, PCI_MSIQSTATE_IDLE);
-	if (err != H_EOK) {
-		printf("%s: pci_msiq_setstate: err %d\n", __func__, err);
-		goto disable_queue;
+		eq->eq_ih = vpci_intr_establish_cpu(pbm->vp_memt, pbm->vp_memt,
+		    sysino, IPL_HIGH, BUS_INTR_ESTABLISH_MPSAFE, ci,
+		    vpci_msi_eq_intr, eq, eq->eq_name);
+		if (eq->eq_ih == NULL)
+			goto free_queues;
+
+		err = hv_pci_msiq_setvalid(pbm->vp_devhandle, unit,
+		    PCI_MSIQ_VALID);
+		if (err != H_EOK) {
+			printf("%s: pci_msiq_setvalid(%d): err %d\n", __func__,
+			    unit, err);
+			goto free_queues;
+		}
+
+		err = hv_pci_msiq_setstate(pbm->vp_devhandle, unit,
+		    PCI_MSIQSTATE_IDLE);
+		if (err != H_EOK) {
+			printf("%s: pci_msiq_setstate(%d): err %d\n", __func__,
+			    unit, err);
+			goto free_queues;
+		}
 	}
 
 	pbm->vp_flags |= PCI_FLAGS_MSI_ENABLED;
 	return;
 
-disable_queue:
-	hv_pci_msiq_conf(pbm->vp_devhandle, 0, 0, 0);
-free_queue:
-	msi_eq_free(sc->sc_dmat, pbm->vp_meq);
+free_queues:
+	CPU_INFO_FOREACH(cii, ci) {
+		unit = CPU_INFO_UNIT(ci);
+		eq = &pbm->vp_eq[unit];
+
+		if (eq->eq_meq != NULL)
+			msi_eq_free(sc->sc_dmat, eq->eq_meq);
+
+		hv_pci_msiq_conf(pbm->vp_devhandle, unit, 0, 0);
+	}
+	free(pbm->vp_eq, M_DEVBUF, num_eq * sizeof(*eq));
 free_table:
 	free(pbm->vp_msi, M_DEVBUF, 0);
 }
@@ -418,6 +474,7 @@ vpci_alloc_bus_tag(struct vpci_pbm *pbm, const char *name, int ss,
 	bt->sparc_bus_map = vpci_bus_map;
 	bt->sparc_bus_mmap = vpci_bus_mmap;
 	bt->sparc_intr_establish = vpci_intr_establish;
+	bt->sparc_intr_establish_cpu = vpci_intr_establish_cpu;
 	return (bt);
 }
 
@@ -547,6 +604,15 @@ void *
 vpci_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
     int level, int flags, int (*handler)(void *), void *arg, const char *what)
 {
+	return (vpci_intr_establish_cpu(t, t0, ihandle, level, flags, NULL,
+	    handler, arg, what));
+}
+
+void *
+vpci_intr_establish_cpu(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
+    int level, int flags, struct cpu_info *cpu, int (*handler)(void *),
+    void *arg, const char *what)
+{
 	struct vpci_pbm *pbm = t->cookie;
 	uint64_t devhandle = pbm->vp_devhandle;
 	uint64_t sysino = INTVEC(ihandle);
@@ -566,6 +632,7 @@ vpci_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 		pcitag_t tag = PCI_INTR_TAG(ihandle);
 		int msinum = pbm->vp_msinum++;
 		int msi = pbm->vp_msibase + msinum;
+		int eq = 0;
 
 		evcount_attach(&ih->ih_count, ih->ih_name, NULL);
 
@@ -584,7 +651,15 @@ vpci_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 			break;
 		}
 
-		err = hv_pci_msi_setmsiq(devhandle, msi, 0, 0);
+		if (cpu != NULL) {
+			/*
+			 * For now, if we have fewer eqs than cpus, map
+			 * interrupts for the eq-less cpus onto other cpus.
+			 */
+			eq = CPU_INFO_UNIT(cpu) % pbm->vp_neq;
+		}
+
+		err = hv_pci_msi_setmsiq(devhandle, msi, eq, 0);
 		if (err != H_EOK) {
 			printf("%s: pci_msi_setmsiq: err %d\n", __func__, err);
 			return (NULL);
@@ -609,6 +684,7 @@ vpci_intr_establish(bus_space_tag_t t, bus_space_tag_t t0, int ihandle,
 	if (err != H_EOK)
 		return (NULL);
 
+	ih->ih_cpu = cpu;
 	intr_establish(ih->ih_pil, ih);
 	ih->ih_ack = vpci_intr_ack;
 
@@ -647,8 +723,8 @@ vpci_msi_ack(struct intrhand *ih)
 int
 vpci_msi_eq_intr(void *arg)
 {
-	struct vpci_pbm *pbm = arg;
-	struct msi_eq *meq = pbm->vp_meq;
+	struct vpci_eq *eq = arg;
+	struct vpci_pbm *pbm = eq->eq_pbm;
 	struct vpci_msi_msg *msg;
 	uint64_t devhandle = pbm->vp_devhandle;
 	uint64_t head, tail;
@@ -656,19 +732,21 @@ vpci_msi_eq_intr(void *arg)
 	int msinum, msi;
 	int err;
 
-	err = hv_pci_msiq_gethead(devhandle, 0, &head);
+	err = hv_pci_msiq_gethead(devhandle, eq->eq_id, &head);
 	if (err != H_EOK)
-		printf("%s: pci_msiq_gethead: %d\n", __func__, err);
+		printf("%s: pci_msiq_gethead(%d): %d\n", __func__, eq->eq_id,
+		    err);
 
-	err = hv_pci_msiq_gettail(devhandle, 0, &tail);
+	err = hv_pci_msiq_gettail(devhandle, eq->eq_id, &tail);
 	if (err != H_EOK)
-		printf("%s: pci_msiq_gettail: %d\n", __func__, err);
+		printf("%s: pci_msiq_gettail(%d): %d\n", __func__, eq->eq_id,
+		    err);
 
 	if (head == tail)
 		return (0);
 
 	while (head != tail) {
-		msg = (struct vpci_msi_msg *)(meq->meq_va + head);
+		msg = (struct vpci_msi_msg *)(eq->eq_meq->meq_va + head);
 
 		if (msg->mm_type == 0)
 			break;
@@ -684,12 +762,13 @@ vpci_msi_eq_intr(void *arg)
 		send_softint(-1, ih->ih_pil, ih);
 
 		head += sizeof(struct vpci_msi_msg);
-		head &= ((meq->meq_nentries * sizeof(struct vpci_msi_msg)) - 1);
+		head &= eq->eq_mask;
 	}
 
-	err = hv_pci_msiq_sethead(devhandle, 0, head);
+	err = hv_pci_msiq_sethead(devhandle, eq->eq_id, head);
 	if (err != H_EOK)
-		printf("%s: pci_msiq_sethead: %d\n", __func__, err);
+		printf("%s: pci_msiq_sethead(%d): %d\n", __func__, eq->eq_id,
+		    err);
 
 	return (1);
 }
diff --git a/sys/arch/sparc64/include/bus.h b/sys/arch/sparc64/include/bus.h
index d1288962105..cd2710feb82 100644
--- a/sys/arch/sparc64/include/bus.h
+++ b/sys/arch/sparc64/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.33 2017/05/25 03:19:39 dlg Exp $	*/
+/*	$OpenBSD: bus.h,v 1.34 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*	$NetBSD: bus.h,v 1.31 2001/09/21 15:30:41 wiz Exp $	*/
 
 /*-
@@ -206,6 +206,12 @@ struct sparc_bus_space_tag {
 		int, int, int,
 		int (*)(void *), void *,
 		const char *);
+	void	*(*sparc_intr_establish_cpu)(bus_space_tag_t,
+		bus_space_tag_t,
+		int, int, int,
+		struct cpu_info *,
+		int (*)(void *), void *,
+		const char *);
 
 	bus_addr_t (*sparc_bus_addr)(bus_space_tag_t,
 		bus_space_tag_t, bus_space_handle_t);
@@ -270,6 +276,16 @@ void	       *bus_intr_establish(
 				int (*)(void *),	/*handler*/
 				void *,			/*handler arg*/
 				const char *);		/*what*/
+void	       *bus_intr_establish_cpu(
+				bus_space_tag_t,
+				int,			/*bus-specific intr*/
+				int,			/*device class level,
+							  see machine/intr.h*/
+				int,			/*flags*/
+				struct cpu_info *,	/*cpu*/
+				int (*)(void *),	/*handler*/
+				void *,			/*handler arg*/
+				const char *);		/*what*/
 void	       *bus_intr_allocate(
 				bus_space_tag_t,
 				int (*)(void *),	/*handler*/
diff --git a/sys/arch/sparc64/include/pci_machdep.h b/sys/arch/sparc64/include/pci_machdep.h
index ca3576ff2ee..649c116aa13 100644
--- a/sys/arch/sparc64/include/pci_machdep.h
+++ b/sys/arch/sparc64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.37 2020/06/17 01:15:32 dlg Exp $	*/
+/*	$OpenBSD: pci_machdep.h,v 1.38 2020/06/23 01:21:29 jmatthew Exp $	*/
 /* $NetBSD: pci_machdep.h,v 1.7 2001/07/20 00:07:14 eeh Exp $ */
 
 /*
@@ -107,6 +107,9 @@ int		pci_intr_line(pci_chipset_tag_t, pci_intr_handle_t);
 const char	*pci_intr_string(pci_chipset_tag_t, pci_intr_handle_t);
 void		*pci_intr_establish(pci_chipset_tag_t, pci_intr_handle_t,
 				 int, int (*)(void *), void *, const char *);
+void		*pci_intr_establish_cpu(pci_chipset_tag_t, pci_intr_handle_t,
+		    int, struct cpu_info *,
+		    int (*)(void *), void *, const char *);
 void		pci_intr_disestablish(pci_chipset_tag_t, void *);
 
 void		pci_msi_enable(pci_chipset_tag_t, pcitag_t, bus_addr_t, int);
diff --git a/sys/arch/sparc64/sparc64/intr.c b/sys/arch/sparc64/sparc64/intr.c
index 74c94760af0..c5a29a26b34 100644
--- a/sys/arch/sparc64/sparc64/intr.c
+++ b/sys/arch/sparc64/sparc64/intr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.c,v 1.59 2018/03/21 14:25:14 kettenis Exp $	*/
+/*	$OpenBSD: intr.c,v 1.60 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*	$NetBSD: intr.c,v 1.39 2001/07/19 23:38:11 eeh Exp $ */
 
 /*
@@ -202,7 +202,12 @@ intr_establish(int level, struct intrhand *ih)
 	ih->ih_pil = level; /* XXXX caller should have done this before */
 	ih->ih_pending = 0; /* XXXX caller should have done this before */
 	ih->ih_next = NULL;
-	ih->ih_cpu = cpus;
+	if (ih->ih_cpu == NULL)
+		ih->ih_cpu = curcpu();
+	else if (!ih->ih_mpsafe) {
+		panic("non-mpsafe interrupt \"%s\" "
+		    "established on a specific cpu", ih->ih_name);
+	}
 	if (ih->ih_clr)
 		ih->ih_ack = intr_ack;
 	else
@@ -288,7 +293,7 @@ intr_establish(int level, struct intrhand *ih)
 		*ih->ih_clr = INTCLR_IDLE;
 
 	if (ih->ih_map) {
-		id = CPU_UPAID;
+		id = ih->ih_cpu->ci_upaid;
 		m = *ih->ih_map;
 		if (INTTID(m) != id) {
 #ifdef DEBUG
diff --git a/sys/arch/sparc64/sparc64/machdep.c b/sys/arch/sparc64/sparc64/machdep.c
index eabbac0cf74..aaf05e1e6dd 100644
--- a/sys/arch/sparc64/sparc64/machdep.c
+++ b/sys/arch/sparc64/sparc64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.197 2020/06/05 14:25:05 naddy Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.198 2020/06/23 01:21:29 jmatthew Exp $	*/
 /*	$NetBSD: machdep.c,v 1.108 2001/07/24 19:30:14 eeh Exp $ */
 
 /*-
@@ -1822,21 +1822,22 @@ sparc_bus_free(bus_space_tag_t t, bus_space_tag_t t0, bus_space_handle_t h,
 }
 
 static const struct sparc_bus_space_tag _mainbus_space_tag = {
-	NULL,				/* cookie */
-	NULL,				/* parent bus tag */
-	UPA_BUS_SPACE,			/* type */
-	ASI_PRIMARY,
-	ASI_PRIMARY,
-	"mainbus",
-	sparc_bus_alloc,
-	sparc_bus_free,
-	sparc_bus_map,			/* bus_space_map */
-	sparc_bus_protect,		/* bus_space_protect */
-	sparc_bus_unmap,		/* bus_space_unmap */
-	sparc_bus_subregion,		/* bus_space_subregion */
-	sparc_bus_mmap,			/* bus_space_mmap */
-	sparc_mainbus_intr_establish,	/* bus_intr_establish */
-	sparc_bus_addr			/* bus_space_addr */
+	.cookie =			NULL,
+	.parent =			NULL,
+	.default_type =			UPA_BUS_SPACE,
+	.asi =				ASI_PRIMARY,
+	.sasi =				ASI_PRIMARY,
+	.name =				"mainbus",
+	.sparc_bus_alloc =		sparc_bus_alloc,
+	.sparc_bus_free =		sparc_bus_free,
+	.sparc_bus_map =		sparc_bus_map,
+	.sparc_bus_protect =		sparc_bus_protect,
+	.sparc_bus_unmap =		sparc_bus_unmap,
+	.sparc_bus_subregion =		sparc_bus_subregion,
+	.sparc_bus_mmap =		sparc_bus_mmap,	
+	.sparc_intr_establish =		sparc_mainbus_intr_establish,
+	/*.sparc_intr_establish_cpu*/
+	.sparc_bus_addr =		sparc_bus_addr
 };
 const bus_space_tag_t mainbus_space_tag = &_mainbus_space_tag;
 
@@ -2000,6 +2001,23 @@ bus_intr_establish(bus_space_tag_t t, int p, int l, int f, int (*h)(void *),
 	return (ret);
 }
 
+void *
+bus_intr_establish_cpu(bus_space_tag_t t, int p, int l, int f,
+    struct cpu_info *ci, int (*h)(void *), void *a, const char *w)
+{
+	const bus_space_tag_t t0 = t;
+	void *ret;
+
+	if (t->sparc_intr_establish_cpu == NULL)
+		return (bus_intr_establish(t, p, l, f, h, a, w));
+
+	_BS_PRECALL(t, sparc_intr_establish_cpu);
+	ret = _BS_CALL(t, sparc_intr_establish_cpu)(t, t0, p, l, f, ci,
+	    h, a, w);
+	_BS_POSTCALL;
+	return (ret);
+}
+
 /* XXXX Things get complicated if we use unmapped register accesses. */
 void *
 bus_space_vaddr(bus_space_tag_t t, bus_space_handle_t h)