Load MSI pages through bus_dma(9). Our interrupt controllers for MSIs

typically pass the physical address, however retrieved, to our PCIe
controller code.  This physical address can in practise be directly
given to the PCIe, but it is not a given that the CPU and the PCIe
controller are able to use the same physical addresses.

This is even more obvious with an smmu(4) inbetween, which can change
the world view by introducing I/O virtual addresses.  Hence for this
it is indeed necessary to map those pages, which thanks to integration
with bus_dma(9) works easily.

For this we remember the PCI devices' DMA tag in the interrupt handle
during the MSI map, so that we can use the smmu(4)-hooked DMA tag to
load the physical address.

While some systems might prefer to implement "trapping" pages for MSIs,
to make sure devices cannot trigger other devices' interrupts, we only
make sure the whole page is mapped.

Having the IOMMU create a mapping for each MSI is a bit wasteful, but
for now it's the simplest way to implement it.

Discussed with and ok kettenis@

1 files changed, 53 insertions, 4 deletions

diff --git a/sys/dev/fdt/dwpcie.c b/sys/dev/fdt/dwpcie.c
index 9d87788a8f5..21b5b6bf9de 100644
--- a/sys/dev/fdt/dwpcie.c
+++ b/sys/dev/fdt/dwpcie.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dwpcie.c,v 1.27 2021/03/01 21:03:24 patrick Exp $	*/
+/*	$OpenBSD: dwpcie.c,v 1.28 2021/03/22 20:30:21 patrick Exp $	*/
 /*
  * Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -225,6 +225,12 @@ struct dwpcie_softc {
 	void			*sc_ih;
 };
 
+struct dwpcie_intr_handle {
+	struct arm_intr_handle	 pih_ih;
+	bus_dma_tag_t		 pih_dmat;
+	bus_dmamap_t		 pih_map;
+};
+
 int dwpcie_match(struct device *, void *, void *);
 void dwpcie_attach(struct device *, struct device *, void *);
 
@@ -285,6 +291,10 @@ int	dwpcie_bs_iomap(bus_space_tag_t, bus_addr_t, bus_size_t, int,
 int	dwpcie_bs_memmap(bus_space_tag_t, bus_addr_t, bus_size_t, int,
 	    bus_space_handle_t *);
 
+struct interrupt_controller dwpcie_ic = {
+	.ic_barrier = intr_barrier
+};
+
 void
 dwpcie_attach(struct device *parent, struct device *self, void *aux)
 {
@@ -1195,6 +1205,8 @@ dwpcie_intr_establish(void *v, pci_intr_handle_t ih, int level,
     struct cpu_info *ci, int (*func)(void *), void *arg, char *name)
 {
 	struct dwpcie_softc *sc = v;
+	struct dwpcie_intr_handle *pih;
+	bus_dma_segment_t seg;
 	void *cookie;
 
 	KASSERT(ih.ih_type != PCI_NONE);
@@ -1209,8 +1221,31 @@ dwpcie_intr_establish(void *v, pci_intr_handle_t ih, int level,
 		if (cookie == NULL)
 			return NULL;
 
-		/* TODO: translate address to the PCI device's view */
+		pih = malloc(sizeof(*pih), M_DEVBUF, M_WAITOK);
+		pih->pih_ih.ih_ic = &dwpcie_ic;
+		pih->pih_ih.ih_ih = cookie;
+		pih->pih_dmat = ih.ih_dmat;
+
+		if (bus_dmamap_create(pih->pih_dmat, sizeof(uint32_t), 1,
+		    sizeof(uint32_t), 0, BUS_DMA_WAITOK, &pih->pih_map)) {
+			free(pih, M_DEVBUF, sizeof(*pih));
+			fdt_intr_disestablish(cookie);
+			return NULL;
+		}
+
+		memset(&seg, 0, sizeof(seg));
+		seg.ds_addr = addr;
+		seg.ds_len = sizeof(uint32_t);
 
+		if (bus_dmamap_load_raw(pih->pih_dmat, pih->pih_map,
+		    &seg, 1, sizeof(uint32_t), BUS_DMA_WAITOK)) {
+			bus_dmamap_destroy(pih->pih_dmat, pih->pih_map);
+			free(pih, M_DEVBUF, sizeof(*pih));
+			fdt_intr_disestablish(cookie);
+			return NULL;
+		}
+
+		addr = pih->pih_map->dm_segs[0].ds_addr;
 		if (ih.ih_type == PCI_MSIX) {
 			pci_msix_enable(ih.ih_pc, ih.ih_tag,
 			    &sc->sc_bus_memt, ih.ih_intrpin, addr, data);
@@ -1228,15 +1263,29 @@ dwpcie_intr_establish(void *v, pci_intr_handle_t ih, int level,
 
 		cookie = fdt_intr_establish_imap_cpu(sc->sc_node, reg,
 		    sizeof(reg), level, ci, func, arg, name);
+		if (cookie == NULL)
+			return NULL;
+
+		pih = malloc(sizeof(*pih), M_DEVBUF, M_WAITOK);
+		pih->pih_ih.ih_ic = &dwpcie_ic;
+		pih->pih_ih.ih_ih = cookie;
+		pih->pih_dmat = NULL;
 	}
 
-	return cookie;
+	return pih;
 }
 
 void
 dwpcie_intr_disestablish(void *v, void *cookie)
 {
-	panic("%s", __func__);
+	struct dwpcie_intr_handle *pih = cookie;
+
+	fdt_intr_disestablish(pih->pih_ih.ih_ih);
+	if (pih->pih_dmat) {
+		bus_dmamap_unload(pih->pih_dmat, pih->pih_map);
+		bus_dmamap_destroy(pih->pih_dmat, pih->pih_map);
+	}
+	free(pih, M_DEVBUF, sizeof(*pih));
 }
 
 int