Allow to grant memory access to domains other than dom0.

Extend xen_grant_table_enter to take an additional "domain" argument
and extract it from the upper part of the bus_dmamap_load flags (sigh.)
to be able to punch it into the grant table entry.

Issue reported by Marco Peereboom who found that we wouldn't run under
QubesOS that "chains" VMs.  He also did the hard work getting the debug
data out of the aforementioned system.

2 files changed, 26 insertions, 18 deletions

diff --git a/sys/dev/pv/if_xnf.c b/sys/dev/pv/if_xnf.c
index 3bd471febc4..2f9a02fbe36 100644
--- a/sys/dev/pv/if_xnf.c
+++ b/sys/dev/pv/if_xnf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_xnf.c,v 1.20 2016/04/19 13:55:19 mikeb Exp $	*/
+/*	$OpenBSD: if_xnf.c,v 1.21 2016/04/19 14:19:44 mikeb Exp $	*/
 
 /*
  * Copyright (c) 2015, 2016 Mike Belopuhov
@@ -531,7 +531,7 @@ xnf_encap(struct xnf_softc *sc, struct mbuf *m_head, uint32_t *prod)
 	struct mbuf *m;
 	bus_dmamap_t dmap;
 	uint32_t oprod = *prod;
-	int i, id, n = 0;
+	int i, id, flags, n = 0;
 
 	if ((XNF_TX_DESC - (*prod - sc->sc_tx_cons)) < sc->sc_tx_frags)
 		return (ENOENT);
@@ -552,8 +552,9 @@ xnf_encap(struct xnf_softc *sc, struct mbuf *m_head, uint32_t *prod)
 			    ifp->if_xname, txr->txr_cons, sc->sc_tx_cons,
 			    txr->txr_prod, *prod, n, dmap->dm_nsegs - 1);
 
+		flags = (sc->sc_domid << 16) | BUS_DMA_WRITE | BUS_DMA_WAITOK;
 		if (bus_dmamap_load(sc->sc_dmat, dmap, m->m_data, m->m_len,
-		    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT))
+		    NULL, flags))
 			goto unroll;
 
 		if (m == m_head) {
@@ -793,7 +794,7 @@ xnf_rx_ring_fill(void *arg)
 	struct mbuf *m;
 	uint32_t cons, prod;
 	static int timer = 0;
-	int i, n;
+	int i, flags, n;
 
 	cons = rxr->rxr_cons;
 	prod = rxr->rxr_prod;
@@ -818,8 +819,8 @@ xnf_rx_ring_fill(void *arg)
 			break;
 		m->m_len = m->m_pkthdr.len = XNF_MCLEN;
 		dmap = sc->sc_rx_dmap[i];
-		if (bus_dmamap_load_mbuf(sc->sc_dmat, dmap, m, BUS_DMA_READ |
-		    BUS_DMA_NOWAIT)) {
+		flags = (sc->sc_domid << 16) | BUS_DMA_READ |BUS_DMA_NOWAIT;
+		if (bus_dmamap_load_mbuf(sc->sc_dmat, dmap, m, flags)) {
 			m_freem(m);
 			break;
 		}
@@ -841,7 +842,7 @@ xnf_rx_ring_fill(void *arg)
 int
 xnf_rx_ring_create(struct xnf_softc *sc)
 {
-	int i, rsegs;
+	int i, flags, rsegs;
 
 	/* Allocate a page of memory for the ring */
 	if (bus_dmamem_alloc(sc->sc_dmat, PAGE_SIZE, PAGE_SIZE, 0,
@@ -865,8 +866,9 @@ xnf_rx_ring_create(struct xnf_softc *sc)
 		goto errout;
 	}
 	/* Load the ring into the ring map to extract the PA */
+	flags = (sc->sc_domid << 16) | BUS_DMA_WAITOK;
 	if (bus_dmamap_load(sc->sc_dmat, sc->sc_rx_rmap, sc->sc_rx_ring,
-	    PAGE_SIZE, NULL, BUS_DMA_WAITOK)) {
+	    PAGE_SIZE, NULL, flags)) {
 		printf("%s: failed to load the rx ring map\n",
 		    sc->sc_dev.dv_xname);
 		goto errout;
@@ -947,7 +949,7 @@ xnf_rx_ring_destroy(struct xnf_softc *sc)
 int
 xnf_tx_ring_create(struct xnf_softc *sc)
 {
-	int i, rsegs;
+	int i, flags, rsegs;
 
 	sc->sc_tx_frags = sc->sc_caps & XNF_CAP_SG ? XNF_TX_FRAG : 1;
 
@@ -973,8 +975,9 @@ xnf_tx_ring_create(struct xnf_softc *sc)
 		goto errout;
 	}
 	/* Load the ring into the ring map to extract the PA */
+	flags = (sc->sc_domid << 16) | BUS_DMA_WAITOK;
 	if (bus_dmamap_load(sc->sc_dmat, sc->sc_tx_rmap, sc->sc_tx_ring,
-	    PAGE_SIZE, NULL, BUS_DMA_WAITOK)) {
+	    PAGE_SIZE, NULL, flags)) {
 		printf("%s: failed to load the tx ring map\n",
 		    sc->sc_dev.dv_xname);
 		goto errout;
diff --git a/sys/dev/pv/xen.c b/sys/dev/pv/xen.c
index 6f57daa490a..117409a5163 100644
--- a/sys/dev/pv/xen.c
+++ b/sys/dev/pv/xen.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xen.c,v 1.53 2016/04/19 13:55:19 mikeb Exp $	*/
+/*	$OpenBSD: xen.c,v 1.54 2016/04/19 14:19:44 mikeb Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Belopuhov
@@ -66,7 +66,8 @@ struct xen_gntent *
 	xen_grant_table_grow(struct xen_softc *);
 int	xen_grant_table_alloc(struct xen_softc *, grant_ref_t *);
 void	xen_grant_table_free(struct xen_softc *, grant_ref_t);
-void	xen_grant_table_enter(struct xen_softc *, grant_ref_t, paddr_t, int);
+void	xen_grant_table_enter(struct xen_softc *, grant_ref_t, paddr_t,
+	    int, int);
 void	xen_grant_table_remove(struct xen_softc *, grant_ref_t);
 void	xen_disable_emulated_devices(struct xen_softc *);
 
@@ -1039,7 +1040,7 @@ xen_grant_table_free(struct xen_softc *sc, grant_ref_t ref)
 
 void
 xen_grant_table_enter(struct xen_softc *sc, grant_ref_t ref, paddr_t pa,
-    int flags)
+    int domain, int flags)
 {
 	struct xen_gntent *ge;
 
@@ -1057,7 +1058,7 @@ xen_grant_table_enter(struct xen_softc *sc, grant_ref_t ref, paddr_t pa,
 #endif
 	ref -= ge->ge_start;
 	ge->ge_table[ref].frame = atop(pa);
-	ge->ge_table[ref].domid = 0;
+	ge->ge_table[ref].domid = domain;
 	virtio_membar_sync();
 	ge->ge_table[ref].flags = GTF_permit_access | flags;
 	virtio_membar_sync();
@@ -1160,14 +1161,16 @@ xen_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 {
 	struct xen_softc *sc = t->_cookie;
 	struct xen_gntmap *gm = map->_dm_cookie;
-	int i, error;
+	int i, domain, error;
 
+	domain = flags >> 16;
+	flags &= 0xffff;
 	error = _bus_dmamap_load(t, map, buf, buflen, p, flags);
 	if (error)
 		return (error);
 	for (i = 0; i < map->dm_nsegs; i++) {
 		xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr,
-		    flags & BUS_DMA_WRITE ? GTF_readonly : 0);
+		    domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0);
 		gm[i].gm_paddr = map->dm_segs[i].ds_addr;
 		map->dm_segs[i].ds_addr = gm[i].gm_ref;
 	}
@@ -1180,14 +1183,16 @@ xen_bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0,
 {
 	struct xen_softc *sc = t->_cookie;
 	struct xen_gntmap *gm = map->_dm_cookie;
-	int i, error;
+	int i, domain, error;
 
+	domain = flags >> 16;
+	flags &= 0xffff;
 	error = _bus_dmamap_load_mbuf(t, map, m0, flags);
 	if (error)
 		return (error);
 	for (i = 0; i < map->dm_nsegs; i++) {
 		xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr,
-		    flags & BUS_DMA_WRITE ? GTF_readonly : 0);
+		    domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0);
 		gm[i].gm_paddr = map->dm_segs[i].ds_addr;
 		map->dm_segs[i].ds_addr = gm[i].gm_ref;
 	}