Add ability to establish virtual interrupts via Xen event

channel ports.

During boot, Xen will use polling mode, but once the system
enables interrupts after cpu_configure(), xen_intr_enable
will be called from the mountrook hook to unmask event ports.

OK mlarkin, mpi, reyk

3 files changed, 405 insertions, 1 deletions

diff --git a/sys/dev/pv/xen.c b/sys/dev/pv/xen.c
index 3a4e75f7012..318d2d14d1d 100644
--- a/sys/dev/pv/xen.c
+++ b/sys/dev/pv/xen.c
@@ -40,9 +40,11 @@ int	xen_getversion(struct xen_softc *);
 int	xen_getfeatures(struct xen_softc *);
 int	xen_init_info_page(struct xen_softc *);
 int	xen_init_cbvec(struct xen_softc *);
+int	xen_init_interrupts(struct xen_softc *);
 
 int 	xen_match(struct device *, void *, void *);
 void	xen_attach(struct device *, struct device *, void *);
+void	xen_deferred(void *);
 void	xen_resume(struct device *);
 int	xen_activate(struct device *, int);
 
@@ -92,6 +94,25 @@ xen_attach(struct device *parent, struct device *self, void *aux)
 		return;
 
 	xen_init_cbvec(sc);
+
+	if (xen_init_interrupts(sc))
+		return;
+
+	mountroothook_establish(xen_deferred, sc);
+}
+
+void
+xen_deferred(void *arg)
+{
+	struct xen_softc *sc = arg;
+
+	if (!sc->sc_cbvec) {
+		DPRINTF("%s: callback vector hasn't been established\n",
+		    sc->sc_dev.dv_xname);
+		return;
+	}
+
+	xen_intr_enable();
 }
 
 void
@@ -428,8 +449,249 @@ xen_init_cbvec(struct xen_softc *sc)
 	return (0);
 }
 
+int
+xen_init_interrupts(struct xen_softc *sc)
+{
+	int i;
+
+	sc->sc_irq = LAPIC_XEN_VECTOR;
+	evcount_attach(&sc->sc_evcnt, sc->sc_dev.dv_xname, &sc->sc_irq);
+
+	/*
+	 * Clear all pending events and mask all interrupts
+	 */
+	for (i = 0; i < nitems(sc->sc_ipg->evtchn_pending); i++) {
+		sc->sc_ipg->evtchn_pending[i] = 0;
+		sc->sc_ipg->evtchn_mask[i] = ~0UL;
+		membar_producer();
+	}
+
+	SLIST_INIT(&sc->sc_intrs);
+
+	return (0);
+}
+
+static inline struct xen_intsrc *
+xen_lookup_intsrc(struct xen_softc *sc, evtchn_port_t port)
+{
+	struct xen_intsrc *xi;
+
+	SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry)
+		if (xi->xi_port == port)
+			break;
+	return (xi);
+}
+
+void
+xen_intr_ack(void)
+{
+	struct xen_softc *sc = xen_sc;
+	struct shared_info *s = sc->sc_ipg;
+	struct cpu_info *ci = curcpu();
+	struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)];
+
+	v->evtchn_upcall_pending = 0;
+}
+
 void
 xen_intr(void)
 {
-	/* stub */
+	struct xen_softc *sc = xen_sc;
+	struct xen_intsrc *xi;
+	struct shared_info *s = sc->sc_ipg;
+	struct cpu_info *ci = curcpu();
+	struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)];
+	ulong pending, selector;
+	int port, bit, row;
+
+	sc->sc_evcnt.ec_count++;
+
+	v->evtchn_upcall_pending = 0;
+	selector = atomic_swap_ulong(&v->evtchn_pending_sel, 0);
+
+	for (row = 0; selector > 0; selector >>= 1, row++) {
+		if ((selector & 1) == 0)
+			continue;
+		pending = sc->sc_ipg->evtchn_pending[row] &
+		    ~(sc->sc_ipg->evtchn_mask[row]);
+		for (bit = 0; pending > 0; pending >>= 1, bit++) {
+			if ((pending & 1) == 0)
+				continue;
+			sc->sc_ipg->evtchn_pending[row] &= ~(1 << bit);
+			membar_producer();
+			port = (row * LONG_BIT) + bit;
+			if ((xi = xen_lookup_intsrc(sc, port)) == NULL)
+				continue;
+			xi->xi_evcnt.ec_count++;
+
+			if (xi->xi_handler)
+				xi->xi_handler(xi->xi_arg);
+		}
+	}
+}
+
+void
+xen_intr_signal(xen_intr_handle_t xih)
+{
+	struct xen_softc *sc = xen_sc;
+	struct xen_intsrc *xi;
+	struct evtchn_send es;
+
+	if ((xi = xen_lookup_intsrc(sc, (evtchn_port_t)xih)) != NULL) {
+		es.port = xi->xi_port;
+		xen_hypercall(sc, event_channel_op, 2, EVTCHNOP_send, &es);
+	}
+}
+
+int
+xen_intr_establish(evtchn_port_t port, xen_intr_handle_t *xih,
+    void (*handler)(void *), void *arg, char *name)
+{
+	struct xen_softc *sc = xen_sc;
+	struct xen_intsrc *xi;
+	struct evtchn_alloc_unbound eau;
+	struct evtchn_unmask eu;
+#if notyet
+	struct evtchn_bind_vcpu ebv;
+#endif
+#ifdef XEN_DEBUG
+	struct evtchn_status es;
+#endif
+
+	if (port && xen_lookup_intsrc(sc, port)) {
+		printf("%s: interrupt handler has already been established "
+		    "for port %u\n", sc->sc_dev.dv_xname, port);
+		return (-1);
+	}
+
+	xi = malloc(sizeof(*xi), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (xi == NULL)
+		return (-1);
+
+	xi->xi_handler = handler;
+	xi->xi_arg = arg;
+	xi->xi_port = (evtchn_port_t)*xih;
+
+	if (port == 0) {
+		/* We're being asked to allocate a new event port */
+		memset(&eau, 0, sizeof(eau));
+		eau.dom = DOMID_SELF;
+		if (xen_hypercall(sc, event_channel_op, 2,
+		    EVTCHNOP_alloc_unbound, &eau) != 0) {
+			DPRINTF("%s: failed to allocate new event port\n",
+			    sc->sc_dev.dv_xname);
+			free(xi, M_DEVBUF, sizeof(*xi));
+			return (-1);
+		}
+		*xih = xi->xi_port = eau.port;
+	} else {
+		*xih = xi->xi_port = port;
+		/*
+		 * The Event Channel API didn't open this port, so it is not
+		 * responsible for closing it automatically on unbind.
+		 */
+		xi->xi_noclose = 1;
+	}
+
+#ifdef notyet
+	/* Bind interrupt to VCPU#0 */
+	memset(&ebv, 0, sizeof(ebv));
+	ebv.port = xi->xi_port;
+	ebv.vcpu = 0;
+	if (xen_hypercall(sc, event_channel_op, 2, EVTCHNOP_bind_vcpu, &ebv)) {
+		printf("%s: failed to bind interrupt on port %u to vcpu%d\n",
+		    sc->sc_dev.dv_xname, ebv.port, ebv.vcpu);
+	}
+#endif
+
+	evcount_attach(&xi->xi_evcnt, name, &sc->sc_irq);
+
+	SLIST_INSERT_HEAD(&sc->sc_intrs, xi, xi_entry);
+
+	if (!cold) {
+		eu.port = xi->xi_port;
+		if (xen_hypercall(sc, event_channel_op, 2, EVTCHNOP_unmask,
+		    &eu) || isset(sc->sc_ipg->evtchn_mask, port))
+			printf("%s: unmasking port %u failed\n",
+			    sc->sc_dev.dv_xname, port);
+	}
+
+#ifdef XEN_DEBUG
+	memset(&es, 0, sizeof(es));
+	es.dom = DOMID_SELF;
+	es.port = xi->xi_port;
+	if (xen_hypercall(sc, event_channel_op, 2, EVTCHNOP_status, &es)) {
+		printf("%s: failed to obtain status for port %d\n",
+		    sc->sc_dev.dv_xname, es.port);
+	}
+	printf("%s: port %u bound to vcpu%u",
+	    sc->sc_dev.dv_xname, es.port, es.vcpu);
+	if (es.status == EVTCHNSTAT_interdomain)
+		printf(": domain %d port %u\n", es.u.interdomain.dom,
+		    es.u.interdomain.port);
+	else if (es.status == EVTCHNSTAT_unbound)
+		printf(": domain %d\n", es.u.unbound.dom);
+	else if (es.status == EVTCHNSTAT_pirq)
+		printf(": pirq %u\n", es.u.pirq);
+	else if (es.status == EVTCHNSTAT_virq)
+		printf(": virq %u\n", es.u.virq);
+	else
+		printf("\n");
+#endif
+
+	return (0);
+}
+
+int
+xen_intr_disestablish(xen_intr_handle_t xih)
+{
+	struct xen_softc *sc = xen_sc;
+	evtchn_port_t port = (evtchn_port_t)xih;
+	struct evtchn_close ec;
+	struct xen_intsrc *xi;
+
+	if (xen_lookup_intsrc(sc, port) != NULL) {
+		DPRINTF("%s: failed to lookup an established interrupt handler "
+		    "for port %u\n", sc->sc_dev.dv_xname, port);
+		return (-1);
+	}
+
+	evcount_detach(&xi->xi_evcnt);
+
+	SLIST_REMOVE(&sc->sc_intrs, xi, xen_intsrc, xi_entry);
+
+	setbit((char *)sc->sc_ipg->evtchn_mask, xi->xi_port);
+	clrbit((char *)sc->sc_ipg->evtchn_pending[0], xi->xi_port);
+	membar_producer();
+
+	if (!xi->xi_noclose) {
+		ec.port = xi->xi_port;
+		if (xen_hypercall(sc, event_channel_op, 2, EVTCHNOP_close,
+		    &ec)) {
+			DPRINTF("%s: failed to close event port %u\n",
+			    sc->sc_dev.dv_xname, xi->xi_port);
+		}
+	}
+
+	free(xi, M_DEVBUF, sizeof(*xi));
+	return (0);
+}
+
+void
+xen_intr_enable(void)
+{
+	struct xen_softc *sc = xen_sc;
+	struct xen_intsrc *xi;
+	struct evtchn_unmask eu;
+
+	SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) {
+		if (!xi->xi_masked) {
+			eu.port = xi->xi_port;
+			if (xen_hypercall(sc, event_channel_op, 2,
+			    EVTCHNOP_unmask, &eu) ||
+			    isset(sc->sc_ipg->evtchn_mask, xi->xi_port))
+				printf("%s: unmasking port %u failed\n",
+				    sc->sc_dev.dv_xname, xi->xi_port);
+		}
+	}
 }
diff --git a/sys/dev/pv/xenreg.h b/sys/dev/pv/xenreg.h
index ec45722576d..36fc635097d 100644
--- a/sys/dev/pv/xenreg.h
+++ b/sys/dev/pv/xenreg.h
@@ -414,6 +414,121 @@ enum {
 	 (((vector) & HVM_CB_GSI_GSI_MASK) << HVM_CB_GSI_GSI_SHIFT))
 
 
+/*
+ * interface/event_channel.h
+ *
+ * Event channels between domains.
+ */
+
+#define EVTCHNOP_bind_interdomain	0
+#define EVTCHNOP_bind_virq		1
+#define EVTCHNOP_bind_pirq		2
+#define EVTCHNOP_close			3
+#define EVTCHNOP_send			4
+#define EVTCHNOP_status			5
+#define EVTCHNOP_alloc_unbound		6
+#define EVTCHNOP_bind_ipi		7
+#define EVTCHNOP_bind_vcpu		8
+#define EVTCHNOP_unmask			9
+#define EVTCHNOP_reset			10
+
+typedef uint32_t evtchn_port_t;
+
+/*
+ * EVTCHNOP_alloc_unbound: Allocate a port in domain <dom> and mark as
+ * accepting interdomain bindings from domain <remote_dom>. A fresh port
+ * is allocated in <dom> and returned as <port>.
+ * NOTES:
+ *  1. If the caller is unprivileged then <dom> must be DOMID_SELF.
+ *  2. <rdom> may be DOMID_SELF, allowing loopback connections.
+ */
+struct evtchn_alloc_unbound {
+	/* IN parameters */
+	domid_t dom, remote_dom;
+	/* OUT parameters */
+	evtchn_port_t port;
+};
+
+/*
+ * EVTCHNOP_close: Close a local event channel <port>. If the channel is
+ * interdomain then the remote end is placed in the unbound state
+ * (EVTCHNSTAT_unbound), awaiting a new connection.
+ */
+struct evtchn_close {
+	/* IN parameters. */
+	evtchn_port_t port;
+};
+
+/*
+ * EVTCHNOP_send: Send an event to the remote end of the channel whose local
+ * endpoint is <port>.
+ */
+struct evtchn_send {
+	/* IN parameters. */
+	evtchn_port_t port;
+};
+
+/*
+ * EVTCHNOP_status: Get the current status of the communication channel which
+ * has an endpoint at <dom, port>.
+ * NOTES:
+ *  1. <dom> may be specified as DOMID_SELF.
+ *  2. Only a sufficiently-privileged domain may obtain the status of an event
+ *     channel for which <dom> is not DOMID_SELF.
+ */
+struct evtchn_status {
+	/* IN parameters */
+	domid_t  dom;
+	evtchn_port_t port;
+	/* OUT parameters */
+#define EVTCHNSTAT_closed	0  /* Channel is not in use.                 */
+#define EVTCHNSTAT_unbound	1  /* Channel is waiting interdom connection.*/
+#define EVTCHNSTAT_interdomain	2  /* Channel is connected to remote domain. */
+#define EVTCHNSTAT_pirq		3  /* Channel is bound to a phys IRQ line.   */
+#define EVTCHNSTAT_virq		4  /* Channel is bound to a virtual IRQ line */
+#define EVTCHNSTAT_ipi		5  /* Channel is bound to a virtual IPI line */
+	uint32_t status;
+	uint32_t vcpu;		   /* VCPU to which this channel is bound.   */
+	union {
+		struct {
+			domid_t dom;
+		} unbound;	   /* EVTCHNSTAT_unbound */
+		struct {
+			domid_t dom;
+			evtchn_port_t port;
+		} interdomain;	   /* EVTCHNSTAT_interdomain */
+		uint32_t pirq;	   /* EVTCHNSTAT_pirq */
+		uint32_t virq;	   /* EVTCHNSTAT_virq */
+	} u;
+};
+
+/*
+ * EVTCHNOP_bind_vcpu: Specify which vcpu a channel should notify when an
+ * event is pending.
+ * NOTES:
+ *  1. IPI-bound channels always notify the vcpu specified at bind time.
+ *     This binding cannot be changed.
+ *  2. Per-VCPU VIRQ channels always notify the vcpu specified at bind time.
+ *     This binding cannot be changed.
+ *  3. All other channels notify vcpu0 by default. This default is set when
+ *     the channel is allocated (a port that is freed and subsequently reused
+ *     has its binding reset to vcpu0).
+ */
+struct evtchn_bind_vcpu {
+	/* IN parameters. */
+	evtchn_port_t port;
+	uint32_t vcpu;
+};
+
+/*
+ * EVTCHNOP_unmask: Unmask the specified local event-channel port and deliver
+ * a notification to the appropriate VCPU if an event is pending.
+ */
+struct evtchn_unmask {
+	/* IN parameters. */
+	evtchn_port_t port;
+};
+
 
 /*
  * interface/features.h
diff --git a/sys/dev/pv/xenvar.h b/sys/dev/pv/xenvar.h
index 7c5d2446b3f..e58d2335828 100644
--- a/sys/dev/pv/xenvar.h
+++ b/sys/dev/pv/xenvar.h
@@ -25,6 +25,16 @@
 #define DPRINTF(x...)
 #endif
 
+struct xen_intsrc {
+	SLIST_ENTRY(xen_intsrc)	  xi_entry;
+	void			(*xi_handler)(void *);
+	void			 *xi_arg;
+	struct evcount		  xi_evcnt;
+	evtchn_port_t		  xi_port;
+	int			  xi_noclose;
+	int			  xi_masked;
+};
+
 struct xen_softc {
 	struct device		 sc_dev;
 	uint32_t		 sc_base;
@@ -35,6 +45,9 @@ struct xen_softc {
 	struct shared_info	*sc_ipg;	/* HYPERVISOR_shared_info */
 
 	int			 sc_cbvec;	/* callback was installed */
+	uint64_t		 sc_irq;	/* IDT vector number */
+	struct evcount		 sc_evcnt;	/* upcall counter */
+	SLIST_HEAD(, xen_intsrc) sc_intrs;
 };
 
 extern struct xen_softc *xen_sc;
@@ -44,9 +57,23 @@ extern struct xen_softc *xen_sc;
  */
 #define memory_op		12
 #define xen_version		17
+#define event_channel_op	32
 #define hvm_op			34
 
 int	xen_hypercall(struct xen_softc *, int, int, ...);
 int	xen_hypercallv(struct xen_softc *, int, int, ulong *);
 
+/*
+ *  Interrupts
+ */
+typedef uint32_t xen_intr_handle_t;
+
+void	xen_intr(void);
+void	xen_intr_ack(void);
+void	xen_intr_signal(xen_intr_handle_t);
+int	xen_intr_establish(evtchn_port_t, xen_intr_handle_t *, void (*)(void *),
+	    void *, char *);
+int	xen_intr_disestablish(xen_intr_handle_t);
+void	xen_intr_enable(void);
+
 #endif	/* _XENVAR_H_ */