have a go at using msi interrupts.

vmx has an interesting feature where config in the hypervisor can
say what type of interrupts the guest should configure for the nic,
with the options of auto, msix, msi, and intx. depending on this,
the driver should try to map the type specified and fall back from
there.

also interesting is that my guest gets "auto" from the hypervisor,
which i fall through to msi with, but an msi interrupt cannot be
mapped. i cannot see any msi interrupts in this guest actually.
there must be something funky at the platform level that we don't
like, and that prevents msi from being mapped.

if msi does get mapped, we should be able to avoid a register read
on every interrupt. that should probably provide a noticable
performance improvement if we can ever take advantage of it.

2 files changed, 51 insertions, 8 deletions

diff --git a/sys/dev/pci/if_vmx.c b/sys/dev/pci/if_vmx.c
index f1fbdf90c57..1d980f29cf2 100644
--- a/sys/dev/pci/if_vmx.c
+++ b/sys/dev/pci/if_vmx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_vmx.c,v 1.48 2019/08/06 10:09:37 dlg Exp $	*/
+/*	$OpenBSD: if_vmx.c,v 1.49 2019/08/06 10:15:27 dlg Exp $	*/
 
 /*
  * Copyright (c) 2013 Tsubai Masanari
@@ -158,6 +158,7 @@ void vmxnet3_link_state(struct vmxnet3_softc *);
 void vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
 void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
 int vmxnet3_intr(void *);
+int vmxnet3_intr_intx(void *);
 void vmxnet3_evintr(struct vmxnet3_softc *);
 void vmxnet3_txintr(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 void vmxnet3_rxintr(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
@@ -203,8 +204,9 @@ vmxnet3_attach(struct device *parent, struct device *self, void *aux)
 	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
 	pci_intr_handle_t ih;
 	const char *intrstr;
-	u_int memtype, ver, macl, mach;
+	u_int memtype, ver, macl, mach, intrcfg;
 	u_char enaddr[ETHER_ADDR_LEN];
+	int (*isr)(void *);
 
 	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x10);
 	if (pci_mapreg_map(pa, 0x10, memtype, 0, &sc->sc_iot0, &sc->sc_ioh0,
@@ -239,15 +241,36 @@ vmxnet3_attach(struct device *parent, struct device *self, void *aux)
 		return;
 	}
 
-	if (pci_intr_map(pa, &ih)) {
+	WRITE_CMD(sc, VMXNET3_CMD_GET_INTRCFG);
+	intrcfg = READ_BAR1(sc, VMXNET3_BAR1_CMD);
+	isr = vmxnet3_intr;
+
+	switch (intrcfg & VMXNET3_INTRCFG_TYPE_MASK) {
+	case VMXNET3_INTRCFG_TYPE_AUTO:
+		printf(", auto");
+	case VMXNET3_INTRCFG_TYPE_MSIX:
+		printf(", msix");
+		/* FALLTHROUGH */
+	case VMXNET3_INTRCFG_TYPE_MSI:
+		printf(", msi");
+		if (pci_intr_map_msi(pa, &ih) == 0)
+			break;
+
+		/* FALLTHROUGH */
+	case VMXNET3_INTRCFG_TYPE_INTX:
+		printf(", intx");
+		isr = vmxnet3_intr_intx;
+		if (pci_intr_map(pa, &ih) == 0)
+			break;
+
 		printf(": failed to map interrupt\n");
 		return;
 	}
 	sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE,
-	    vmxnet3_intr, sc, self->dv_xname);
+	    isr, sc, self->dv_xname);
 	intrstr = pci_intr_string(pa->pa_pc, ih);
 	if (intrstr)
-		printf(": %s", intrstr);
+		printf(": %x %s", intrcfg, intrstr);
 
 	WRITE_CMD(sc, VMXNET3_CMD_GET_MACL);
 	macl = READ_BAR1(sc, VMXNET3_BAR1_CMD);
@@ -611,14 +634,22 @@ vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
 }
 
 int
-vmxnet3_intr(void *arg)
+vmxnet3_intr_intx(void *arg)
 {
 	struct vmxnet3_softc *sc = arg;
-	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
 
 	if (READ_BAR1(sc, VMXNET3_BAR1_INTR) == 0)
 		return 0;
 
+	return (vmxnet3_intr(sc));
+}
+
+int
+vmxnet3_intr(void *arg)
+{
+	struct vmxnet3_softc *sc = arg;
+	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
+
 	if (sc->sc_ds->event) {
 		KERNEL_LOCK();
 		vmxnet3_evintr(sc);
diff --git a/sys/dev/pci/if_vmxreg.h b/sys/dev/pci/if_vmxreg.h
index 7f47851964e..ca5debdb6b0 100644
--- a/sys/dev/pci/if_vmxreg.h
+++ b/sys/dev/pci/if_vmxreg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_vmxreg.h,v 1.3 2013/08/28 10:19:19 reyk Exp $	*/
+/*	$OpenBSD: if_vmxreg.h,v 1.4 2019/08/06 10:15:27 dlg Exp $	*/
 
 /*
  * Copyright (c) 2013 Tsubai Masanari
@@ -76,6 +76,18 @@ struct UPT1_RxStats {
 #define VMXNET3_CMD_GET_LINK	0xf00d0002	/* get link status */
 #define VMXNET3_CMD_GET_MACL	0xf00d0003
 #define VMXNET3_CMD_GET_MACH	0xf00d0004
+#define VMXNET3_CMD_GET_INTRCFG	0xf00d0008	/* get interrupt config */
+#define VMXNET3_INTRCFG_TYPE_SHIFT	0
+#define VMXNET3_INTRCFG_TYPE_MASK	(0x3 << VMXNET3_INTRCFG_TYPE_SHIFT)
+#define VMXNET3_INTRCFG_TYPE_AUTO	(0x0 << VMXNET3_INTRCFG_TYPE_SHIFT)
+#define VMXNET3_INTRCFG_TYPE_INTX	(0x1 << VMXNET3_INTRCFG_TYPE_SHIFT)
+#define VMXNET3_INTRCFG_TYPE_MSI	(0x2 << VMXNET3_INTRCFG_TYPE_SHIFT)
+#define VMXNET3_INTRCFG_TYPE_MSIX	(0x3 << VMXNET3_INTRCFG_TYPE_SHIFT)
+#define VMXNET3_INTRCFG_MODE_SHIFT	2
+#define VMXNET3_INTRCFG_MODE_MASK	(0x3 << VMXNET3_INTRCFG_MODE_SHIFT)
+#define VMXNET3_INTRCFG_MODE_AUTO	(0x0 << VMXNET3_INTRCFG_MODE_SHIFT)
+#define VMXNET3_INTRCFG_MODE_ACTIVE	(0x1 << VMXNET3_INTRCFG_MODE_SHIFT)
+#define VMXNET3_INTRCFG_MODE_LAZY	(0x2 << VMXNET3_INTRCFG_MODE_SHIFT)
 
 #define VMXNET3_DMADESC_ALIGN	128