Fix at least one cause of VMs spinning at 100% host CPU

After debugging with ori@, it looks like an event ends up on the wrong
libevent queue, and we end continually de-queueing and re-queueing the
event continually. While it's unclear exactly why this happened, a clue
on libevent's github issues page for the same problem pointed us to using
a different event base for the device events. This seems to have unstuck
ori@'s problematic VM, and I have also seen no more hangs after this.

We have not completely separated the queues; ori@ will work on setting
new libevent bases for those later. But those events are pretty
frequency.

with help from and ok ori@

5 files changed, 56 insertions, 21 deletions

diff --git a/usr.sbin/vmd/i8253.c b/usr.sbin/vmd/i8253.c
index b7950608e65..0f4307d9681 100644
--- a/usr.sbin/vmd/i8253.c
+++ b/usr.sbin/vmd/i8253.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: i8253.c,v 1.29 2018/12/10 21:24:22 claudio Exp $ */
+/* $OpenBSD: i8253.c,v 1.30 2019/11/29 00:51:27 mlarkin Exp $ */
 /*
  * Copyright (c) 2016 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -34,6 +34,7 @@
 #include "atomicio.h"
 
 extern char *__progname;
+extern struct event_base *evbase;
 
 /*
  * Channel 0 is used to generate the legacy hardclock interrupt (HZ).
@@ -75,8 +76,11 @@ i8253_init(uint32_t vm_id)
 	i8253_channel[2].state = 0;
 
 	evtimer_set(&i8253_channel[0].timer, i8253_fire, &i8253_channel[0]);
+	event_base_set(evbase, &i8253_channel[0].timer);
 	evtimer_set(&i8253_channel[1].timer, i8253_fire, &i8253_channel[1]);
+	event_base_set(evbase, &i8253_channel[1].timer);
 	evtimer_set(&i8253_channel[2].timer, i8253_fire, &i8253_channel[2]);
+	event_base_set(evbase, &i8253_channel[2].timer);
 }
 
 /*
@@ -377,6 +381,7 @@ i8253_restore(int fd, uint32_t vm_id)
 		i8253_channel[i].vm_id = vm_id;
 		evtimer_set(&i8253_channel[i].timer, i8253_fire,
 		    &i8253_channel[i]);
+		event_base_set(evbase, &i8253_channel[i].timer);
 		i8253_reset(i);
 	}
 	return (0);
diff --git a/usr.sbin/vmd/mc146818.c b/usr.sbin/vmd/mc146818.c
index e4a2b84f41c..a400fe7718d 100644
--- a/usr.sbin/vmd/mc146818.c
+++ b/usr.sbin/vmd/mc146818.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: mc146818.c,v 1.19 2019/05/27 19:21:43 jasper Exp $ */
+/* $OpenBSD: mc146818.c,v 1.20 2019/11/29 00:51:27 mlarkin Exp $ */
 /*
  * Copyright (c) 2016 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -35,6 +35,8 @@
 #include "vmm.h"
 #include "atomicio.h"
 
+extern struct event_base *evbase;
+
 #define MC_DIVIDER_MASK 0xe0
 #define MC_RATE_MASK 0xf
 
@@ -172,9 +174,11 @@ mc146818_init(uint32_t vm_id, uint64_t memlo, uint64_t memhi)
 	timerclear(&rtc.per_tv);
 
 	evtimer_set(&rtc.sec, rtc_fire1, NULL);
+	event_base_set(evbase, &rtc.sec);
 	evtimer_add(&rtc.sec, &rtc.sec_tv);
 
 	evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+	event_base_set(evbase, &rtc.per);
 }
 
 /*
@@ -341,7 +345,9 @@ mc146818_restore(int fd, uint32_t vm_id)
 	memset(&rtc.sec, 0, sizeof(struct event));
 	memset(&rtc.per, 0, sizeof(struct event));
 	evtimer_set(&rtc.sec, rtc_fire1, NULL);
+	event_base_set(evbase, &rtc.sec);
 	evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+	event_base_set(evbase, &rtc.per);
 	return (0);
 }
 
diff --git a/usr.sbin/vmd/ns8250.c b/usr.sbin/vmd/ns8250.c
index 163e19a49f6..8aa88985623 100644
--- a/usr.sbin/vmd/ns8250.c
+++ b/usr.sbin/vmd/ns8250.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ns8250.c,v 1.21 2019/05/28 07:36:37 mlarkin Exp $ */
+/* $OpenBSD: ns8250.c,v 1.22 2019/11/29 00:51:27 mlarkin Exp $ */
 /*
  * Copyright (c) 2016 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -35,6 +35,7 @@
 #include "atomicio.h"
 
 extern char *__progname;
+extern struct event_base *evbase;
 struct ns8250_dev com1_dev;
 
 static void com_rcv_event(int, short, void *);
@@ -98,6 +99,7 @@ ns8250_init(int fd, uint32_t vmid)
 
 	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.event);
 
 	/*
 	 * Whenever fd is writable implies that the pty slave is connected.
@@ -106,12 +108,14 @@ ns8250_init(int fd, uint32_t vmid)
 	 */
 	event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.wake);
 	event_add(&com1_dev.wake, NULL);
 
 	/* Rate limiter for simulating baud rate */
 	timerclear(&com1_dev.rate_tv);
 	com1_dev.rate_tv.tv_usec = 10000;
 	evtimer_set(&com1_dev.rate, ratelimit, NULL);
+	event_base_set(evbase, &com1_dev.rate);
 }
 
 static void
@@ -126,25 +130,32 @@ com_rcv_event(int fd, short kind, void *arg)
 	}
 
 	/*
-	 * We already have other data pending to be received. The data that
-	 * has become available now will be moved to the com port later.
+	 * More than one byte pending to be read by the guest.
 	 */
-	if (com1_dev.rcv_pending) {
-		mutex_unlock(&com1_dev.mutex);
-		return;
-	}
+	if (com1_dev.rcv_pending)
+		goto end;
 
-	if (com1_dev.regs.lsr & LSR_RXRDY)
+	/*
+	 * One byte pending to be read by the guest.
+	 */
+	if (com1_dev.regs.lsr & LSR_RXRDY) {
 		com1_dev.rcv_pending = 1;
-	else {
+	} else {
+		/* Nothing pending */
 		com_rcv(&com1_dev, (uintptr_t)arg, 0);
+	}
 
-		/* If pending interrupt, inject */
-		if ((com1_dev.regs.iir & IIR_NOPEND) == 0) {
-			/* XXX: vcpu_id */
-			vcpu_assert_pic_irq((uintptr_t)arg, 0, com1_dev.irq);
-			vcpu_deassert_pic_irq((uintptr_t)arg, 0, com1_dev.irq);
-		}
+end:
+	if (com1_dev.regs.ier & IER_ERXRDY) {
+		com1_dev.regs.iir |= IIR_RXRDY;
+		com1_dev.regs.iir &= ~IIR_NOPEND;
+	}
+
+	/* If pending interrupt, inject */
+	if ((com1_dev.regs.iir & IIR_NOPEND) == 0) {
+		/* XXX: vcpu_id */
+		vcpu_assert_pic_irq((uintptr_t)arg, 0, com1_dev.irq);
+		vcpu_deassert_pic_irq((uintptr_t)arg, 0, com1_dev.irq);
 	}
 
 	mutex_unlock(&com1_dev.mutex);
@@ -299,6 +310,11 @@ vcpu_process_com_data(struct vm_exit *vei, uint32_t vm_id, uint32_t vcpu_id)
 		if (com1_dev.regs.iir == 0x0)
 			com1_dev.regs.iir = 0x1;
 
+		/*
+		 * Even though we just read a byte, there may be more bytes
+		 * waiting to be read - process them here, this may resassert
+		 * RXRDY.
+		 */
 		if (com1_dev.rcv_pending)
 			com_rcv(&com1_dev, vm_id, vcpu_id);
 	}
@@ -657,12 +673,15 @@ ns8250_restore(int fd, int con_fd, uint32_t vmid)
 	com1_dev.rate_tv.tv_usec = 10000;
 	com1_dev.pause_ct = (com1_dev.baudrate / 8) / 1000 * 10;
 	evtimer_set(&com1_dev.rate, ratelimit, NULL);
+	event_base_set(evbase, &com1_dev.rate);
 
 	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.event);
 
 	event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.wake);
 	event_add(&com1_dev.wake, NULL);
 
 	return (0);
diff --git a/usr.sbin/vmd/virtio.c b/usr.sbin/vmd/virtio.c
index e1c49fe7209..8dc42ffe5c1 100644
--- a/usr.sbin/vmd/virtio.c
+++ b/usr.sbin/vmd/virtio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio.c,v 1.79 2019/09/24 12:14:54 mlarkin Exp $	*/
+/*	$OpenBSD: virtio.c,v 1.80 2019/11/29 00:51:27 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -48,6 +48,7 @@
 #include "atomicio.h"
 
 extern char *__progname;
+extern struct event_base *evbase;
 
 struct viornd_dev viornd;
 struct vioblk_dev *vioblk;
@@ -1880,6 +1881,7 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
 
 			event_set(&vionet[i].event, vionet[i].fd,
 			    EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
+			event_base_set(evbase, &vionet[i].event);
 			if (event_add(&vionet[i].event, NULL)) {
 				log_warn("could not initialize vionet event "
 				    "handler");
@@ -2034,6 +2036,7 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
 	vmmci.pci_id = id;
 
 	evtimer_set(&vmmci.timeout, vmmci_timeout, NULL);
+	event_base_set(evbase, &vmmci.timeout);
 }
 
 void
diff --git a/usr.sbin/vmd/vm.c b/usr.sbin/vmd/vm.c
index 48776a74971..d787839cc1e 100644
--- a/usr.sbin/vmd/vm.c
+++ b/usr.sbin/vmd/vm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm.c,v 1.51 2019/07/17 05:51:07 pd Exp $	*/
+/*	$OpenBSD: vm.c,v 1.52 2019/11/29 00:51:28 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -114,6 +114,8 @@ pthread_mutex_t vcpu_run_mtx[VMM_MAX_VCPUS_PER_VM];
 uint8_t vcpu_hlt[VMM_MAX_VCPUS_PER_VM];
 uint8_t vcpu_done[VMM_MAX_VCPUS_PER_VM];
 
+struct event_base *evbase;
+
 /*
  * Represents a standard register set for an OS to be booted
  * as a flat 64 bit address space.
@@ -360,7 +362,7 @@ start_vm(struct vmd_vm *vm, int fd)
 	for (i = 0; i < VMM_MAX_NICS_PER_VM; i++)
 		nicfds[i] = vm->vm_ifs[i].vif_fd;
 
-	event_init();
+	evbase = event_base_new();
 
 	if (vm->vm_state & VM_STATE_RECEIVED) {
 		restore_emulated_hw(vcp, vm->vm_receive_fd, nicfds,
@@ -1297,7 +1299,7 @@ event_thread(void *arg)
 	uint8_t *donep = arg;
 	intptr_t ret;
 
-	ret = event_dispatch();
+	ret = event_base_dispatch(evbase);
 
 	mutex_lock(&threadmutex);
 	*donep = 1;