Move vcpu register state init to vmd. Allows vmd bootloader to make the

decision as to how the vcpu should be set up for initial start and
reset. Also removes some hardcoded register constants from vmm(4).

ok jsing@, mpi@

5 files changed, 194 insertions, 79 deletions

diff --git a/sys/arch/amd64/amd64/vmm.c b/sys/arch/amd64/amd64/vmm.c
index 56e29bb2b01..e9acba4e691 100644
--- a/sys/arch/amd64/amd64/vmm.c
+++ b/sys/arch/amd64/amd64/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.25 2015/12/15 03:24:26 mlarkin Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.26 2015/12/17 09:29:28 mlarkin Exp $	*/
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -102,9 +102,9 @@ int vm_get_info(struct vm_info_params *);
 int vm_writepage(struct vm_writepage_params *);
 int vm_readpage(struct vm_readpage_params *);
 int vm_resetcpu(struct vm_resetcpu_params *);
-int vcpu_reset_regs(struct vcpu *);
-int vcpu_reset_regs_vmx(struct vcpu *);
-int vcpu_reset_regs_svm(struct vcpu *);
+int vcpu_reset_regs(struct vcpu *, struct vcpu_init_state *);
+int vcpu_reset_regs_vmx(struct vcpu *, struct vcpu_init_state *);
+int vcpu_reset_regs_svm(struct vcpu *, struct vcpu_init_state *);
 int vcpu_init(struct vcpu *);
 int vcpu_init_vmx(struct vcpu *);
 int vcpu_init_svm(struct vcpu *);
@@ -486,7 +486,7 @@ vm_resetcpu(struct vm_resetcpu_params *vrp)
 	DPRINTF("vm_resetcpu: resetting vm %d vcpu %d to power on defaults\n",
 	    vm->vm_id, vcpu->vc_id);
 
-	if (vcpu_reset_regs(vcpu))
+	if (vcpu_reset_regs(vcpu, &vrp->vrp_init_state))
 		return (EIO);
 
 	return (0);
@@ -1021,7 +1021,7 @@ vm_impl_deinit(struct vm *vm)
  * XXX - unimplemented
  */
 int
-vcpu_reset_regs_svm(struct vcpu *vcpu)
+vcpu_reset_regs_svm(struct vcpu *vcpu, struct vcpu_init_state *vis)
 {
 	return (0);
 }
@@ -1029,17 +1029,18 @@ vcpu_reset_regs_svm(struct vcpu *vcpu)
 /*
  * vcpu_reset_regs_vmx
  *
- * Initializes 'vcpu's registers to default power-on state
+ * Initializes 'vcpu's registers to supplied state
  *
  * Parameters:
  *  vcpu: the vcpu whose register state is to be initialized
+ *  vis: the register state to set
  * 
  * Return values:
  *  0: registers init'ed successfully
  *  EINVAL: an error occurred setting register state
  */
 int
-vcpu_reset_regs_vmx(struct vcpu *vcpu)
+vcpu_reset_regs_vmx(struct vcpu *vcpu, struct vcpu_init_state *vis)
 {
 	int ret;
 	uint32_t cr0, cr4;
@@ -1355,7 +1356,7 @@ vcpu_reset_regs_vmx(struct vcpu *vcpu)
 	 * we want during VCPU start. This matches what the CPU state would
 	 * be after a bootloader transition to 'start'.
 	 */
-	if (vmwrite(VMCS_GUEST_IA32_RFLAGS, 0x2)) {
+	if (vmwrite(VMCS_GUEST_IA32_RFLAGS, vis->vis_rflags)) {
 		ret = EINVAL;
 		goto exit;
 	}
@@ -1373,8 +1374,8 @@ vcpu_reset_regs_vmx(struct vcpu *vcpu)
 	 * to the marks[start] from vmd's bootloader. That needs to
 	 * be hoisted up into vcpu create parameters via vm create params.
 	 */
-	vcpu->vc_gueststate.vg_rip = 0x01000160;
-	if (vmwrite(VMCS_GUEST_IA32_RIP, 0x01000160)) {
+	vcpu->vc_gueststate.vg_rip = vis->vis_rip;
+	if (vmwrite(VMCS_GUEST_IA32_RIP, vis->vis_rip)) {
 		ret = EINVAL;
 		goto exit;
 	}
@@ -1400,7 +1401,7 @@ vcpu_reset_regs_vmx(struct vcpu *vcpu)
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_CR3, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_CR3, vis->vis_cr3)) {
 		ret = EINVAL;
 		goto exit;
 	}
@@ -1417,188 +1418,187 @@ vcpu_reset_regs_vmx(struct vcpu *vcpu)
 		goto exit;
 	}
 
-	/* Set guest stack for 0x10000 - sizeof(bootloader stack setup) */
-	if (vmwrite(VMCS_GUEST_IA32_RSP, 0xFFDC)) {
+	if (vmwrite(VMCS_GUEST_IA32_RSP, vis->vis_rsp)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_SS_SEL, 0x10)) {
+	if (vmwrite(VMCS_GUEST_IA32_SS_SEL, vis->vis_ss.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_SS_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_SS_LIMIT, vis->vis_ss.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_SS_AR, 0xC093)) {
+	if (vmwrite(VMCS_GUEST_IA32_SS_AR, vis->vis_ss.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_SS_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_SS_BASE, vis->vis_ss.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_DS_SEL, 0x10)) {
+	if (vmwrite(VMCS_GUEST_IA32_DS_SEL, vis->vis_ds.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_DS_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_DS_LIMIT, vis->vis_ds.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_DS_AR, 0xC093)) {
+	if (vmwrite(VMCS_GUEST_IA32_DS_AR, vis->vis_ds.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_DS_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_DS_BASE, vis->vis_ds.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_ES_SEL, 0x10)) {
+	if (vmwrite(VMCS_GUEST_IA32_ES_SEL, vis->vis_es.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_ES_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_ES_LIMIT, vis->vis_es.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_ES_AR, 0xC093)) {
+	if (vmwrite(VMCS_GUEST_IA32_ES_AR, vis->vis_es.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_ES_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_ES_BASE, vis->vis_es.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_FS_SEL, 0x10)) {
+	if (vmwrite(VMCS_GUEST_IA32_FS_SEL, vis->vis_fs.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_FS_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_FS_LIMIT, vis->vis_fs.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_FS_AR, 0xC093)) {
+	if (vmwrite(VMCS_GUEST_IA32_FS_AR, vis->vis_fs.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_FS_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_FS_BASE, vis->vis_fs.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GS_SEL, 0x10)) {
+	if (vmwrite(VMCS_GUEST_IA32_GS_SEL, vis->vis_gs.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GS_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_GS_LIMIT, vis->vis_gs.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GS_AR, 0xC093)) {
+	if (vmwrite(VMCS_GUEST_IA32_GS_AR, vis->vis_gs.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GS_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_GS_BASE, vis->vis_gs.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_CS_SEL, 0x8)) {
+	if (vmwrite(VMCS_GUEST_IA32_CS_SEL, vis->vis_cs.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_CS_LIMIT, 0xFFFFFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_CS_LIMIT, vis->vis_cs.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_CS_AR, 0xC09F)) {
+	if (vmwrite(VMCS_GUEST_IA32_CS_AR, vis->vis_cs.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_CS_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_CS_BASE, vis->vis_cs.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GDTR_LIMIT, 0xFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_GDTR_LIMIT, vis->vis_gdtr.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_GDTR_BASE, 0x10000)) {
+	if (vmwrite(VMCS_GUEST_IA32_GDTR_BASE, vis->vis_gdtr.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_IDTR_LIMIT, 0xFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_IDTR_LIMIT, vis->vis_idtr.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_IDTR_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_IDTR_BASE, vis->vis_idtr.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_LDTR_SEL, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_LDTR_SEL, vis->vis_ldtr.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_LDTR_LIMIT, 0xFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_LDTR_LIMIT, vis->vis_ldtr.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_LDTR_AR, 0x0082)) {
+	if (vmwrite(VMCS_GUEST_IA32_LDTR_AR, vis->vis_ldtr.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_LDTR_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_LDTR_BASE, vis->vis_ldtr.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_TR_SEL, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_TR_SEL, vis->vis_tr.vsi_sel)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_TR_LIMIT, 0xFFFF)) {
+	if (vmwrite(VMCS_GUEST_IA32_TR_LIMIT, vis->vis_tr.vsi_limit)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_TR_AR, 0x008B)) {
+	if (vmwrite(VMCS_GUEST_IA32_TR_AR, vis->vis_tr.vsi_ar)) {
 		ret = EINVAL;
 		goto exit;
 	}
 
-	if (vmwrite(VMCS_GUEST_IA32_TR_BASE, 0x0)) {
+	if (vmwrite(VMCS_GUEST_IA32_TR_BASE, vis->vis_tr.vsi_base)) {
 		ret = EINVAL;
 		goto exit;
 	}
@@ -1875,12 +1875,6 @@ vcpu_init_vmx(struct vcpu *vcpu)
 		goto exit;
 	}
 
-	/* Initialize default register state */
-	if (vcpu_reset_regs(vcpu)) {
-		ret = EINVAL;
-		goto exit;
-	}
-
 exit:
 	if (ret) {
 		if (vcpu->vc_control_va)
@@ -1903,10 +1897,11 @@ exit:
 /*
  * vcpu_reset_regs
  *
- * Resets a vcpu's registers to factory power-on state
+ * Resets a vcpu's registers to the provided state
  *
  * Parameters:
  *  vcpu: the vcpu whose registers shall be reset
+ *  vis: the desired register state
  *
  * Return values:
  *  0: the vcpu's registers were successfully reset
@@ -1914,16 +1909,16 @@ exit:
  *      function for various values that can be returned here)
  */
 int 
-vcpu_reset_regs(struct vcpu *vcpu)
+vcpu_reset_regs(struct vcpu *vcpu, struct vcpu_init_state *vis)
 {
 	int ret;
 
 	if (vmm_softc->mode == VMM_MODE_VMX ||
 	    vmm_softc->mode == VMM_MODE_EPT)
-		ret = vcpu_reset_regs_vmx(vcpu);
+		ret = vcpu_reset_regs_vmx(vcpu, vis);
 	else if (vmm_softc->mode == VMM_MODE_SVM ||
 		 vmm_softc->mode == VMM_MODE_RVI)
-		ret = vcpu_reset_regs_svm(vcpu);
+		ret = vcpu_reset_regs_svm(vcpu, vis);
 	else
 		panic("unknown vmm mode\n");
 
diff --git a/sys/arch/amd64/include/vmmvar.h b/sys/arch/amd64/include/vmmvar.h
index 89ca264aca1..cf52ea6940f 100644
--- a/sys/arch/amd64/include/vmmvar.h
+++ b/sys/arch/amd64/include/vmmvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmmvar.h,v 1.5 2015/12/15 01:56:51 mlarkin Exp $	*/
+/*	$OpenBSD: vmmvar.h,v 1.6 2015/12/17 09:29:28 mlarkin Exp $	*/
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -112,7 +112,6 @@ enum {
 	VEI_DIR_IN
 };
 
-
 /*
  * vm exit data
  *  vm_exit_inout		: describes an IN/OUT exit
@@ -131,6 +130,41 @@ union vm_exit {
 	struct vm_exit_inout	vei;		/* IN/OUT exit */
 };
 
+/*
+ * struct vcpu_segment_info describes a segment + selector set, used
+ * in constructing the initial vcpu register content
+ */
+struct vcpu_segment_info {
+	uint16_t vsi_sel;
+	uint32_t vsi_limit;
+	uint32_t vsi_ar;
+	uint64_t vsi_base;
+};
+
+/*
+ * struct vcpu_init_state describes the set of vmd-settable registers
+ * that the VM's vcpus will be set to during VM boot or reset. Certain
+ * registers are always set to 0 (eg, the GP regs) and certain registers
+ * have fixed values based on hardware requirements and calculated by
+ * vmm (eg, CR0/CR4)
+ */
+struct vcpu_init_state {
+	uint64_t			vis_rflags;
+	uint64_t			vis_rip;
+	uint64_t			vis_rsp;
+	uint64_t			vis_cr3;
+
+	struct vcpu_segment_info	vis_cs;
+	struct vcpu_segment_info	vis_ds;
+	struct vcpu_segment_info	vis_es;
+	struct vcpu_segment_info	vis_fs;
+	struct vcpu_segment_info	vis_gs;
+	struct vcpu_segment_info	vis_ss;
+	struct vcpu_segment_info	vis_gdtr;
+	struct vcpu_segment_info	vis_idtr;
+	struct vcpu_segment_info	vis_ldtr;
+	struct vcpu_segment_info	vis_tr;
+};
 
 struct vm_create_params {
 	/* Input parameters to VMM_IOC_CREATE */
@@ -208,6 +242,7 @@ struct vm_resetcpu_params {
 	/* Input parameters to VMM_IOC_RESETCPU */
 	uint32_t		vrp_vm_id;
 	uint32_t		vrp_vcpu_id;
+	struct vcpu_init_state	vrp_init_state;
 };
 
 /* IOCTL definitions */
diff --git a/usr.sbin/vmd/loadfile.h b/usr.sbin/vmd/loadfile.h
index 3feb3aafc67..92155921b95 100644
--- a/usr.sbin/vmd/loadfile.h
+++ b/usr.sbin/vmd/loadfile.h
@@ -1,5 +1,5 @@
 /*	$NetBSD: loadfile.h,v 1.1 1999/04/28 09:08:50 christos Exp $	 */
-/*	$OpenBSD: loadfile.h,v 1.1 2015/11/22 20:20:32 mlarkin Exp $	 */
+/*	$OpenBSD: loadfile.h,v 1.2 2015/12/17 09:29:28 mlarkin Exp $	 */
 
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -63,6 +63,6 @@
 #define	COUNT_RANDOM	0x4000
 #define COUNT_ALL	0x7f00
 
-int loadelf_main(int, int, int);
+int loadelf_main(int, int, int, struct vcpu_init_state *);
 
 #include <machine/loadfile_machdep.h>
diff --git a/usr.sbin/vmd/loadfile_elf.c b/usr.sbin/vmd/loadfile_elf.c
index 9bfc14039b4..cf5791abeff 100644
--- a/usr.sbin/vmd/loadfile_elf.c
+++ b/usr.sbin/vmd/loadfile_elf.c
@@ -1,5 +1,5 @@
 /* $NetBSD: loadfile.c,v 1.10 2000/12/03 02:53:04 tsutsui Exp $ */
-/* $OpenBSD: loadfile_elf.c,v 1.4 2015/12/06 17:42:15 mlarkin Exp $ */
+/* $OpenBSD: loadfile_elf.c,v 1.5 2015/12/17 09:29:28 mlarkin Exp $ */
 
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -95,13 +95,14 @@
 #include <sys/param.h>
 #include <sys/exec.h>
 
-#include "loadfile.h"
-#include "vmd.h"
 #include <sys/exec_elf.h>
 #include <machine/vmmvar.h>
 #include <machine/biosvar.h>
 #include <machine/segments.h>
 
+#include "loadfile.h"
+#include "vmd.h"
+
 #define BOOTARGS_PAGE 0x2000
 #define GDT_PAGE 0x10000
 #define STACK_PAGE 0xF000
@@ -115,11 +116,10 @@ union {
 
 static void setsegment(struct mem_segment_descriptor *, uint32_t,
     size_t, int, int, int, int);
-int loadelf_main(int fd, int, int);
 int elf32_exec(int, Elf32_Ehdr *, u_long *, int);
 int elf64_exec(int, Elf64_Ehdr *, u_long *, int);
 static void push_bootargs(int);
-static void push_stack(int, uint32_t);
+static size_t push_stack(int, uint32_t);
 static void push_gdt(void);
 static size_t mread(int, uint32_t, size_t);
 static void marc4random_buf(uint32_t, int);
@@ -206,15 +206,17 @@ push_gdt(void)
  *  vm_id_in: ID of the VM to load the kernel into
  *  mem_sz: memory size in MB assigned to the guest (passed through to
  *      push_bootargs)
+ *  (out) vis: register state to set on init for this kernel
  *
  * Return values:
  *  0 if successful
  *  various error codes returned from read(2) or loadelf functions
  */
 int
-loadelf_main(int fd, int vm_id_in, int mem_sz)
+loadelf_main(int fd, int vm_id_in, int mem_sz, struct vcpu_init_state *vis)
 {
 	int r;
+	size_t stacksize;
 	u_long marks[MARK_MAX];
 
 	vm_id = vm_id_in;
@@ -233,7 +235,11 @@ loadelf_main(int fd, int vm_id_in, int mem_sz)
 
 	push_bootargs(mem_sz);
 	push_gdt();
-	push_stack(mem_sz, marks[MARK_END]);
+	stacksize = push_stack(mem_sz, marks[MARK_END]);
+
+	vis->vis_rip = (uint64_t)marks[MARK_ENTRY];
+	vis->vis_rsp = (uint64_t)(STACK_PAGE + PAGE_SIZE) - stacksize;
+	vis->vis_gdtr.vsi_base = GDT_PAGE;
 
 	return r;
 }
@@ -319,9 +325,9 @@ push_bootargs(int mem_sz)
  *  end: kernel 'end' symbol value
  *
  * Return values:
- *  nothing
+ *  size of the stack
  */
-static void
+static size_t
 push_stack(int mem_sz, uint32_t end)
 {
 	uint32_t stack[1024];
@@ -342,6 +348,8 @@ push_stack(int mem_sz, uint32_t end)
 	stack[--loc] = 0x0;
 
 	write_page(STACK_PAGE, &stack, PAGE_SIZE, 1);
+
+	return (1024 - (loc - 1)) * sizeof(uint32_t);
 }
 
 /*
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index 481b655540e..7b405dc988f 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.13 2015/12/15 02:18:34 mlarkin Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.14 2015/12/17 09:29:28 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -111,9 +111,10 @@ int opentap(void);
 int start_vm(struct imsg *, uint32_t *);
 int terminate_vm(struct vm_terminate_params *);
 int get_info_vm(struct privsep *, struct imsg *, int);
-int run_vm(int *, int *, struct vm_create_params *);
+int run_vm(int *, int *, struct vm_create_params *, struct vcpu_init_state *);
 void *vcpu_run_loop(void *);
 int vcpu_exit(struct vm_run_params *);
+int vcpu_reset(uint32_t, uint32_t, struct vcpu_init_state *);
 int vmm_create_vm(struct vm_create_params *);
 void init_emulated_hw(struct vm_create_params *, int *, int *);
 void vcpu_exit_inout(struct vm_run_params *);
@@ -142,6 +143,35 @@ static struct privsep_proc procs[] = {
 	{ "parent",	PROC_PARENT,	vmm_dispatch_parent  },
 };
 
+/*
+ * Represents a standard register set for an OS to be booted
+ * as a flat 32 bit address space, before paging is enabled.
+ *
+ * NOT set here are:
+ *  RIP
+ *  RSP
+ *  GDTR BASE
+ *
+ * Specific bootloaders should clone this structure and override
+ * those fields as needed.
+ */
+static const struct vcpu_init_state vcpu_init_flat32 = {
+	0x2,					/* RFLAGS */
+	0x0,					/* RIP */
+	0x0,					/* RSP */
+	0x0,					/* CR3 */
+	{ 0x8, 0xFFFFFFFF, 0xC09F, 0x0},	/* CS */
+	{ 0x10, 0xFFFFFFFF, 0xC093, 0x0},	/* DS */
+	{ 0x10, 0xFFFFFFFF, 0xC093, 0x0},	/* ES */
+	{ 0x10, 0xFFFFFFFF, 0xC093, 0x0},	/* FS */
+	{ 0x10, 0xFFFFFFFF, 0xC093, 0x0},	/* GS */
+	{ 0x10, 0xFFFFFFFF, 0xC093, 0x0},	/* SS */
+	{ 0x0, 0xFFFF, 0x0, 0x0},		/* GDTR */
+	{ 0x0, 0xFFFF, 0x0, 0x0},		/* IDTR */
+	{ 0x0, 0xFFFF, 0x0082, 0x0},		/* LDTR */
+	{ 0x0, 0xFFFF, 0x008B, 0x0},		/* TR */
+};
+
 pid_t
 vmm(struct privsep *ps, struct privsep_proc *p)
 {
@@ -257,6 +287,38 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 }
 
 /*
+ * vcpu_reset
+ *
+ * Requests vmm(4) to reset the VCPUs in the indicated VM to
+ * the register state provided
+ *
+ * Parameters
+ *  vmid: VM ID to reset
+ *  vcpu_id: VCPU ID to reset
+ *  vis: the register state to initialize 
+ *
+ * Return values:
+ *  0: success
+ *  !0 : ioctl to vmm(4) failed (eg, ENOENT if the supplied VM ID is not
+ *      valid)
+ */
+int
+vcpu_reset(uint32_t vmid, uint32_t vcpu_id, struct vcpu_init_state *vis)
+{
+	struct vm_resetcpu_params vrp;
+
+	memset(&vrp, 0, sizeof(vrp));
+	vrp.vrp_vm_id = vmid;
+	vrp.vrp_vcpu_id = vcpu_id;
+	memcpy(&vrp.vrp_init_state, vis, sizeof(struct vcpu_init_state));
+
+	if (ioctl(env->vmd_fd, VMM_IOC_RESETCPU, &vrp) < 0)
+		return (errno);
+
+	return (0);
+}
+
+/*
  * terminate_vm
  *
  * Requests vmm(4) to terminate the VM whose ID is provided in the
@@ -336,6 +398,7 @@ start_vm(struct imsg *imsg, uint32_t *id)
 	size_t			 i;
 	int			 ret = EINVAL;
 	int			 fds[2];
+	struct vcpu_init_state vis;
 
 	if ((vm = vm_getbyvmid(imsg->hdr.peerid)) == NULL) {
 		log_warn("%s: can't find vm", __func__);
@@ -412,9 +475,15 @@ start_vm(struct imsg *imsg, uint32_t *id)
 			fatal("create vmm ioctl failed - exiting");
 		}
 
+		/*
+		 * Set up default "flat 32 bit" register state - RIP,
+		 * RSP, and GDT info will be set in bootloader
+	 	 */
+		memcpy(&vis, &vcpu_init_flat32, sizeof(struct vcpu_init_state));
+
 		/* Load kernel image */
-		ret = loadelf_main(vm->vm_kernel,
-		    vcp->vcp_id, vcp->vcp_memory_size);
+		ret = loadelf_main(vm->vm_kernel, vcp->vcp_id,
+		    vcp->vcp_memory_size, &vis);
 		if (ret) {
 			errno = ret;
 			fatal("failed to load kernel - exiting");
@@ -427,7 +496,7 @@ start_vm(struct imsg *imsg, uint32_t *id)
 			fatal("failed to set nonblocking mode on console");
 
 		/* Execute the vcpu run loop(s) for this VM */
-		ret = run_vm(vm->vm_disks, vm->vm_ifs, vcp);
+		ret = run_vm(vm->vm_disks, vm->vm_ifs, vcp, &vis);
 
 		_exit(ret != 0);
 	}
@@ -625,13 +694,15 @@ init_emulated_hw(struct vm_create_params *vcp, int *child_disks,
  *      configuration
  *  child_disks: previously-opened child VM disk file file descriptors
  *  child_taps: previously-opened child tap file descriptors
+ *  vis: VCPU register state to initialize
  *
  * Return values:
  *  0: the VM exited normally
  *  !0 : the VM exited abnormally or failed to start
  */
 int
-run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp)
+run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp,
+    struct vcpu_init_state *vis)
 {
 	size_t i;
 	int ret;
@@ -681,6 +752,12 @@ run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp)
 		vrp[i]->vrp_vm_id = vcp->vcp_id;
 		vrp[i]->vrp_vcpu_id = i;
 
+		if (vcpu_reset(vcp->vcp_id, i, vis)) {
+			log_warn("%s: cannot reset VCPU %zu - exiting.",
+			    __progname, i);
+			return (EIO);
+		}
+
 		/* Start each VCPU run thread at vcpu_run_loop */
 		ret = pthread_create(&tid[i], NULL, vcpu_run_loop, vrp[i]);
 		if (ret) {