Skip site navigation (1)Skip section navigation (2) 
Date:      Tue, 9 Jul 2013 23:54:44 +0000 (UTC)
From:      Neel Natu <neel@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-projects@freebsd.org
Subject:   svn commit: r253123 - in projects/bhyve_npt_pmap/sys/amd64: include vmm
Message-ID:  <201307092354.r69Nsi6S094490@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: neel
Date: Tue Jul  9 23:54:43 2013
New Revision: 253123
URL: http://svnweb.freebsd.org/changeset/base/253123

Log:
  Change the vcpu state handling in preparation for doing instruction emulation.
  
  The long story:
  
  Currently the instruction fetch, decode and emulation are done in the
  vmx run loop i.e. in a critical section from the point of view of the host.
  This is no longer feasible because the VM's memory is now pageable.
  
  Accessing the VM's address space now relies on 'vm_fault_hold()' to hold
  the 'vm_page' associated with the GPA and this cannot be done inside a
  critical section.
  
  However, as soon as we exit the critical section around VMRUN(), the VMCS
  is no longer "active" and thus the vcpu cannot be in the RUNNING state.
  
  We work around this by keeping the vcpu in the FROZEN state while it is
  outside the VMRUN critical section. A vcpu in this state may transition
  back into RUNNING state or may return to IDLE state on its way back to
  userland.

Modified:
  projects/bhyve_npt_pmap/sys/amd64/include/vmm.h
  projects/bhyve_npt_pmap/sys/amd64/vmm/vmm.c
  projects/bhyve_npt_pmap/sys/amd64/vmm/vmm_dev.c

Modified: projects/bhyve_npt_pmap/sys/amd64/include/vmm.h
==============================================================================
--- projects/bhyve_npt_pmap/sys/amd64/include/vmm.h	Tue Jul  9 23:47:28 2013	(r253122)
+++ projects/bhyve_npt_pmap/sys/amd64/include/vmm.h	Tue Jul  9 23:54:43 2013	(r253123)
@@ -133,8 +133,9 @@ void *vm_iommu_domain(struct vm *vm);
 
 enum vcpu_state {
 	VCPU_IDLE,
+	VCPU_FROZEN,
 	VCPU_RUNNING,
-	VCPU_CANNOT_RUN,
+	VCPU_SLEEPING,
 };
 
 int vcpu_set_state(struct vm *vm, int vcpu, enum vcpu_state state);

Modified: projects/bhyve_npt_pmap/sys/amd64/vmm/vmm.c
==============================================================================
--- projects/bhyve_npt_pmap/sys/amd64/vmm/vmm.c	Tue Jul  9 23:47:28 2013	(r253122)
+++ projects/bhyve_npt_pmap/sys/amd64/vmm/vmm.c	Tue Jul  9 23:54:43 2013	(r253123)
@@ -93,6 +93,7 @@ struct vcpu {
 #define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
 #define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
 #define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
+#define	vcpu_assert_locked(v)	mtx_assert(&((v)->mtx), MA_OWNED)
 
 struct mem_seg {
 	vm_paddr_t	gpa;
@@ -611,6 +612,59 @@ save_guest_fpustate(struct vcpu *vcpu)
 
 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
 
+static int
+vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
+{
+	int error;
+
+	vcpu_assert_locked(vcpu);
+
+	/*
+	 * The following state transitions are allowed:
+	 * IDLE -> FROZEN -> IDLE
+	 * FROZEN -> RUNNING -> FROZEN
+	 * FROZEN -> SLEEPING -> FROZEN
+	 */
+	switch (vcpu->state) {
+	case VCPU_IDLE:
+	case VCPU_RUNNING:
+	case VCPU_SLEEPING:
+		error = (newstate != VCPU_FROZEN);
+		break;
+	case VCPU_FROZEN:
+		error = (newstate == VCPU_FROZEN);
+		break;
+	default:
+		error = 1;
+		break;
+	}
+
+	if (error == 0)
+		vcpu->state = newstate;
+	else
+		error = EBUSY;
+
+	return (error);
+}
+
+static void
+vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
+{
+	int error;
+
+	if ((error = vcpu_set_state(vm, vcpuid, newstate)) != 0)
+		panic("Error %d setting state to %d\n", error, newstate);
+}
+
+static void
+vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
+{
+	int error;
+
+	if ((error = vcpu_set_state_locked(vcpu, newstate)) != 0)
+		panic("Error %d setting state to %d", error, newstate);
+}
+
 int
 vm_run(struct vm *vm, struct vm_run *vmrun)
 {
@@ -639,9 +693,11 @@ restart:
 	restore_guest_msrs(vm, vcpuid);	
 	restore_guest_fpustate(vcpu);
 
+	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
 	vcpu->hostcpu = curcpu;
 	error = VMRUN(vm->cookie, vcpuid, rip);
 	vcpu->hostcpu = NOCPU;
+	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
 
 	save_guest_fpustate(vcpu);
 	restore_host_msrs(vm, vcpuid);
@@ -688,7 +744,9 @@ restart:
 			if (sleepticks <= 0)
 				panic("invalid sleepticks %d", sleepticks);
 			t = ticks;
+			vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
 			msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks);
+			vcpu_require_state_locked(vcpu, VCPU_FROZEN);
 			vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
 		}
 
@@ -883,7 +941,7 @@ vm_iommu_domain(struct vm *vm)
 }
 
 int
-vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state state)
+vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
 {
 	int error;
 	struct vcpu *vcpu;
@@ -894,20 +952,7 @@ vcpu_set_state(struct vm *vm, int vcpuid
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu_lock(vcpu);
-
-	/*
-	 * The following state transitions are allowed:
-	 * IDLE -> RUNNING -> IDLE
-	 * IDLE -> CANNOT_RUN -> IDLE
-	 */
-	if ((vcpu->state == VCPU_IDLE && state != VCPU_IDLE) ||
-	    (vcpu->state != VCPU_IDLE && state == VCPU_IDLE)) {
-		error = 0;
-		vcpu->state = state;
-	} else {
-		error = EBUSY;
-	}
-
+	error = vcpu_set_state_locked(vcpu, newstate);
 	vcpu_unlock(vcpu);
 
 	return (error);
@@ -993,16 +1038,7 @@ vm_interrupt_hostcpu(struct vm *vm, int 
 	vcpu_lock(vcpu);
 	hostcpu = vcpu->hostcpu;
 	if (hostcpu == NOCPU) {
-		/*
-		 * If the vcpu is 'RUNNING' but without a valid 'hostcpu' then
-		 * the host thread must be sleeping waiting for an event to
-		 * kick the vcpu out of 'hlt'.
-		 *
-		 * XXX this is racy because the condition exists right before
-		 * and after calling VMRUN() in vm_run(). The wakeup() is
-		 * benign in this case.
-		 */
-		if (vcpu->state == VCPU_RUNNING)
+		if (vcpu->state == VCPU_SLEEPING)
 			wakeup_one(vcpu);
 	} else {
 		if (vcpu->state != VCPU_RUNNING)

Modified: projects/bhyve_npt_pmap/sys/amd64/vmm/vmm_dev.c
==============================================================================
--- projects/bhyve_npt_pmap/sys/amd64/vmm/vmm_dev.c	Tue Jul  9 23:47:28 2013	(r253122)
+++ projects/bhyve_npt_pmap/sys/amd64/vmm/vmm_dev.c	Tue Jul  9 23:54:43 2013	(r253123)
@@ -139,7 +139,6 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
 	     struct thread *td)
 {
 	int error, vcpu, state_changed;
-	enum vcpu_state new_state;
 	struct vmmdev_softc *sc;
 	struct vm_memory_segment *seg;
 	struct vm_register *vmreg;
@@ -189,12 +188,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
 			goto done;
 		}
 
-		if (cmd == VM_RUN)
-			new_state = VCPU_RUNNING;
-		else
-			new_state = VCPU_CANNOT_RUN;
-
-		error = vcpu_set_state(sc->vm, vcpu, new_state);
+		error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN);
 		if (error)
 			goto done;
 
@@ -211,7 +205,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
 		 */
 		error = 0;
 		for (vcpu = 0; vcpu < VM_MAXCPU; vcpu++) {
-			error = vcpu_set_state(sc->vm, vcpu, VCPU_CANNOT_RUN);
+			error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN);
 			if (error)
 				break;
 		}

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307092354.r69Nsi6S094490>