From owner-svn-src-all@FreeBSD.ORG Sat May 31 23:37:36 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id A7E991B6; Sat, 31 May 2014 23:37:36 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 87BD62823; Sat, 31 May 2014 23:37:36 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s4VNbawQ093704; Sat, 31 May 2014 23:37:36 GMT (envelope-from neel@svn.freebsd.org) Received: (from neel@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s4VNbY0X093693; Sat, 31 May 2014 23:37:34 GMT (envelope-from neel@svn.freebsd.org) Message-Id: <201405312337.s4VNbY0X093693@svn.freebsd.org> From: Neel Natu Date: Sat, 31 May 2014 23:37:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r266933 - in head: lib/libvmmapi sys/amd64/include sys/amd64/vmm sys/amd64/vmm/io usr.sbin/bhyve usr.sbin/bhyvectl X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 31 May 2014 23:37:36 -0000 Author: neel Date: Sat May 31 23:37:34 2014 New Revision: 266933 URL: http://svnweb.freebsd.org/changeset/base/266933 Log: Activate vcpus from bhyve(8) using the ioctl VM_ACTIVATE_CPU instead of doing it implicitly in vmm.ko. Add ioctl VM_GET_CPUS to get the current set of 'active' and 'suspended' cpus and display them via /usr/sbin/bhyvectl using the "--get-active-cpus" and "--get-suspended-cpus" options. This is in preparation for being able to reset virtual machine state without having to destroy and recreate it. Modified: head/lib/libvmmapi/vmmapi.c head/lib/libvmmapi/vmmapi.h head/sys/amd64/include/vmm.h head/sys/amd64/include/vmm_dev.h head/sys/amd64/vmm/io/vlapic.c head/sys/amd64/vmm/vmm.c head/sys/amd64/vmm/vmm_dev.c head/usr.sbin/bhyve/bhyverun.c head/usr.sbin/bhyve/pci_lpc.c head/usr.sbin/bhyvectl/bhyvectl.c Modified: head/lib/libvmmapi/vmmapi.c ============================================================================== --- head/lib/libvmmapi/vmmapi.c Sat May 31 22:25:45 2014 (r266932) +++ head/lib/libvmmapi/vmmapi.c Sat May 31 23:37:34 2014 (r266933) @@ -29,11 +29,12 @@ #include __FBSDID("$FreeBSD$"); -#include +#include #include #include #include #include +#include #include #include @@ -1043,3 +1044,44 @@ vm_copyout(struct vmctx *ctx, int vcpu, len -= n; } } + +static int +vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus) +{ + struct vm_cpuset vm_cpuset; + int error; + + bzero(&vm_cpuset, sizeof(struct vm_cpuset)); + vm_cpuset.which = which; + vm_cpuset.cpusetsize = sizeof(cpuset_t); + vm_cpuset.cpus = cpus; + + error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset); + return (error); +} + +int +vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus) +{ + + return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus)); +} + +int +vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus) +{ + + return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus)); +} + +int +vm_activate_cpu(struct vmctx *ctx, int vcpu) +{ + struct vm_activate_cpu ac; + int error; + + bzero(&ac, sizeof(struct vm_activate_cpu)); + ac.vcpuid = vcpu; + error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac); + return (error); +} Modified: head/lib/libvmmapi/vmmapi.h ============================================================================== --- head/lib/libvmmapi/vmmapi.h Sat May 31 22:25:45 2014 (r266932) +++ head/lib/libvmmapi/vmmapi.h Sat May 31 23:37:34 2014 (r266933) @@ -29,6 +29,9 @@ #ifndef _VMMAPI_H_ #define _VMMAPI_H_ +#include +#include + struct iovec; struct vmctx; enum x2apic_state; @@ -125,6 +128,10 @@ void vm_copyout(struct vmctx *ctx, int v /* Reset vcpu register state */ int vcpu_reset(struct vmctx *ctx, int vcpu); +int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus); +int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus); +int vm_activate_cpu(struct vmctx *ctx, int vcpu); + /* * FreeBSD specific APIs */ Modified: head/sys/amd64/include/vmm.h ============================================================================== --- head/sys/amd64/include/vmm.h Sat May 31 22:25:45 2014 (r266932) +++ head/sys/amd64/include/vmm.h Sat May 31 23:37:34 2014 (r266933) @@ -140,8 +140,9 @@ int vm_set_capability(struct vm *vm, int int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state); int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state); int vm_apicid2vcpuid(struct vm *vm, int apicid); -void vm_activate_cpu(struct vm *vm, int vcpu); +int vm_activate_cpu(struct vm *vm, int vcpu); cpuset_t vm_active_cpus(struct vm *vm); +cpuset_t vm_suspended_cpus(struct vm *vm); struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid); void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip); Modified: head/sys/amd64/include/vmm_dev.h ============================================================================== --- head/sys/amd64/include/vmm_dev.h Sat May 31 22:25:45 2014 (r266932) +++ head/sys/amd64/include/vmm_dev.h Sat May 31 23:37:34 2014 (r266933) @@ -177,6 +177,18 @@ struct vm_gla2gpa { uint64_t gpa; }; +struct vm_activate_cpu { + int vcpuid; +}; + +struct vm_cpuset { + int which; + int cpusetsize; + cpuset_t *cpus; +}; +#define VM_ACTIVE_CPUS 0 +#define VM_SUSPENDED_CPUS 1 + enum { /* general routines */ IOCNUM_ABIVERS = 0, @@ -229,6 +241,10 @@ enum { IOCNUM_ISA_DEASSERT_IRQ = 81, IOCNUM_ISA_PULSE_IRQ = 82, IOCNUM_ISA_SET_IRQ_TRIGGER = 83, + + /* vm_cpuset */ + IOCNUM_ACTIVATE_CPU = 90, + IOCNUM_GET_CPUSET = 91, }; #define VM_RUN \ @@ -301,4 +317,8 @@ enum { _IOWR('v', IOCNUM_GET_GPA_PMAP, struct vm_gpa_pte) #define VM_GLA2GPA \ _IOWR('v', IOCNUM_GLA2GPA, struct vm_gla2gpa) +#define VM_ACTIVATE_CPU \ + _IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu) +#define VM_GET_CPUS \ + _IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset) #endif Modified: head/sys/amd64/vmm/io/vlapic.c ============================================================================== --- head/sys/amd64/vmm/io/vlapic.c Sat May 31 22:25:45 2014 (r266932) +++ head/sys/amd64/vmm/io/vlapic.c Sat May 31 23:37:34 2014 (r266933) @@ -1004,11 +1004,7 @@ vlapic_icrlo_write_handler(struct vlapic if (vlapic2->boot_state != BS_SIPI) return (0); - /* - * XXX this assumes that the startup IPI always succeeds - */ vlapic2->boot_state = BS_RUNNING; - vm_activate_cpu(vlapic2->vm, dest); *retu = true; vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); Modified: head/sys/amd64/vmm/vmm.c ============================================================================== --- head/sys/amd64/vmm/vmm.c Sat May 31 22:25:45 2014 (r266932) +++ head/sys/amd64/vmm/vmm.c Sat May 31 23:37:34 2014 (r266933) @@ -342,8 +342,6 @@ vm_create(const char *name, struct vm ** struct vm *vm; struct vmspace *vmspace; - const int BSP = 0; - /* * If vmm.ko could not be successfully initialized then don't attempt * to create the virtual machine. @@ -373,8 +371,6 @@ vm_create(const char *name, struct vm ** guest_msrs_init(vm, i); } - vm_activate_cpu(vm, BSP); - *retvm = vm; return (0); } @@ -1294,6 +1290,12 @@ vm_run(struct vm *vm, struct vm_run *vmr if (vcpuid < 0 || vcpuid >= VM_MAXCPU) return (EINVAL); + if (!CPU_ISSET(vcpuid, &vm->active_cpus)) + return (EINVAL); + + if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) + return (EINVAL); + rptr = &vm->rendezvous_func; sptr = &vm->suspend; pmap = vmspace_pmap(vm->vmspace); @@ -1708,17 +1710,19 @@ vcpu_get_state(struct vm *vm, int vcpuid return (state); } -void +int vm_activate_cpu(struct vm *vm, int vcpuid) { - KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, - ("vm_activate_cpu: invalid vcpuid %d", vcpuid)); - KASSERT(!CPU_ISSET(vcpuid, &vm->active_cpus), - ("vm_activate_cpu: vcpuid %d is already active", vcpuid)); + if (vcpuid < 0 || vcpuid >= VM_MAXCPU) + return (EINVAL); + + if (CPU_ISSET(vcpuid, &vm->active_cpus)) + return (EBUSY); VCPU_CTR0(vm, vcpuid, "activated"); CPU_SET_ATOMIC(vcpuid, &vm->active_cpus); + return (0); } cpuset_t @@ -1728,6 +1732,13 @@ vm_active_cpus(struct vm *vm) return (vm->active_cpus); } +cpuset_t +vm_suspended_cpus(struct vm *vm) +{ + + return (vm->suspended_cpus); +} + void * vcpu_stats(struct vm *vm, int vcpuid) { Modified: head/sys/amd64/vmm/vmm_dev.c ============================================================================== --- head/sys/amd64/vmm/vmm_dev.c Sat May 31 22:25:45 2014 (r266932) +++ head/sys/amd64/vmm/vmm_dev.c Sat May 31 23:37:34 2014 (r266933) @@ -146,7 +146,8 @@ static int vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, struct thread *td) { - int error, vcpu, state_changed; + int error, vcpu, state_changed, size; + cpuset_t *cpuset; struct vmmdev_softc *sc; struct vm_memory_segment *seg; struct vm_register *vmreg; @@ -170,6 +171,8 @@ vmmdev_ioctl(struct cdev *cdev, u_long c struct vm_gpa_pte *gpapte; struct vm_suspend *vmsuspend; struct vm_gla2gpa *gg; + struct vm_activate_cpu *vac; + struct vm_cpuset *vm_cpuset; sc = vmmdev_lookup2(cdev); if (sc == NULL) @@ -195,6 +198,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long c case VM_PPTDEV_MSIX: case VM_SET_X2APIC_STATE: case VM_GLA2GPA: + case VM_ACTIVATE_CPU: /* * XXX fragile, handle with care * Assumes that the first field of the ioctl data is the vcpu. @@ -439,6 +443,29 @@ vmmdev_ioctl(struct cdev *cdev, u_long c } break; } + case VM_ACTIVATE_CPU: + vac = (struct vm_activate_cpu *)data; + error = vm_activate_cpu(sc->vm, vac->vcpuid); + break; + case VM_GET_CPUS: + error = 0; + vm_cpuset = (struct vm_cpuset *)data; + size = vm_cpuset->cpusetsize; + if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) { + error = ERANGE; + break; + } + cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO); + if (vm_cpuset->which == VM_ACTIVE_CPUS) + *cpuset = vm_active_cpus(sc->vm); + else if (vm_cpuset->which == VM_SUSPENDED_CPUS) + *cpuset = vm_suspended_cpus(sc->vm); + else + error = EINVAL; + if (error == 0) + error = copyout(cpuset, vm_cpuset->cpus, size); + free(cpuset, M_TEMP); + break; default: error = ENOTTY; break; Modified: head/usr.sbin/bhyve/bhyverun.c ============================================================================== --- head/usr.sbin/bhyve/bhyverun.c Sat May 31 22:25:45 2014 (r266932) +++ head/usr.sbin/bhyve/bhyverun.c Sat May 31 23:37:34 2014 (r266933) @@ -242,6 +242,15 @@ fbsdrun_addcpu(struct vmctx *ctx, int fr assert(fromcpu == BSP); + /* + * The 'newcpu' must be activated in the context of 'fromcpu'. If + * vm_activate_cpu() is delayed until newcpu's pthread starts running + * then vmm.ko is out-of-sync with bhyve and this can create a race + * with vm_suspend(). + */ + error = vm_activate_cpu(ctx, newcpu); + assert(error == 0); + CPU_SET_ATOMIC(newcpu, &cpumask); /* @@ -532,6 +541,7 @@ vm_loop(struct vmctx *ctx, int vcpu, uin int error, rc, prevcpu; enum vm_exitcode exitcode; enum vm_suspend_how how; + cpuset_t active_cpus; if (vcpumap[vcpu] != NULL) { error = pthread_setaffinity_np(pthread_self(), @@ -539,6 +549,9 @@ vm_loop(struct vmctx *ctx, int vcpu, uin assert(error == 0); } + error = vm_active_cpus(ctx, &active_cpus); + assert(CPU_ISSET(vcpu, &active_cpus)); + while (1) { error = vm_run(ctx, vcpu, rip, &vmexit[vcpu]); if (error != 0) Modified: head/usr.sbin/bhyve/pci_lpc.c ============================================================================== --- head/usr.sbin/bhyve/pci_lpc.c Sat May 31 22:25:45 2014 (r266932) +++ head/usr.sbin/bhyve/pci_lpc.c Sat May 31 23:37:34 2014 (r266933) @@ -32,7 +32,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include #include Modified: head/usr.sbin/bhyvectl/bhyvectl.c ============================================================================== --- head/usr.sbin/bhyvectl/bhyvectl.c Sat May 31 22:25:45 2014 (r266932) +++ head/usr.sbin/bhyvectl/bhyvectl.c Sat May 31 23:37:34 2014 (r266933) @@ -193,7 +193,9 @@ usage(void) " [--assert-lapic-lvt=]\n" " [--inject-nmi]\n" " [--force-reset]\n" - " [--force-poweroff]\n", + " [--force-poweroff]\n" + " [--get-active-cpus]\n" + " [--get-suspended-cpus]\n", progname); exit(1); } @@ -203,6 +205,7 @@ static int inject_nmi, assert_lapic_lvt; static int force_reset, force_poweroff; static const char *capname; static int create, destroy, get_lowmem, get_highmem; +static int get_active_cpus, get_suspended_cpus; static uint64_t memsize; static int set_cr0, get_cr0, set_cr3, get_cr3, set_cr4, get_cr4; static int set_efer, get_efer; @@ -390,6 +393,25 @@ enum { ASSERT_LAPIC_LVT, }; +static void +print_cpus(const char *banner, const cpuset_t *cpus) +{ + int i, first; + + first = 1; + printf("%s:\t", banner); + if (!CPU_EMPTY(cpus)) { + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, cpus)) { + printf("%s%d", first ? " " : ", ", i); + first = 0; + } + } + } else + printf(" (none)"); + printf("\n"); +} + int main(int argc, char *argv[]) { @@ -401,6 +423,7 @@ main(int argc, char *argv[]) uint64_t ctl, eptp, bm, addr, u64, pteval[4], *pte; struct vmctx *ctx; int wired; + cpuset_t cpus; uint64_t cr0, cr3, cr4, dr7, rsp, rip, rflags, efer, pat; uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp; @@ -570,6 +593,8 @@ main(int argc, char *argv[]) { "inject-nmi", NO_ARG, &inject_nmi, 1 }, { "force-reset", NO_ARG, &force_reset, 1 }, { "force-poweroff", NO_ARG, &force_poweroff, 1 }, + { "get-active-cpus", NO_ARG, &get_active_cpus, 1 }, + { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 }, { NULL, 0, NULL, 0 } }; @@ -1529,6 +1554,18 @@ main(int argc, char *argv[]) } } + if (!error && (get_active_cpus || get_all)) { + error = vm_active_cpus(ctx, &cpus); + if (!error) + print_cpus("active cpus", &cpus); + } + + if (!error && (get_suspended_cpus || get_all)) { + error = vm_suspended_cpus(ctx, &cpus); + if (!error) + print_cpus("suspended cpus", &cpus); + } + if (!error && run) { error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip); assert(error == 0);