Date: Sun, 17 Aug 2014 00:52:08 +0000 (UTC) From: Peter Grehan <grehan@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r270070 - in stable/10: lib/libvmmapi sys/amd64/include sys/amd64/vmm sys/amd64/vmm/io usr.sbin/bhyve usr.sbin/bhyvectl Message-ID: <201408170052.s7H0q8nn067016@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: grehan Date: Sun Aug 17 00:52:07 2014 New Revision: 270070 URL: http://svnweb.freebsd.org/changeset/base/270070 Log: MFC r266933 Activate vcpus from bhyve(8) using the ioctl VM_ACTIVATE_CPU instead of doing it implicitly in vmm.ko. Modified: stable/10/lib/libvmmapi/vmmapi.c stable/10/lib/libvmmapi/vmmapi.h stable/10/sys/amd64/include/vmm.h stable/10/sys/amd64/include/vmm_dev.h stable/10/sys/amd64/vmm/io/vlapic.c stable/10/sys/amd64/vmm/vmm.c stable/10/sys/amd64/vmm/vmm_dev.c stable/10/usr.sbin/bhyve/bhyverun.c stable/10/usr.sbin/bhyve/pci_lpc.c stable/10/usr.sbin/bhyvectl/bhyvectl.c Directory Properties: stable/10/ (props changed) Modified: stable/10/lib/libvmmapi/vmmapi.c ============================================================================== --- stable/10/lib/libvmmapi/vmmapi.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/lib/libvmmapi/vmmapi.c Sun Aug 17 00:52:07 2014 (r270070) @@ -29,11 +29,12 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <sys/types.h> +#include <sys/param.h> #include <sys/sysctl.h> #include <sys/ioctl.h> #include <sys/mman.h> #include <sys/_iovec.h> +#include <sys/cpuset.h> #include <machine/specialreg.h> #include <machine/param.h> @@ -1043,3 +1044,44 @@ vm_copyout(struct vmctx *ctx, int vcpu, len -= n; } } + +static int +vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus) +{ + struct vm_cpuset vm_cpuset; + int error; + + bzero(&vm_cpuset, sizeof(struct vm_cpuset)); + vm_cpuset.which = which; + vm_cpuset.cpusetsize = sizeof(cpuset_t); + vm_cpuset.cpus = cpus; + + error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset); + return (error); +} + +int +vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus) +{ + + return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus)); +} + +int +vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus) +{ + + return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus)); +} + +int +vm_activate_cpu(struct vmctx *ctx, int vcpu) +{ + struct vm_activate_cpu ac; + int error; + + bzero(&ac, sizeof(struct vm_activate_cpu)); + ac.vcpuid = vcpu; + error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac); + return (error); +} Modified: stable/10/lib/libvmmapi/vmmapi.h ============================================================================== --- stable/10/lib/libvmmapi/vmmapi.h Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/lib/libvmmapi/vmmapi.h Sun Aug 17 00:52:07 2014 (r270070) @@ -29,6 +29,9 @@ #ifndef _VMMAPI_H_ #define _VMMAPI_H_ +#include <sys/param.h> +#include <sys/cpuset.h> + struct iovec; struct vmctx; enum x2apic_state; @@ -125,6 +128,10 @@ void vm_copyout(struct vmctx *ctx, int v /* Reset vcpu register state */ int vcpu_reset(struct vmctx *ctx, int vcpu); +int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus); +int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus); +int vm_activate_cpu(struct vmctx *ctx, int vcpu); + /* * FreeBSD specific APIs */ Modified: stable/10/sys/amd64/include/vmm.h ============================================================================== --- stable/10/sys/amd64/include/vmm.h Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/sys/amd64/include/vmm.h Sun Aug 17 00:52:07 2014 (r270070) @@ -140,8 +140,9 @@ int vm_set_capability(struct vm *vm, int int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state); int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state); int vm_apicid2vcpuid(struct vm *vm, int apicid); -void vm_activate_cpu(struct vm *vm, int vcpu); +int vm_activate_cpu(struct vm *vm, int vcpu); cpuset_t vm_active_cpus(struct vm *vm); +cpuset_t vm_suspended_cpus(struct vm *vm); struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid); void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip); Modified: stable/10/sys/amd64/include/vmm_dev.h ============================================================================== --- stable/10/sys/amd64/include/vmm_dev.h Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/sys/amd64/include/vmm_dev.h Sun Aug 17 00:52:07 2014 (r270070) @@ -177,6 +177,18 @@ struct vm_gla2gpa { uint64_t gpa; }; +struct vm_activate_cpu { + int vcpuid; +}; + +struct vm_cpuset { + int which; + int cpusetsize; + cpuset_t *cpus; +}; +#define VM_ACTIVE_CPUS 0 +#define VM_SUSPENDED_CPUS 1 + enum { /* general routines */ IOCNUM_ABIVERS = 0, @@ -229,6 +241,10 @@ enum { IOCNUM_ISA_DEASSERT_IRQ = 81, IOCNUM_ISA_PULSE_IRQ = 82, IOCNUM_ISA_SET_IRQ_TRIGGER = 83, + + /* vm_cpuset */ + IOCNUM_ACTIVATE_CPU = 90, + IOCNUM_GET_CPUSET = 91, }; #define VM_RUN \ @@ -301,4 +317,8 @@ enum { _IOWR('v', IOCNUM_GET_GPA_PMAP, struct vm_gpa_pte) #define VM_GLA2GPA \ _IOWR('v', IOCNUM_GLA2GPA, struct vm_gla2gpa) +#define VM_ACTIVATE_CPU \ + _IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu) +#define VM_GET_CPUS \ + _IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset) #endif Modified: stable/10/sys/amd64/vmm/io/vlapic.c ============================================================================== --- stable/10/sys/amd64/vmm/io/vlapic.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/sys/amd64/vmm/io/vlapic.c Sun Aug 17 00:52:07 2014 (r270070) @@ -1004,11 +1004,7 @@ vlapic_icrlo_write_handler(struct vlapic if (vlapic2->boot_state != BS_SIPI) return (0); - /* - * XXX this assumes that the startup IPI always succeeds - */ vlapic2->boot_state = BS_RUNNING; - vm_activate_cpu(vlapic2->vm, dest); *retu = true; vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); Modified: stable/10/sys/amd64/vmm/vmm.c ============================================================================== --- stable/10/sys/amd64/vmm/vmm.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/sys/amd64/vmm/vmm.c Sun Aug 17 00:52:07 2014 (r270070) @@ -342,8 +342,6 @@ vm_create(const char *name, struct vm ** struct vm *vm; struct vmspace *vmspace; - const int BSP = 0; - /* * If vmm.ko could not be successfully initialized then don't attempt * to create the virtual machine. @@ -373,8 +371,6 @@ vm_create(const char *name, struct vm ** guest_msrs_init(vm, i); } - vm_activate_cpu(vm, BSP); - *retvm = vm; return (0); } @@ -1294,6 +1290,12 @@ vm_run(struct vm *vm, struct vm_run *vmr if (vcpuid < 0 || vcpuid >= VM_MAXCPU) return (EINVAL); + if (!CPU_ISSET(vcpuid, &vm->active_cpus)) + return (EINVAL); + + if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) + return (EINVAL); + rptr = &vm->rendezvous_func; sptr = &vm->suspend; pmap = vmspace_pmap(vm->vmspace); @@ -1708,17 +1710,19 @@ vcpu_get_state(struct vm *vm, int vcpuid return (state); } -void +int vm_activate_cpu(struct vm *vm, int vcpuid) { - KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, - ("vm_activate_cpu: invalid vcpuid %d", vcpuid)); - KASSERT(!CPU_ISSET(vcpuid, &vm->active_cpus), - ("vm_activate_cpu: vcpuid %d is already active", vcpuid)); + if (vcpuid < 0 || vcpuid >= VM_MAXCPU) + return (EINVAL); + + if (CPU_ISSET(vcpuid, &vm->active_cpus)) + return (EBUSY); VCPU_CTR0(vm, vcpuid, "activated"); CPU_SET_ATOMIC(vcpuid, &vm->active_cpus); + return (0); } cpuset_t @@ -1728,6 +1732,13 @@ vm_active_cpus(struct vm *vm) return (vm->active_cpus); } +cpuset_t +vm_suspended_cpus(struct vm *vm) +{ + + return (vm->suspended_cpus); +} + void * vcpu_stats(struct vm *vm, int vcpuid) { Modified: stable/10/sys/amd64/vmm/vmm_dev.c ============================================================================== --- stable/10/sys/amd64/vmm/vmm_dev.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/sys/amd64/vmm/vmm_dev.c Sun Aug 17 00:52:07 2014 (r270070) @@ -146,7 +146,8 @@ static int vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, struct thread *td) { - int error, vcpu, state_changed; + int error, vcpu, state_changed, size; + cpuset_t *cpuset; struct vmmdev_softc *sc; struct vm_memory_segment *seg; struct vm_register *vmreg; @@ -170,6 +171,8 @@ vmmdev_ioctl(struct cdev *cdev, u_long c struct vm_gpa_pte *gpapte; struct vm_suspend *vmsuspend; struct vm_gla2gpa *gg; + struct vm_activate_cpu *vac; + struct vm_cpuset *vm_cpuset; sc = vmmdev_lookup2(cdev); if (sc == NULL) @@ -195,6 +198,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long c case VM_PPTDEV_MSIX: case VM_SET_X2APIC_STATE: case VM_GLA2GPA: + case VM_ACTIVATE_CPU: /* * XXX fragile, handle with care * Assumes that the first field of the ioctl data is the vcpu. @@ -439,6 +443,29 @@ vmmdev_ioctl(struct cdev *cdev, u_long c } break; } + case VM_ACTIVATE_CPU: + vac = (struct vm_activate_cpu *)data; + error = vm_activate_cpu(sc->vm, vac->vcpuid); + break; + case VM_GET_CPUS: + error = 0; + vm_cpuset = (struct vm_cpuset *)data; + size = vm_cpuset->cpusetsize; + if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) { + error = ERANGE; + break; + } + cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO); + if (vm_cpuset->which == VM_ACTIVE_CPUS) + *cpuset = vm_active_cpus(sc->vm); + else if (vm_cpuset->which == VM_SUSPENDED_CPUS) + *cpuset = vm_suspended_cpus(sc->vm); + else + error = EINVAL; + if (error == 0) + error = copyout(cpuset, vm_cpuset->cpus, size); + free(cpuset, M_TEMP); + break; default: error = ENOTTY; break; Modified: stable/10/usr.sbin/bhyve/bhyverun.c ============================================================================== --- stable/10/usr.sbin/bhyve/bhyverun.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/usr.sbin/bhyve/bhyverun.c Sun Aug 17 00:52:07 2014 (r270070) @@ -242,6 +242,15 @@ fbsdrun_addcpu(struct vmctx *ctx, int fr assert(fromcpu == BSP); + /* + * The 'newcpu' must be activated in the context of 'fromcpu'. If + * vm_activate_cpu() is delayed until newcpu's pthread starts running + * then vmm.ko is out-of-sync with bhyve and this can create a race + * with vm_suspend(). + */ + error = vm_activate_cpu(ctx, newcpu); + assert(error == 0); + CPU_SET_ATOMIC(newcpu, &cpumask); /* @@ -532,6 +541,7 @@ vm_loop(struct vmctx *ctx, int vcpu, uin int error, rc, prevcpu; enum vm_exitcode exitcode; enum vm_suspend_how how; + cpuset_t active_cpus; if (vcpumap[vcpu] != NULL) { error = pthread_setaffinity_np(pthread_self(), @@ -539,6 +549,9 @@ vm_loop(struct vmctx *ctx, int vcpu, uin assert(error == 0); } + error = vm_active_cpus(ctx, &active_cpus); + assert(CPU_ISSET(vcpu, &active_cpus)); + while (1) { error = vm_run(ctx, vcpu, rip, &vmexit[vcpu]); if (error != 0) Modified: stable/10/usr.sbin/bhyve/pci_lpc.c ============================================================================== --- stable/10/usr.sbin/bhyve/pci_lpc.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/usr.sbin/bhyve/pci_lpc.c Sun Aug 17 00:52:07 2014 (r270070) @@ -32,7 +32,6 @@ __FBSDID("$FreeBSD$"); #include <sys/types.h> #include <machine/vmm.h> -#include <machine/vmm_dev.h> #include <stdio.h> #include <stdlib.h> Modified: stable/10/usr.sbin/bhyvectl/bhyvectl.c ============================================================================== --- stable/10/usr.sbin/bhyvectl/bhyvectl.c Sat Aug 16 22:55:58 2014 (r270069) +++ stable/10/usr.sbin/bhyvectl/bhyvectl.c Sun Aug 17 00:52:07 2014 (r270070) @@ -193,7 +193,9 @@ usage(void) " [--assert-lapic-lvt=<pin>]\n" " [--inject-nmi]\n" " [--force-reset]\n" - " [--force-poweroff]\n", + " [--force-poweroff]\n" + " [--get-active-cpus]\n" + " [--get-suspended-cpus]\n", progname); exit(1); } @@ -203,6 +205,7 @@ static int inject_nmi, assert_lapic_lvt; static int force_reset, force_poweroff; static const char *capname; static int create, destroy, get_lowmem, get_highmem; +static int get_active_cpus, get_suspended_cpus; static uint64_t memsize; static int set_cr0, get_cr0, set_cr3, get_cr3, set_cr4, get_cr4; static int set_efer, get_efer; @@ -390,6 +393,25 @@ enum { ASSERT_LAPIC_LVT, }; +static void +print_cpus(const char *banner, const cpuset_t *cpus) +{ + int i, first; + + first = 1; + printf("%s:\t", banner); + if (!CPU_EMPTY(cpus)) { + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, cpus)) { + printf("%s%d", first ? " " : ", ", i); + first = 0; + } + } + } else + printf(" (none)"); + printf("\n"); +} + int main(int argc, char *argv[]) { @@ -401,6 +423,7 @@ main(int argc, char *argv[]) uint64_t ctl, eptp, bm, addr, u64, pteval[4], *pte; struct vmctx *ctx; int wired; + cpuset_t cpus; uint64_t cr0, cr3, cr4, dr7, rsp, rip, rflags, efer, pat; uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp; @@ -570,6 +593,8 @@ main(int argc, char *argv[]) { "inject-nmi", NO_ARG, &inject_nmi, 1 }, { "force-reset", NO_ARG, &force_reset, 1 }, { "force-poweroff", NO_ARG, &force_poweroff, 1 }, + { "get-active-cpus", NO_ARG, &get_active_cpus, 1 }, + { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 }, { NULL, 0, NULL, 0 } }; @@ -1529,6 +1554,18 @@ main(int argc, char *argv[]) } } + if (!error && (get_active_cpus || get_all)) { + error = vm_active_cpus(ctx, &cpus); + if (!error) + print_cpus("active cpus", &cpus); + } + + if (!error && (get_suspended_cpus || get_all)) { + error = vm_suspended_cpus(ctx, &cpus); + if (!error) + print_cpus("suspended cpus", &cpus); + } + if (!error && run) { error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip); assert(error == 0);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201408170052.s7H0q8nn067016>