From owner-svn-src-all@FreeBSD.ORG Mon Dec 22 21:32:44 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id AAA075C7; Mon, 22 Dec 2014 21:32:44 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 941C01925; Mon, 22 Dec 2014 21:32:44 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id sBMLWiDU074555; Mon, 22 Dec 2014 21:32:44 GMT (envelope-from jhb@FreeBSD.org) Received: (from jhb@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id sBMLWecl074522; Mon, 22 Dec 2014 21:32:40 GMT (envelope-from jhb@FreeBSD.org) Message-Id: <201412222132.sBMLWecl074522@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: jhb set sender to jhb@FreeBSD.org using -f From: John Baldwin Date: Mon, 22 Dec 2014 21:32:40 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r276084 - in stable/10/sys: amd64/amd64 i386/i386 i386/include i386/isa i386/linux x86/acpica X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 22 Dec 2014 21:32:44 -0000 Author: jhb Date: Mon Dec 22 21:32:39 2014 New Revision: 276084 URL: https://svnweb.freebsd.org/changeset/base/276084 Log: MFC 273988,273989,273995,274057: MFamd64: Add support for extended FPU states on i386. This includes support for AVX on i386. Modified: stable/10/sys/amd64/amd64/genassym.c stable/10/sys/amd64/amd64/sys_machdep.c stable/10/sys/amd64/amd64/vm_machdep.c stable/10/sys/i386/i386/genassym.c stable/10/sys/i386/i386/initcpu.c stable/10/sys/i386/i386/locore.s stable/10/sys/i386/i386/machdep.c stable/10/sys/i386/i386/mp_machdep.c stable/10/sys/i386/i386/ptrace_machdep.c stable/10/sys/i386/i386/sys_machdep.c stable/10/sys/i386/i386/trap.c stable/10/sys/i386/i386/vm_machdep.c stable/10/sys/i386/include/cpufunc.h stable/10/sys/i386/include/md_var.h stable/10/sys/i386/include/npx.h stable/10/sys/i386/include/pcb.h stable/10/sys/i386/isa/npx.c stable/10/sys/i386/linux/linux_ptrace.c stable/10/sys/x86/acpica/acpi_wakeup.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/amd64/amd64/genassym.c ============================================================================== --- stable/10/sys/amd64/amd64/genassym.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/amd64/amd64/genassym.c Mon Dec 22 21:32:39 2014 (r276084) @@ -156,8 +156,6 @@ ASSYM(PCB_ONFAULT, offsetof(struct pcb, ASSYM(PCB_GS32SD, offsetof(struct pcb, pcb_gs32sd)); ASSYM(PCB_TSSP, offsetof(struct pcb, pcb_tssp)); ASSYM(PCB_SAVEFPU, offsetof(struct pcb, pcb_save)); -ASSYM(PCB_SAVEFPU_SIZE, sizeof(struct savefpu)); -ASSYM(PCB_USERFPU, sizeof(struct pcb)); ASSYM(PCB_EFER, offsetof(struct pcb, pcb_efer)); ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star)); ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar)); Modified: stable/10/sys/amd64/amd64/sys_machdep.c ============================================================================== --- stable/10/sys/amd64/amd64/sys_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/amd64/amd64/sys_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -320,7 +320,7 @@ sysarch(td, uap) fpugetregs(td); error = copyout((char *)(get_pcb_user_save_td(td) + 1), a64xfpu.addr, a64xfpu.len); - return (error); + break; default: error = EINVAL; Modified: stable/10/sys/amd64/amd64/vm_machdep.c ============================================================================== --- stable/10/sys/amd64/amd64/vm_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/amd64/amd64/vm_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -127,7 +127,7 @@ get_pcb_td(struct thread *td) void * alloc_fpusave(int flags) { - struct pcb *res; + void *res; struct savefpu_ymm *sf; res = malloc(cpu_max_ext_state_size, M_DEVBUF, flags); Modified: stable/10/sys/i386/i386/genassym.c ============================================================================== --- stable/10/sys/i386/i386/genassym.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/genassym.c Mon Dec 22 21:32:39 2014 (r276084) @@ -144,7 +144,6 @@ ASSYM(PCB_DR2, offsetof(struct pcb, pcb_ ASSYM(PCB_DR3, offsetof(struct pcb, pcb_dr3)); ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6)); ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7)); -ASSYM(PCB_USERFPU, offsetof(struct pcb, pcb_user_save)); ASSYM(PCB_PSL, offsetof(struct pcb, pcb_psl)); ASSYM(PCB_DBREGS, PCB_DBREGS); ASSYM(PCB_EXT, offsetof(struct pcb, pcb_ext)); @@ -154,7 +153,6 @@ ASSYM(PCB_GSD, offsetof(struct pcb, pcb_ ASSYM(PCB_VM86, offsetof(struct pcb, pcb_vm86)); ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); ASSYM(PCB_SAVEFPU, offsetof(struct pcb, pcb_save)); -ASSYM(PCB_SAVEFPU_SIZE, sizeof(union savefpu)); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); ASSYM(PCB_SIZE, sizeof(struct pcb)); Modified: stable/10/sys/i386/i386/initcpu.c ============================================================================== --- stable/10/sys/i386/i386/initcpu.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/initcpu.c Mon Dec 22 21:32:39 2014 (r276084) @@ -102,6 +102,7 @@ u_int cpu_mxcsr_mask; /* Valid bits in #endif u_int cpu_clflush_line_size = 32; u_int cpu_stdext_feature; +u_int cpu_max_ext_state_size; u_int cpu_mon_mwait_flags; /* MONITOR/MWAIT flags (CPUID.05H.ECX) */ u_int cpu_mon_min_size; /* MONITOR minimum range size, bytes */ u_int cpu_mon_max_size; /* MONITOR minimum range size, bytes */ Modified: stable/10/sys/i386/i386/locore.s ============================================================================== --- stable/10/sys/i386/i386/locore.s Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/locore.s Mon Dec 22 21:32:39 2014 (r276084) @@ -302,17 +302,14 @@ NON_GPROF_ENTRY(btext) begin: /* set up bootstrap stack */ movl proc0kstack,%eax /* location of in-kernel stack */ - /* bootstrap stack end location */ - leal (KSTACK_PAGES*PAGE_SIZE-PCB_SIZE)(%eax),%esp - xorl %ebp,%ebp /* mark end of frames */ + /* + * Only use bottom page for init386(). init386() calculates the + * PCB + FPU save area size and returns the true top of stack. + */ + leal PAGE_SIZE(%eax),%esp -#ifdef PAE - movl IdlePDPT,%esi -#else - movl IdlePTD,%esi -#endif - movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax) + xorl %ebp,%ebp /* mark end of frames */ pushl physfree /* value of first for init386(first) */ call init386 /* wire 386 chip for unix operation */ @@ -324,6 +321,9 @@ begin: */ addl $4,%esp + /* Switch to true top of stack. */ + movl %eax,%esp + call mi_startup /* autoconfiguration, mountroot etc */ /* NOTREACHED */ addl $0,%esp /* for db_numargs() again */ Modified: stable/10/sys/i386/i386/machdep.c ============================================================================== --- stable/10/sys/i386/i386/machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -179,7 +179,7 @@ extern unsigned long physfree; /* Sanity check for __curthread() */ CTASSERT(offsetof(struct pcpu, pc_curthread) == 0); -extern void init386(int first); +extern register_t init386(int first); extern void dblfault_handler(void); #define CS_SECURE(cs) (ISPL(cs) == SEL_UPL) @@ -191,8 +191,10 @@ extern void dblfault_handler(void); static void cpu_startup(void *); static void fpstate_drop(struct thread *td); -static void get_fpcontext(struct thread *td, mcontext_t *mcp); -static int set_fpcontext(struct thread *td, const mcontext_t *mcp); +static void get_fpcontext(struct thread *td, mcontext_t *mcp, + char *xfpusave, size_t xfpusave_len); +static int set_fpcontext(struct thread *td, const mcontext_t *mcp, + char *xfpustate, size_t xfpustate_len); #ifdef CPU_ENABLE_SSE static void set_fpregs_xmm(struct save87 *, struct savexmm *); static void fill_fpregs_xmm(struct savexmm *, struct save87 *); @@ -346,7 +348,7 @@ cpu_startup(dummy) * Send an interrupt to process. * * Stack is set up to allow sigcode stored - * at top to call routine, followed by kcall + * at top to call routine, followed by call * to sigreturn routine below. After sigreturn * resets the signal mask, the stack, and the * frame pointer, it returns to the user @@ -625,6 +627,8 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, char *sp; struct trapframe *regs; struct segment_descriptor *sdp; + char *xfpusave; + size_t xfpusave_len; int sig; int oonstack; @@ -649,6 +653,18 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, regs = td->td_frame; oonstack = sigonstack(regs->tf_esp); +#ifdef CPU_ENABLE_SSE + if (cpu_max_ext_state_size > sizeof(union savefpu) && use_xsave) { + xfpusave_len = cpu_max_ext_state_size - sizeof(union savefpu); + xfpusave = __builtin_alloca(xfpusave_len); + } else { +#else + { +#endif + xfpusave_len = 0; + xfpusave = NULL; + } + /* Save user context. */ bzero(&sf, sizeof(sf)); sf.sf_uc.uc_sigmask = *mask; @@ -659,7 +675,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sf.sf_uc.uc_mcontext.mc_gs = rgs(); bcopy(regs, &sf.sf_uc.uc_mcontext.mc_fs, sizeof(*regs)); sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */ - get_fpcontext(td, &sf.sf_uc.uc_mcontext); + get_fpcontext(td, &sf.sf_uc.uc_mcontext, xfpusave, xfpusave_len); fpstate_drop(td); /* * Unconditionally fill the fsbase and gsbase into the mcontext. @@ -670,7 +686,6 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sdp = &td->td_pcb->pcb_gsd; sf.sf_uc.uc_mcontext.mc_gsbase = sdp->sd_hibase << 24 | sdp->sd_lobase; - sf.sf_uc.uc_mcontext.mc_flags = 0; bzero(sf.sf_uc.uc_mcontext.mc_spare2, sizeof(sf.sf_uc.uc_mcontext.mc_spare2)); bzero(sf.sf_uc.__spare__, sizeof(sf.sf_uc.__spare__)); @@ -678,13 +693,19 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, /* Allocate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { - sp = td->td_sigstk.ss_sp + - td->td_sigstk.ss_size - sizeof(struct sigframe); + sp = td->td_sigstk.ss_sp + td->td_sigstk.ss_size; #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else - sp = (char *)regs->tf_esp - sizeof(struct sigframe); + sp = (char *)regs->tf_esp - 128; + if (xfpusave != NULL) { + sp -= xfpusave_len; + sp = (char *)((unsigned int)sp & ~0x3F); + sf.sf_uc.uc_mcontext.mc_xfpustate = (register_t)sp; + } + sp -= sizeof(struct sigframe); + /* Align to 16 bytes. */ sfp = (struct sigframe *)((unsigned int)sp & ~0xF); @@ -745,7 +766,10 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, /* * Copy the sigframe out to the user's stack. */ - if (copyout(&sf, sfp, sizeof(*sfp)) != 0) { + if (copyout(&sf, sfp, sizeof(*sfp)) != 0 || + (xfpusave != NULL && copyout(xfpusave, + (void *)sf.sf_uc.uc_mcontext.mc_xfpustate, xfpusave_len) + != 0)) { #ifdef DEBUG printf("process %ld has trashed its stack\n", (long)p->p_pid); #endif @@ -1005,11 +1029,16 @@ sys_sigreturn(td, uap) } */ *uap; { ucontext_t uc; + struct proc *p; struct trapframe *regs; ucontext_t *ucp; + char *xfpustate; + size_t xfpustate_len; int cs, eflags, error, ret; ksiginfo_t ksi; + p = td->td_proc; + error = copyin(uap->sigcntxp, &uc, sizeof(uc)); if (error != 0) return (error); @@ -1084,7 +1113,30 @@ sys_sigreturn(td, uap) return (EINVAL); } - ret = set_fpcontext(td, &ucp->uc_mcontext); + if ((uc.uc_mcontext.mc_flags & _MC_HASFPXSTATE) != 0) { + xfpustate_len = uc.uc_mcontext.mc_xfpustate_len; + if (xfpustate_len > cpu_max_ext_state_size - + sizeof(union savefpu)) { + uprintf( + "pid %d (%s): sigreturn xfpusave_len = 0x%zx\n", + p->p_pid, td->td_name, xfpustate_len); + return (EINVAL); + } + xfpustate = __builtin_alloca(xfpustate_len); + error = copyin((const void *)uc.uc_mcontext.mc_xfpustate, + xfpustate, xfpustate_len); + if (error != 0) { + uprintf( + "pid %d (%s): sigreturn copying xfpustate failed\n", + p->p_pid, td->td_name); + return (error); + } + } else { + xfpustate = NULL; + xfpustate_len = 0; + } + ret = set_fpcontext(td, &ucp->uc_mcontext, xfpustate, + xfpustate_len); if (ret != 0) return (ret); bcopy(&ucp->uc_mcontext.mc_fs, regs, sizeof(*regs)); @@ -1575,17 +1627,9 @@ exec_setregs(struct thread *td, struct i */ reset_dbregs(); } - pcb->pcb_flags &= ~PCB_DBREGS; + pcb->pcb_flags &= ~PCB_DBREGS; } - /* - * Initialize the math emulator (if any) for the current process. - * Actually, just clear the bit that says that the emulator has - * been initialized. Initialization is delayed until the process - * traps to the emulator (if it is done at all) mainly because - * emulators don't provide an entry point for initialization. - */ - td->td_pcb->pcb_flags &= ~FP_SOFTFP; pcb->pcb_initial_npxcw = __INITIAL_NPXCW__; /* @@ -2552,14 +2596,16 @@ do_next: #ifdef XEN #define MTOPSIZE (1<<(14 + PAGE_SHIFT)) -void +register_t init386(first) int first; { unsigned long gdtmachpfn; int error, gsel_tss, metadata_missing, x, pa; - size_t kstack0_sz; struct pcpu *pc; +#ifdef CPU_ENABLE_SSE + struct xstate_hdr *xhdr; +#endif struct callback_register event = { .type = CALLBACKTYPE_event, .address = {GSEL(GCODE_SEL, SEL_KPL), (unsigned long)Xhypervisor_callback }, @@ -2571,8 +2617,6 @@ init386(first) thread0.td_kstack = proc0kstack; thread0.td_kstack_pages = KSTACK_PAGES; - kstack0_sz = thread0.td_kstack_pages * PAGE_SIZE; - thread0.td_pcb = (struct pcb *)(thread0.td_kstack + kstack0_sz) - 1; /* * This may be done better later if it gets more high level @@ -2652,7 +2696,6 @@ init386(first) PCPU_SET(prvspace, pc); PCPU_SET(curthread, &thread0); - PCPU_SET(curpcb, thread0.td_pcb); /* * Initialize mutexes. @@ -2735,15 +2778,6 @@ init386(first) initializecpu(); /* Initialize CPU registers */ initializecpucache(); - /* make an initial tss so cpu can get interrupt stack on syscall! */ - /* Note: -16 is so we can grow the trapframe if we came from vm86 */ - PCPU_SET(common_tss.tss_esp0, thread0.td_kstack + - kstack0_sz - sizeof(struct pcb) - 16); - PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); - gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); - HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), - PCPU_GET(common_tss.tss_esp0)); - /* pointer to selector slot for %fs/%gs */ PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd); @@ -2771,6 +2805,32 @@ init386(first) /* now running on new page tables, configured,and u/iom is accessible */ msgbufinit(msgbufp, msgbufsize); +#ifdef DEV_NPX + npxinit(true); +#endif + /* + * Set up thread0 pcb after npxinit calculated pcb + fpu save + * area size. Zero out the extended state header in fpu save + * area. + */ + thread0.td_pcb = get_pcb_td(&thread0); + bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size); +#ifdef CPU_ENABLE_SSE + if (use_xsave) { + xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) + + 1); + xhdr->xstate_bv = xsave_mask; + } +#endif + PCPU_SET(curpcb, thread0.td_pcb); + /* make an initial tss so cpu can get interrupt stack on syscall! */ + /* Note: -16 is so we can grow the trapframe if we came from vm86 */ + PCPU_SET(common_tss.tss_esp0, (vm_offset_t)thread0.td_pcb - 16); + PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), + PCPU_GET(common_tss.tss_esp0)); + /* transfer to user mode */ _ucodesel = GSEL(GUCODE_SEL, SEL_UPL); @@ -2789,22 +2849,25 @@ init386(first) thread0.td_pcb->pcb_gsd = PCPU_GET(fsgs_gdt)[1]; cpu_probe_amdc1e(); + + /* Location of kernel stack for locore */ + return ((register_t)thread0.td_pcb); } #else -void +register_t init386(first) int first; { struct gate_descriptor *gdp; int gsel_tss, metadata_missing, x, pa; - size_t kstack0_sz; struct pcpu *pc; +#ifdef CPU_ENABLE_SSE + struct xstate_hdr *xhdr; +#endif thread0.td_kstack = proc0kstack; thread0.td_kstack_pages = KSTACK_PAGES; - kstack0_sz = thread0.td_kstack_pages * PAGE_SIZE; - thread0.td_pcb = (struct pcb *)(thread0.td_kstack + kstack0_sz) - 1; /* * This may be done better later if it gets more high level @@ -2858,7 +2921,6 @@ init386(first) first += DPCPU_SIZE; PCPU_SET(prvspace, pc); PCPU_SET(curthread, &thread0); - PCPU_SET(curpcb, thread0.td_pcb); /* * Initialize mutexes. @@ -3012,17 +3074,6 @@ init386(first) initializecpu(); /* Initialize CPU registers */ initializecpucache(); - /* make an initial tss so cpu can get interrupt stack on syscall! */ - /* Note: -16 is so we can grow the trapframe if we came from vm86 */ - PCPU_SET(common_tss.tss_esp0, thread0.td_kstack + - kstack0_sz - sizeof(struct pcb) - 16); - PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); - gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); - PCPU_SET(tss_gdt, &gdt[GPROC0_SEL].sd); - PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); - PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); - ltr(gsel_tss); - /* pointer to selector slot for %fs/%gs */ PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd); @@ -3050,6 +3101,33 @@ init386(first) /* now running on new page tables, configured,and u/iom is accessible */ msgbufinit(msgbufp, msgbufsize); +#ifdef DEV_NPX + npxinit(true); +#endif + /* + * Set up thread0 pcb after npxinit calculated pcb + fpu save + * area size. Zero out the extended state header in fpu save + * area. + */ + thread0.td_pcb = get_pcb_td(&thread0); + bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size); +#ifdef CPU_ENABLE_SSE + if (use_xsave) { + xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) + + 1); + xhdr->xstate_bv = xsave_mask; + } +#endif + PCPU_SET(curpcb, thread0.td_pcb); + /* make an initial tss so cpu can get interrupt stack on syscall! */ + /* Note: -16 is so we can grow the trapframe if we came from vm86 */ + PCPU_SET(common_tss.tss_esp0, (vm_offset_t)thread0.td_pcb - 16); + PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + PCPU_SET(tss_gdt, &gdt[GPROC0_SEL].sd); + PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); + PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); + ltr(gsel_tss); /* make a call gate to reenter kernel with */ gdp = &ldt[LSYS5CALLS_SEL].gd; @@ -3088,6 +3166,9 @@ init386(first) #ifdef FDT x86_init_fdt(); #endif + + /* Location of kernel stack for locore */ + return ((register_t)thread0.td_pcb); } #endif @@ -3368,11 +3449,11 @@ fill_fpregs(struct thread *td, struct fp #endif #ifdef CPU_ENABLE_SSE if (cpu_fxsr) - fill_fpregs_xmm(&td->td_pcb->pcb_user_save.sv_xmm, + fill_fpregs_xmm(&get_pcb_user_save_td(td)->sv_xmm, (struct save87 *)fpregs); else #endif /* CPU_ENABLE_SSE */ - bcopy(&td->td_pcb->pcb_user_save.sv_87, fpregs, + bcopy(&get_pcb_user_save_td(td)->sv_87, fpregs, sizeof(*fpregs)); return (0); } @@ -3384,10 +3465,10 @@ set_fpregs(struct thread *td, struct fpr #ifdef CPU_ENABLE_SSE if (cpu_fxsr) set_fpregs_xmm((struct save87 *)fpregs, - &td->td_pcb->pcb_user_save.sv_xmm); + &get_pcb_user_save_td(td)->sv_xmm); else #endif /* CPU_ENABLE_SSE */ - bcopy(fpregs, &td->td_pcb->pcb_user_save.sv_87, + bcopy(fpregs, &get_pcb_user_save_td(td)->sv_87, sizeof(*fpregs)); #ifdef DEV_NPX npxuserinited(td); @@ -3433,12 +3514,14 @@ get_mcontext(struct thread *td, mcontext mcp->mc_esp = tp->tf_esp; mcp->mc_ss = tp->tf_ss; mcp->mc_len = sizeof(*mcp); - get_fpcontext(td, mcp); + get_fpcontext(td, mcp, NULL, 0); sdp = &td->td_pcb->pcb_fsd; mcp->mc_fsbase = sdp->sd_hibase << 24 | sdp->sd_lobase; sdp = &td->td_pcb->pcb_gsd; mcp->mc_gsbase = sdp->sd_hibase << 24 | sdp->sd_lobase; mcp->mc_flags = 0; + mcp->mc_xfpustate = 0; + mcp->mc_xfpustate_len = 0; bzero(mcp->mc_spare2, sizeof(mcp->mc_spare2)); return (0); } @@ -3453,6 +3536,7 @@ int set_mcontext(struct thread *td, const mcontext_t *mcp) { struct trapframe *tp; + char *xfpustate; int eflags, ret; tp = td->td_frame; @@ -3460,30 +3544,45 @@ set_mcontext(struct thread *td, const mc return (EINVAL); eflags = (mcp->mc_eflags & PSL_USERCHANGE) | (tp->tf_eflags & ~PSL_USERCHANGE); - if ((ret = set_fpcontext(td, mcp)) == 0) { - tp->tf_fs = mcp->mc_fs; - tp->tf_es = mcp->mc_es; - tp->tf_ds = mcp->mc_ds; - tp->tf_edi = mcp->mc_edi; - tp->tf_esi = mcp->mc_esi; - tp->tf_ebp = mcp->mc_ebp; - tp->tf_ebx = mcp->mc_ebx; - tp->tf_edx = mcp->mc_edx; - tp->tf_ecx = mcp->mc_ecx; - tp->tf_eax = mcp->mc_eax; - tp->tf_eip = mcp->mc_eip; - tp->tf_eflags = eflags; - tp->tf_esp = mcp->mc_esp; - tp->tf_ss = mcp->mc_ss; - td->td_pcb->pcb_gs = mcp->mc_gs; - ret = 0; - } - return (ret); + if (mcp->mc_flags & _MC_HASFPXSTATE) { + if (mcp->mc_xfpustate_len > cpu_max_ext_state_size - + sizeof(union savefpu)) + return (EINVAL); + xfpustate = __builtin_alloca(mcp->mc_xfpustate_len); + ret = copyin((void *)mcp->mc_xfpustate, xfpustate, + mcp->mc_xfpustate_len); + if (ret != 0) + return (ret); + } else + xfpustate = NULL; + ret = set_fpcontext(td, mcp, xfpustate, mcp->mc_xfpustate_len); + if (ret != 0) + return (ret); + tp->tf_fs = mcp->mc_fs; + tp->tf_es = mcp->mc_es; + tp->tf_ds = mcp->mc_ds; + tp->tf_edi = mcp->mc_edi; + tp->tf_esi = mcp->mc_esi; + tp->tf_ebp = mcp->mc_ebp; + tp->tf_ebx = mcp->mc_ebx; + tp->tf_edx = mcp->mc_edx; + tp->tf_ecx = mcp->mc_ecx; + tp->tf_eax = mcp->mc_eax; + tp->tf_eip = mcp->mc_eip; + tp->tf_eflags = eflags; + tp->tf_esp = mcp->mc_esp; + tp->tf_ss = mcp->mc_ss; + td->td_pcb->pcb_gs = mcp->mc_gs; + return (0); } static void -get_fpcontext(struct thread *td, mcontext_t *mcp) +get_fpcontext(struct thread *td, mcontext_t *mcp, char *xfpusave, + size_t xfpusave_len) { +#ifdef CPU_ENABLE_SSE + size_t max_len, len; +#endif #ifndef DEV_NPX mcp->mc_fpformat = _MC_FPFMT_NODEV; @@ -3491,37 +3590,56 @@ get_fpcontext(struct thread *td, mcontex bzero(mcp->mc_fpstate, sizeof(mcp->mc_fpstate)); #else mcp->mc_ownedfp = npxgetregs(td); - bcopy(&td->td_pcb->pcb_user_save, &mcp->mc_fpstate[0], + bcopy(get_pcb_user_save_td(td), &mcp->mc_fpstate[0], sizeof(mcp->mc_fpstate)); mcp->mc_fpformat = npxformat(); +#ifdef CPU_ENABLE_SSE + if (!use_xsave || xfpusave_len == 0) + return; + max_len = cpu_max_ext_state_size - sizeof(union savefpu); + len = xfpusave_len; + if (len > max_len) { + len = max_len; + bzero(xfpusave + max_len, len - max_len); + } + mcp->mc_flags |= _MC_HASFPXSTATE; + mcp->mc_xfpustate_len = len; + bcopy(get_pcb_user_save_td(td) + 1, xfpusave, len); +#endif #endif } static int -set_fpcontext(struct thread *td, const mcontext_t *mcp) +set_fpcontext(struct thread *td, const mcontext_t *mcp, char *xfpustate, + size_t xfpustate_len) { + union savefpu *fpstate; + int error; if (mcp->mc_fpformat == _MC_FPFMT_NODEV) return (0); else if (mcp->mc_fpformat != _MC_FPFMT_387 && mcp->mc_fpformat != _MC_FPFMT_XMM) return (EINVAL); - else if (mcp->mc_ownedfp == _MC_FPOWNED_NONE) + else if (mcp->mc_ownedfp == _MC_FPOWNED_NONE) { /* We don't care what state is left in the FPU or PCB. */ fpstate_drop(td); - else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU || + error = 0; + } else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU || mcp->mc_ownedfp == _MC_FPOWNED_PCB) { #ifdef DEV_NPX + fpstate = (union savefpu *)&mcp->mc_fpstate; #ifdef CPU_ENABLE_SSE if (cpu_fxsr) - ((union savefpu *)&mcp->mc_fpstate)->sv_xmm.sv_env. - en_mxcsr &= cpu_mxcsr_mask; + fpstate->sv_xmm.sv_env.en_mxcsr &= cpu_mxcsr_mask; #endif - npxsetregs(td, (union savefpu *)&mcp->mc_fpstate); + error = npxsetregs(td, fpstate, xfpustate, xfpustate_len); +#else + error = EINVAL; #endif } else return (EINVAL); - return (0); + return (error); } static void Modified: stable/10/sys/i386/i386/mp_machdep.c ============================================================================== --- stable/10/sys/i386/i386/mp_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/mp_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -751,7 +751,7 @@ init_secondary(void) initializecpu(); /* set up FPU state on the AP */ - npxinit(); + npxinit(false); if (cpu_ops.cpu_init) cpu_ops.cpu_init(); @@ -1514,11 +1514,11 @@ cpususpend_handler(void) cpu = PCPU_GET(cpuid); if (savectx(&susppcbs[cpu]->sp_pcb)) { - npxsuspend(&susppcbs[cpu]->sp_fpususpend); + npxsuspend(susppcbs[cpu]->sp_fpususpend); wbinvd(); CPU_SET_ATOMIC(cpu, &suspended_cpus); } else { - npxresume(&susppcbs[cpu]->sp_fpususpend); + npxresume(susppcbs[cpu]->sp_fpususpend); pmap_init_pat(); initializecpu(); PCPU_SET(switchtime, 0); Modified: stable/10/sys/i386/i386/ptrace_machdep.c ============================================================================== --- stable/10/sys/i386/i386/ptrace_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/ptrace_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -41,6 +42,47 @@ __FBSDID("$FreeBSD$"); #define CPU_ENABLE_SSE #endif +#ifdef CPU_ENABLE_SSE +static int +cpu_ptrace_xstate(struct thread *td, int req, void *addr, int data) +{ + char *savefpu; + int error; + + if (!use_xsave) + return (EOPNOTSUPP); + + switch (req) { + case PT_GETXSTATE: + npxgetregs(td); + savefpu = (char *)(get_pcb_user_save_td(td) + 1); + error = copyout(savefpu, addr, + cpu_max_ext_state_size - sizeof(union savefpu)); + break; + + case PT_SETXSTATE: + if (data > cpu_max_ext_state_size - sizeof(union savefpu)) { + error = EINVAL; + break; + } + savefpu = malloc(data, M_TEMP, M_WAITOK); + error = copyin(addr, savefpu, data); + if (error == 0) { + npxgetregs(td); + error = npxsetxstate(td, savefpu, data); + } + free(savefpu, M_TEMP); + break; + + default: + error = EINVAL; + break; + } + + return (error); +} +#endif + int cpu_ptrace(struct thread *td, int req, void *addr, int data) { @@ -51,7 +93,7 @@ cpu_ptrace(struct thread *td, int req, v if (!cpu_fxsr) return (EINVAL); - fpstate = &td->td_pcb->pcb_user_save.sv_xmm; + fpstate = &get_pcb_user_save_td(td)->sv_xmm; switch (req) { case PT_GETXMMREGS: npxgetregs(td); @@ -64,6 +106,11 @@ cpu_ptrace(struct thread *td, int req, v fpstate->sv_env.en_mxcsr &= cpu_mxcsr_mask; break; + case PT_GETXSTATE: + case PT_SETXSTATE: + error = cpu_ptrace_xstate(td, req, addr, data); + break; + default: return (EINVAL); } Modified: stable/10/sys/i386/i386/sys_machdep.c ============================================================================== --- stable/10/sys/i386/i386/sys_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/sys_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -105,6 +105,7 @@ sysarch(td, uap) union { struct i386_ldt_args largs; struct i386_ioperm_args iargs; + struct i386_get_xfpustate xfpu; } kargs; uint32_t base; struct segment_descriptor sd, *sdp; @@ -126,6 +127,7 @@ sysarch(td, uap) case I386_SET_FSBASE: case I386_GET_GSBASE: case I386_SET_GSBASE: + case I386_GET_XFPUSTATE: break; case I386_SET_IOPERM: @@ -154,6 +156,11 @@ sysarch(td, uap) if (kargs.largs.num > MAX_LD || kargs.largs.num <= 0) return (EINVAL); break; + case I386_GET_XFPUSTATE: + if ((error = copyin(uap->parms, &kargs.xfpu, + sizeof(struct i386_get_xfpustate))) != 0) + return (error); + break; default: break; } @@ -270,6 +277,14 @@ sysarch(td, uap) load_gs(GSEL(GUGS_SEL, SEL_UPL)); } break; + case I386_GET_XFPUSTATE: + if (kargs.xfpu.len > cpu_max_ext_state_size - + sizeof(union savefpu)) + return (EINVAL); + npxgetregs(td); + error = copyout((char *)(get_pcb_user_save_td(td) + 1), + kargs.xfpu.addr, kargs.xfpu.len); + break; default: error = EINVAL; break; Modified: stable/10/sys/i386/i386/trap.c ============================================================================== --- stable/10/sys/i386/i386/trap.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/trap.c Mon Dec 22 21:32:39 2014 (r276084) @@ -1160,7 +1160,7 @@ syscall(struct trapframe *frame) KASSERT(PCB_USER_FPU(td->td_pcb), ("System call %s returning with kernel FPU ctx leaked", syscallname(td->td_proc, sa.code))); - KASSERT(td->td_pcb->pcb_save == &td->td_pcb->pcb_user_save, + KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td), ("System call %s returning with mangled pcb_save", syscallname(td->td_proc, sa.code))); Modified: stable/10/sys/i386/i386/vm_machdep.c ============================================================================== --- stable/10/sys/i386/i386/vm_machdep.c Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/i386/vm_machdep.c Mon Dec 22 21:32:39 2014 (r276084) @@ -106,6 +106,10 @@ __FBSDID("$FreeBSD$"); #define NSFBUFS (512 + maxusers * 16) #endif +#if !defined(CPU_DISABLE_SSE) && defined(I686_CPU) +#define CPU_ENABLE_SSE +#endif + _Static_assert(OFFSETOF_CURTHREAD == offsetof(struct pcpu, pc_curthread), "OFFSETOF_CURTHREAD does not correspond with offset of pc_curthread."); _Static_assert(OFFSETOF_CURPCB == offsetof(struct pcpu, pc_curpcb), @@ -150,6 +154,54 @@ static u_int sf_buf_alloc_want; */ static struct mtx sf_buf_lock; +union savefpu * +get_pcb_user_save_td(struct thread *td) +{ + vm_offset_t p; + p = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - + cpu_max_ext_state_size; + KASSERT((p % 64) == 0, ("Unaligned pcb_user_save area")); + return ((union savefpu *)p); +} + +union savefpu * +get_pcb_user_save_pcb(struct pcb *pcb) +{ + vm_offset_t p; + + p = (vm_offset_t)(pcb + 1); + return ((union savefpu *)p); +} + +struct pcb * +get_pcb_td(struct thread *td) +{ + vm_offset_t p; + + p = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - + cpu_max_ext_state_size - sizeof(struct pcb); + return ((struct pcb *)p); +} + +void * +alloc_fpusave(int flags) +{ + void *res; +#ifdef CPU_ENABLE_SSE + struct savefpu_ymm *sf; +#endif + + res = malloc(cpu_max_ext_state_size, M_DEVBUF, flags); +#ifdef CPU_ENABLE_SSE + if (use_xsave) { + sf = (struct savefpu_ymm *)res; + bzero(&sf->sv_xstate.sx_hd, sizeof(sf->sv_xstate.sx_hd)); + sf->sv_xstate.sx_hd.xstate_bv = xsave_mask; + } +#endif + return (res); +} + /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child @@ -199,15 +251,16 @@ cpu_fork(td1, p2, td2, flags) #endif /* Point the pcb to the top of the stack */ - pcb2 = (struct pcb *)(td2->td_kstack + - td2->td_kstack_pages * PAGE_SIZE) - 1; + pcb2 = get_pcb_td(td2); td2->td_pcb = pcb2; /* Copy td1's pcb */ bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); /* Properly initialize pcb_save */ - pcb2->pcb_save = &pcb2->pcb_user_save; + pcb2->pcb_save = get_pcb_user_save_pcb(pcb2); + bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2), + cpu_max_ext_state_size); /* Point mdproc and then copy over td1's contents */ mdp2 = &p2->p_md; @@ -384,12 +437,22 @@ cpu_thread_swapout(struct thread *td) void cpu_thread_alloc(struct thread *td) { + struct pcb *pcb; +#ifdef CPU_ENABLE_SSE + struct xstate_hdr *xhdr; +#endif - td->td_pcb = (struct pcb *)(td->td_kstack + - td->td_kstack_pages * PAGE_SIZE) - 1; - td->td_frame = (struct trapframe *)((caddr_t)td->td_pcb - 16) - 1; - td->td_pcb->pcb_ext = NULL; - td->td_pcb->pcb_save = &td->td_pcb->pcb_user_save; + td->td_pcb = pcb = get_pcb_td(td); + td->td_frame = (struct trapframe *)((caddr_t)pcb - 16) - 1; + pcb->pcb_ext = NULL; + pcb->pcb_save = get_pcb_user_save_pcb(pcb); +#ifdef CPU_ENABLE_SSE + if (use_xsave) { + xhdr = (struct xstate_hdr *)(pcb->pcb_save + 1); + bzero(xhdr, sizeof(*xhdr)); + xhdr->xstate_bv = xsave_mask; + } +#endif } void @@ -457,7 +520,9 @@ cpu_set_upcall(struct thread *td, struct bcopy(td0->td_pcb, pcb2, sizeof(*pcb2)); pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE | PCB_KERNNPX); - pcb2->pcb_save = &pcb2->pcb_user_save; + pcb2->pcb_save = get_pcb_user_save_pcb(pcb2); + bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save, + cpu_max_ext_state_size); /* * Create a new fresh stack for the new thread. Modified: stable/10/sys/i386/include/cpufunc.h ============================================================================== --- stable/10/sys/i386/include/cpufunc.h Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/include/cpufunc.h Mon Dec 22 21:32:39 2014 (r276084) @@ -441,6 +441,25 @@ rcr4(void) return (data); } +static __inline uint64_t +rxcr(u_int reg) +{ + u_int low, high; + + __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); + return (low | ((uint64_t)high << 32)); +} + +static __inline void +load_xcr(u_int reg, uint64_t val) +{ + u_int low, high; + + low = val; + high = val >> 32; + __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); +} + /* * Global TLB flush (except for thise for pages marked PG_G) */ Modified: stable/10/sys/i386/include/md_var.h ============================================================================== --- stable/10/sys/i386/include/md_var.h Mon Dec 22 21:26:49 2014 (r276083) +++ stable/10/sys/i386/include/md_var.h Mon Dec 22 21:32:39 2014 (r276084) @@ -52,6 +52,7 @@ extern u_int cpu_stdext_feature; extern u_int cpu_fxsr; extern u_int cpu_high; extern u_int cpu_id; +extern u_int cpu_max_ext_state_size; extern u_int cpu_mxcsr_mask; extern u_int cpu_procinfo; extern u_int cpu_procinfo2; @@ -78,14 +79,19 @@ extern int vm_page_dump_size; extern int workaround_erratum383; extern int _udatasel; extern int _ucodesel; +extern int use_xsave; +extern uint64_t xsave_mask; typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); +struct pcb; +union savefpu; struct thread; struct reg; struct fpreg; struct dbreg; struct dumperinfo; +void *alloc_fpusave(int flags); void bcopyb(const void *from, void *to, size_t len); void busdma_swi(void); void cpu_setregs(void); @@ -116,5 +122,8 @@ void printcpuinfo(void); void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int selec); int user_dbreg_trap(void); void minidumpsys(struct dumperinfo *); +union savefpu *get_pcb_user_save_td(struct thread *td); +union savefpu *get_pcb_user_save_pcb(struct pcb *pcb); +struct pcb *get_pcb_td(struct thread *td); #endif /* !_MACHINE_MD_VAR_H_ */ Modified: stable/10/sys/i386/include/npx.h ============================================================================== --- stable/10/sys/i386/include/npx.h Mon Dec 22 21:26:49 2014 (r276083) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***