Date: Tue, 14 May 2019 23:19:08 +0000 (UTC) From: Gordon Tetlow <gordon@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-releng@freebsd.org Subject: svn commit: r347594 - in releng/12.0/sys: amd64/amd64 amd64/include dev/cpuctl i386/i386 i386/include x86/include x86/x86 Message-ID: <201905142319.x4ENJ82G058878@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: gordon Date: Tue May 14 23:19:08 2019 New Revision: 347594 URL: https://svnweb.freebsd.org/changeset/base/347594 Log: Mitigations for Microarchitectural Data Sampling. Approved by: so Security: FreeBSD-SA-19:07.mds Security: CVE-2018-12126 Security: CVE-2018-12127 Security: CVE-2018-12130 Security: CVE-2019-11091 Modified: releng/12.0/sys/amd64/amd64/exception.S releng/12.0/sys/amd64/amd64/genassym.c releng/12.0/sys/amd64/amd64/initcpu.c releng/12.0/sys/amd64/amd64/machdep.c releng/12.0/sys/amd64/amd64/support.S releng/12.0/sys/amd64/include/pcpu.h releng/12.0/sys/dev/cpuctl/cpuctl.c releng/12.0/sys/i386/i386/exception.s releng/12.0/sys/i386/i386/genassym.c releng/12.0/sys/i386/i386/initcpu.c releng/12.0/sys/i386/i386/support.s releng/12.0/sys/i386/include/pcpu.h releng/12.0/sys/x86/include/specialreg.h releng/12.0/sys/x86/include/x86_var.h releng/12.0/sys/x86/x86/cpu_machdep.c Modified: releng/12.0/sys/amd64/amd64/exception.S ============================================================================== --- releng/12.0/sys/amd64/amd64/exception.S Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/amd64/exception.S Tue May 14 23:19:08 2019 (r347594) @@ -512,6 +512,7 @@ fast_syscall_common: testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) jne 3f call handle_ibrs_exit + callq *mds_handler /* Restore preserved registers. */ MEXITCOUNT movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ @@ -1157,6 +1158,7 @@ ld_regs: jz 2f /* keep running with kernel GS.base */ cli call handle_ibrs_exit_rs + callq *mds_handler cmpq $~0,PCPU(UCR3) je 1f pushq %rdx Modified: releng/12.0/sys/amd64/amd64/genassym.c ============================================================================== --- releng/12.0/sys/amd64/amd64/genassym.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/amd64/genassym.c Tue May 14 23:19:08 2019 (r347594) @@ -233,6 +233,9 @@ ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ); ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0)); ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); +ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp)); +ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf)); +ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64)); ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL); Modified: releng/12.0/sys/amd64/amd64/initcpu.c ============================================================================== --- releng/12.0/sys/amd64/amd64/initcpu.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/amd64/initcpu.c Tue May 14 23:19:08 2019 (r347594) @@ -253,6 +253,7 @@ initializecpu(void) } hw_ibrs_recalculate(); hw_ssb_recalculate(false); + hw_mds_recalculate(); switch (cpu_vendor_id) { case CPU_VENDOR_AMD: init_amd(); Modified: releng/12.0/sys/amd64/amd64/machdep.c ============================================================================== --- releng/12.0/sys/amd64/amd64/machdep.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/amd64/machdep.c Tue May 14 23:19:08 2019 (r347594) @@ -1878,6 +1878,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); + TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable); TSEXIT(); Modified: releng/12.0/sys/amd64/amd64/support.S ============================================================================== --- releng/12.0/sys/amd64/amd64/support.S Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/amd64/support.S Tue May 14 23:19:08 2019 (r347594) @@ -1,8 +1,13 @@ /*- + * Copyright (c) 2018-2019 The FreeBSD Foundation * Copyright (c) 2003 Peter Wemm. * Copyright (c) 1993 The Regents of the University of California. * All rights reserved. * + * Portions of this software were developed by + * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from + * the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -1586,3 +1591,246 @@ ENTRY(flush_l1d_sw) ret #undef L1D_FLUSH_SIZE END(flush_l1d_sw) + +ENTRY(mds_handler_void) + retq +END(mds_handler_void) + +ENTRY(mds_handler_verw) + subq $8, %rsp + movw %ds, (%rsp) + verw (%rsp) + addq $8, %rsp + retq +END(mds_handler_verw) + +ENTRY(mds_handler_ivb) + pushq %rax + pushq %rdx + pushq %rcx + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%rdx), %xmm0 + orpd (%rdx), %xmm0 + mfence + movl $40, %ecx + addq $16, %rdx +2: movntdq %xmm0, (%rdx) + addq $16, %rdx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_ivb) + +ENTRY(mds_handler_bdw) + pushq %rax + pushq %rbx + pushq %rcx + pushq %rdi + pushq %rsi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rbx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movq %rbx, %rdi + movq %rbx, %rsi + movl $40, %ecx +2: movntdq %xmm0, (%rbx) + addq $16, %rbx + decl %ecx + jnz 2b + mfence + movl $1536, %ecx + rep; movsb + lfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rsi + popq %rdi + popq %rcx + popq %rbx + popq %rax + retq +END(mds_handler_bdw) + +ENTRY(mds_handler_skl_sse) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%rdx), %xmm0 + orpd (%rdx), %xmm0 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + movdqa PCPU(MDS_TMP), %xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_sse) + +ENTRY(mds_handler_skl_avx) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx + vmovdqa %ymm0, PCPU(MDS_TMP) + vpxor %ymm0, %ymm0, %ymm0 + + lfence + vorpd (%rdx), %ymm0, %ymm0 + vorpd (%rdx), %ymm0, %ymm0 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa PCPU(MDS_TMP), %ymm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_avx) + +ENTRY(mds_handler_skl_avx512) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx +/* vmovdqa64 %zmm0, PCPU(MDS_TMP) */ + .byte 0x65, 0x62, 0xf1, 0xfd, 0x48, 0x7f, 0x04, 0x25 + .long PC_MDS_TMP +/* vpxor %zmm0, %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0xef, 0xc0 + + lfence +/* vorpd (%rdx), %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02 +/* vorpd (%rdx), %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + +/* vmovdqa64 PCPU(MDS_TMP), %zmm0 */ + .byte 0x65, 0x62, 0xf1, 0xfd, 0x48, 0x6f, 0x04, 0x25 + .long PC_MDS_TMP + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_avx512) + +ENTRY(mds_handler_silvermont) + pushq %rax + pushq %rdx + pushq %rcx + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl $16, %ecx +2: movntdq %xmm0, (%rdx) + addq $16, %rdx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_silvermont) Modified: releng/12.0/sys/amd64/include/pcpu.h ============================================================================== --- releng/12.0/sys/amd64/include/pcpu.h Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/amd64/include/pcpu.h Tue May 14 23:19:08 2019 (r347594) @@ -76,7 +76,11 @@ uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ - char __pad[3288] /* pad to UMA_PCPU_ALLOC_SIZE */ + void *pc_mds_buf; \ + void *pc_mds_buf64; \ + uint32_t pc_pad[2]; \ + uint8_t pc_mds_tmp[64]; \ + char __pad[3176] /* pad to UMA_PCPU_ALLOC_SIZE */ #define PC_DBREG_CMD_NONE 0 #define PC_DBREG_CMD_LOAD 1 Modified: releng/12.0/sys/dev/cpuctl/cpuctl.c ============================================================================== --- releng/12.0/sys/dev/cpuctl/cpuctl.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/dev/cpuctl/cpuctl.c Tue May 14 23:19:08 2019 (r347594) @@ -521,6 +521,7 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td hw_ibrs_recalculate(); restore_cpu(oldcpu, is_bound, td); hw_ssb_recalculate(true); + hw_mds_recalculate(); printcpuinfo(); return (0); } Modified: releng/12.0/sys/i386/i386/exception.s ============================================================================== --- releng/12.0/sys/i386/i386/exception.s Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/i386/i386/exception.s Tue May 14 23:19:08 2019 (r347594) @@ -518,6 +518,8 @@ doreti_exit: 2: movl $handle_ibrs_exit,%eax pushl %ecx /* preserve enough call-used regs */ call *%eax + movl mds_handler,%eax + call *%eax popl %ecx movl %esp, %esi movl PCPU(TRAMPSTK), %edx Modified: releng/12.0/sys/i386/i386/genassym.c ============================================================================== --- releng/12.0/sys/i386/i386/genassym.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/i386/i386/genassym.c Tue May 14 23:19:08 2019 (r347594) @@ -222,6 +222,9 @@ ASSYM(PC_KESP0, offsetof(struct pcpu, pc_kesp0)); ASSYM(PC_TRAMPSTK, offsetof(struct pcpu, pc_trampstk)); ASSYM(PC_COPYOUT_BUF, offsetof(struct pcpu, pc_copyout_buf)); ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); +ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp)); +ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf)); +ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64)); #ifdef DEV_APIC ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); Modified: releng/12.0/sys/i386/i386/initcpu.c ============================================================================== --- releng/12.0/sys/i386/i386/initcpu.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/i386/i386/initcpu.c Tue May 14 23:19:08 2019 (r347594) @@ -745,6 +745,7 @@ initializecpu(void) cpu_fxsr = hw_instruction_sse = 1; } #if defined(PAE) || defined(PAE_TABLES) + hw_mds_recalculate(); if ((amd_feature & AMDID_NX) != 0) { uint64_t msr; Modified: releng/12.0/sys/i386/i386/support.s ============================================================================== --- releng/12.0/sys/i386/i386/support.s Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/i386/i386/support.s Tue May 14 23:19:08 2019 (r347594) @@ -472,3 +472,194 @@ ENTRY(handle_ibrs_exit) movb $0,PCPU(IBPB_SET) 1: ret END(handle_ibrs_exit) + +ENTRY(mds_handler_void) + ret +END(mds_handler_void) + +ENTRY(mds_handler_verw) + subl $4, %esp + movw %ds, (%esp) + verw (%esp) + addl $4, %esp + ret +END(mds_handler_verw) + +ENTRY(mds_handler_ivb) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%edx), %xmm0 + orpd (%edx), %xmm0 + mfence + movl $40, %ecx + addl $16, %edx +2: movntdq %xmm0, (%edx) + addl $16, %edx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_ivb) + +ENTRY(mds_handler_bdw) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %ebx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl %ebx, %edi + movl %ebx, %esi + movl $40, %ecx +2: movntdq %xmm0, (%ebx) + addl $16, %ebx + decl %ecx + jnz 2b + mfence + movl $1536, %ecx + rep; movsb + lfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_bdw) + +ENTRY(mds_handler_skl_sse) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%edx), %xmm0 + orpd (%edx), %xmm0 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + movdqa PCPU(MDS_TMP), %xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_sse) + +ENTRY(mds_handler_skl_avx) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx + vmovdqa %ymm0, PCPU(MDS_TMP) + vpxor %ymm0, %ymm0, %ymm0 + + lfence + vorpd (%edx), %ymm0, %ymm0 + vorpd (%edx), %ymm0, %ymm0 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa PCPU(MDS_TMP), %ymm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_avx) + +ENTRY(mds_handler_skl_avx512) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx +/* vmovdqa64 %zmm0, PCPU(MDS_TMP) */ + .byte 0x64, 0x62, 0xf1, 0xfd, 0x48, 0x7f, 0x05 + .long PC_MDS_TMP +/* vpxor %zmm0, %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0xef, 0xc0 + + lfence +/* vorpd (%edx), %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02 +/* vorpd (%edx), %zmm0, %zmm0 */ + .byte 0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + +/* vmovdqa64 PCPU(MDS_TMP), %zmm0 */ + .byte 0x64, 0x62, 0xf1, 0xfd, 0x48, 0x6f, 0x05 + .long PC_MDS_TMP + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_avx512) + +ENTRY(mds_handler_silvermont) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl $16, %ecx +2: movntdq %xmm0, (%edx) + addl $16, %edx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_silvermont) Modified: releng/12.0/sys/i386/include/pcpu.h ============================================================================== --- releng/12.0/sys/i386/include/pcpu.h Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/i386/include/pcpu.h Tue May 14 23:19:08 2019 (r347594) @@ -77,10 +77,14 @@ struct sx pc_copyout_slock; \ char *pc_copyout_buf; \ vm_offset_t pc_pmap_eh_va; \ - caddr_t pc_pmap_eh_ptep; \ + caddr_t pc_pmap_eh_ptep; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ - char __pad[3610] + void *pc_mds_buf; \ + void *pc_mds_buf64; \ + uint32_t pc_pad[4]; \ + uint8_t pc_mds_tmp[64]; \ + char __pad[3522] #ifdef _KERNEL Modified: releng/12.0/sys/x86/include/specialreg.h ============================================================================== --- releng/12.0/sys/x86/include/specialreg.h Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/x86/include/specialreg.h Tue May 14 23:19:08 2019 (r347594) @@ -425,6 +425,7 @@ /* * CPUID instruction 7 Structured Extended Features, leaf 0 edx info */ +#define CPUID_STDEXT3_MD_CLEAR 0x00000400 #define CPUID_STDEXT3_IBPB 0x04000000 #define CPUID_STDEXT3_STIBP 0x08000000 #define CPUID_STDEXT3_L1D_FLUSH 0x10000000 @@ -437,6 +438,7 @@ #define IA32_ARCH_CAP_RSBA 0x00000004 #define IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY 0x00000008 #define IA32_ARCH_CAP_SSB_NO 0x00000010 +#define IA32_ARCH_CAP_MDS_NO 0x00000020 /* * CPUID manufacturers identifiers Modified: releng/12.0/sys/x86/include/x86_var.h ============================================================================== --- releng/12.0/sys/x86/include/x86_var.h Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/x86/include/x86_var.h Tue May 14 23:19:08 2019 (r347594) @@ -85,6 +85,7 @@ extern uint64_t xsave_mask; extern u_int max_apic_id; extern int pti; extern int hw_ibrs_active; +extern int hw_mds_disable; extern int hw_ssb_active; struct pcb; @@ -139,6 +140,7 @@ int isa_nmi(int cd); void handle_ibrs_entry(void); void handle_ibrs_exit(void); void hw_ibrs_recalculate(void); +void hw_mds_recalculate(void); void hw_ssb_recalculate(bool all_cpus); void nmi_call_kdb(u_int cpu, u_int type, struct trapframe *frame); void nmi_call_kdb_smp(u_int type, struct trapframe *frame); Modified: releng/12.0/sys/x86/x86/cpu_machdep.c ============================================================================== --- releng/12.0/sys/x86/x86/cpu_machdep.c Tue May 14 23:12:22 2019 (r347593) +++ releng/12.0/sys/x86/x86/cpu_machdep.c Tue May 14 23:19:08 2019 (r347594) @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/bus.h> #include <sys/cpu.h> +#include <sys/domainset.h> #include <sys/kdb.h> #include <sys/kernel.h> #include <sys/ktr.h> @@ -914,6 +915,203 @@ SYSCTL_PROC(_hw, OID_AUTO, spec_store_bypass_disable, CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0, hw_ssb_disable_handler, "I", "Speculative Store Bypass Disable (0 - off, 1 - on, 2 - auto"); + +int hw_mds_disable; + +/* + * Handler for Microarchitectural Data Sampling issues. Really not a + * pointer to C function: on amd64 the code must not change any CPU + * architectural state except possibly %rflags. Also, it is always + * called with interrupts disabled. + */ +void (*mds_handler)(void); +void mds_handler_void(void); +void mds_handler_verw(void); +void mds_handler_ivb(void); +void mds_handler_bdw(void); +void mds_handler_skl_sse(void); +void mds_handler_skl_avx(void); +void mds_handler_skl_avx512(void); +void mds_handler_silvermont(void); + +static int +sysctl_hw_mds_disable_state_handler(SYSCTL_HANDLER_ARGS) +{ + const char *state; + + if (mds_handler == mds_handler_void) + state = "inactive"; + else if (mds_handler == mds_handler_verw) + state = "VERW"; + else if (mds_handler == mds_handler_ivb) + state = "software IvyBridge"; + else if (mds_handler == mds_handler_bdw) + state = "software Broadwell"; + else if (mds_handler == mds_handler_skl_sse) + state = "software Skylake SSE"; + else if (mds_handler == mds_handler_skl_avx) + state = "software Skylake AVX"; + else if (mds_handler == mds_handler_skl_avx512) + state = "software Skylake AVX512"; + else if (mds_handler == mds_handler_silvermont) + state = "software Silvermont"; + else + state = "unknown"; + return (SYSCTL_OUT(req, state, strlen(state))); +} + +SYSCTL_PROC(_hw, OID_AUTO, mds_disable_state, + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, + sysctl_hw_mds_disable_state_handler, "A", + "Microarchitectural Data Sampling Mitigation state"); + +_Static_assert(__offsetof(struct pcpu, pc_mds_tmp) % 64 == 0, "MDS AVX512"); + +void +hw_mds_recalculate(void) +{ + struct pcpu *pc; + vm_offset_t b64; + u_long xcr0; + int i; + + /* + * Allow user to force VERW variant even if MD_CLEAR is not + * reported. For instance, hypervisor might unknowingly + * filter the cap out. + * For the similar reasons, and for testing, allow to enable + * mitigation even for RDCL_NO or MDS_NO caps. + */ + if (cpu_vendor_id != CPU_VENDOR_INTEL || hw_mds_disable == 0 || + ((cpu_ia32_arch_caps & (IA32_ARCH_CAP_RDCL_NO | + IA32_ARCH_CAP_MDS_NO)) != 0 && hw_mds_disable == 3)) { + mds_handler = mds_handler_void; + } else if (((cpu_stdext_feature3 & CPUID_STDEXT3_MD_CLEAR) != 0 && + hw_mds_disable == 3) || hw_mds_disable == 1) { + mds_handler = mds_handler_verw; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + (CPUID_TO_MODEL(cpu_id) == 0x2e || CPUID_TO_MODEL(cpu_id) == 0x1e || + CPUID_TO_MODEL(cpu_id) == 0x1f || CPUID_TO_MODEL(cpu_id) == 0x1a || + CPUID_TO_MODEL(cpu_id) == 0x2f || CPUID_TO_MODEL(cpu_id) == 0x25 || + CPUID_TO_MODEL(cpu_id) == 0x2c || CPUID_TO_MODEL(cpu_id) == 0x2d || + CPUID_TO_MODEL(cpu_id) == 0x2a || CPUID_TO_MODEL(cpu_id) == 0x3e || + CPUID_TO_MODEL(cpu_id) == 0x3a) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Nehalem, SandyBridge, IvyBridge + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc_domainset(672, M_TEMP, + DOMAINSET_PREF(pc->pc_domain), M_WAITOK); + bzero(pc->pc_mds_buf, 16); + } + } + mds_handler = mds_handler_ivb; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + (CPUID_TO_MODEL(cpu_id) == 0x3f || CPUID_TO_MODEL(cpu_id) == 0x3c || + CPUID_TO_MODEL(cpu_id) == 0x45 || CPUID_TO_MODEL(cpu_id) == 0x46 || + CPUID_TO_MODEL(cpu_id) == 0x56 || CPUID_TO_MODEL(cpu_id) == 0x4f || + CPUID_TO_MODEL(cpu_id) == 0x47 || CPUID_TO_MODEL(cpu_id) == 0x3d) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Haswell, Broadwell + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc_domainset(1536, M_TEMP, + DOMAINSET_PREF(pc->pc_domain), M_WAITOK); + bzero(pc->pc_mds_buf, 16); + } + } + mds_handler = mds_handler_bdw; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + ((CPUID_TO_MODEL(cpu_id) == 0x55 && (cpu_id & + CPUID_STEPPING) <= 5) || + CPUID_TO_MODEL(cpu_id) == 0x4e || CPUID_TO_MODEL(cpu_id) == 0x5e || + (CPUID_TO_MODEL(cpu_id) == 0x8e && (cpu_id & + CPUID_STEPPING) <= 0xb) || + (CPUID_TO_MODEL(cpu_id) == 0x9e && (cpu_id & + CPUID_STEPPING) <= 0xc)) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Skylake, KabyLake, CoffeeLake, WhiskeyLake, + * CascadeLake + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc_domainset(6 * 1024, + M_TEMP, DOMAINSET_PREF(pc->pc_domain), + M_WAITOK); + b64 = (vm_offset_t)malloc_domainset(64 + 63, + M_TEMP, DOMAINSET_PREF(pc->pc_domain), + M_WAITOK); + pc->pc_mds_buf64 = (void *)roundup2(b64, 64); + bzero(pc->pc_mds_buf64, 64); + } + } + xcr0 = rxcr(0); + if ((xcr0 & XFEATURE_ENABLED_ZMM_HI256) != 0 && + (cpu_stdext_feature2 & CPUID_STDEXT_AVX512DQ) != 0) + mds_handler = mds_handler_skl_avx512; + else if ((xcr0 & XFEATURE_ENABLED_AVX) != 0 && + (cpu_feature2 & CPUID2_AVX) != 0) + mds_handler = mds_handler_skl_avx; + else + mds_handler = mds_handler_skl_sse; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + ((CPUID_TO_MODEL(cpu_id) == 0x37 || + CPUID_TO_MODEL(cpu_id) == 0x4a || + CPUID_TO_MODEL(cpu_id) == 0x4c || + CPUID_TO_MODEL(cpu_id) == 0x4d || + CPUID_TO_MODEL(cpu_id) == 0x5a || + CPUID_TO_MODEL(cpu_id) == 0x5d || + CPUID_TO_MODEL(cpu_id) == 0x6e || + CPUID_TO_MODEL(cpu_id) == 0x65 || + CPUID_TO_MODEL(cpu_id) == 0x75 || + CPUID_TO_MODEL(cpu_id) == 0x1c || + CPUID_TO_MODEL(cpu_id) == 0x26 || + CPUID_TO_MODEL(cpu_id) == 0x27 || + CPUID_TO_MODEL(cpu_id) == 0x35 || + CPUID_TO_MODEL(cpu_id) == 0x36 || + CPUID_TO_MODEL(cpu_id) == 0x7a))) { + /* Silvermont, Airmont */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) + pc->pc_mds_buf = malloc(256, M_TEMP, M_WAITOK); + } + mds_handler = mds_handler_silvermont; + } else { + hw_mds_disable = 0; + mds_handler = mds_handler_void; + } +} + +static int +sysctl_mds_disable_handler(SYSCTL_HANDLER_ARGS) +{ + int error, val; + + val = hw_mds_disable; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (val < 0 || val > 3) + return (EINVAL); + hw_mds_disable = val; + hw_mds_recalculate(); + return (0); +} + +SYSCTL_PROC(_hw, OID_AUTO, mds_disable, CTLTYPE_INT | + CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0, + sysctl_mds_disable_handler, "I", + "Microarchitectural Data Sampling Mitigation " + "(0 - off, 1 - on VERW, 2 - on SW, 3 - on AUTO"); /* * Enable and restore kernel text write permissions.
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201905142319.x4ENJ82G058878>