Date: Tue, 6 Aug 2019 16:53:25 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r350639 - in head/sys/amd64: amd64 include Message-ID: <201908061653.x76GrP7k031353@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Tue Aug 6 16:53:25 2019 New Revision: 350639 URL: https://svnweb.freebsd.org/changeset/base/350639 Log: amd64: prevents speculations over swapgs reload of %gs base. Such speculations could use user-controlled %gs base, esp. since FreeBSD supports WRGSBASE instructions. Place LFENCEs on entry for each basic block after the test for previous kernel/user mode on the kernel entry, which prevents the speculation. Code accesses %gs-based PCPU before any serialization instructions are executed, like %cr3 reload for KPTI. With pti disabled, on haswell i7-4770S machine, "syscall_timings getppid" shows when no lfence is added to syscall path: test loop time iterations periteration getppid 0 1.040918865 4643611 0.000000224 getppid 1 1.004985962 4481816 0.000000224 getppid 2 1.005196483 4482363 0.000000224 with lfence: getppid 0 1.043701091 4554779 0.000000229 getppid 1 1.016930328 4438094 0.000000229 getppid 2 1.023223117 4466640 0.000000229 and ministat reports 'No difference proven at 95.0% confidence.' Security: CVE-2019-1125 Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/amd64/amd64/exception.S head/sys/amd64/include/asmacros.h Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Tue Aug 6 16:12:43 2019 (r350638) +++ head/sys/amd64/amd64/exception.S Tue Aug 6 16:53:25 2019 (r350639) @@ -129,6 +129,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_noen_k swapgs + lfence jmp alltraps_noen_u .endm @@ -163,6 +164,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_k swapgs + lfence jmp alltraps_u .endm @@ -198,6 +200,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_k swapgs + lfence jmp alltraps_u .endm @@ -227,6 +230,7 @@ alltraps_u: .globl alltraps_k .type alltraps_k,@function alltraps_k: + lfence movq %rdi,TF_RDI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) @@ -304,6 +308,7 @@ alltraps_noen_u: .globl alltraps_noen_k .type alltraps_noen_k,@function alltraps_noen_k: + lfence movq %rdi,TF_RDI(%rsp) alltraps_noen_save_segs: SAVE_SEGS @@ -343,7 +348,7 @@ IDTVEC(dblfault) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs -1: +1: lfence movq PCPU(KCR3),%rax cmpq $~0,%rax je 2f @@ -358,6 +363,7 @@ IDTVEC(page_pti) testb $SEL_RPL_MASK,PTI_CS-PTI_ERR(%rsp) jz page_k swapgs + lfence pushq %rax movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) @@ -373,6 +379,7 @@ IDTVEC(page) testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) /* Did we come from kernel? */ jnz page_u_swapgs /* already running with kernel GS.base */ page_k: + lfence subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) /* free up GP registers */ movq %rax,TF_RAX(%rsp) @@ -382,6 +389,7 @@ page_k: ALIGN_TEXT page_u_swapgs: swapgs + lfence page_u: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) @@ -419,6 +427,7 @@ page_cr2: .macro PROTF_ENTRY name,trapno \name\()_pti_doreti: swapgs + lfence cmpq $~0,PCPU(UCR3) je 1f pushq %rax @@ -441,9 +450,9 @@ IDTVEC(\name\()_pti) cmpq $doreti_iret,PTI_RIP-2*8(%rsp) je \name\()_pti_doreti testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ - jz X\name + jz X\name /* lfence is not needed until %gs: use */ PTI_UENTRY has_err=1 - swapgs + swapgs /* fence provided by PTI_UENTRY */ IDTVEC(\name) subq $TF_ERR,%rsp movl $\trapno,TF_TRAPNO(%rsp) @@ -476,6 +485,7 @@ prot_addrf: jne 2f rdgsbase %rdx 2: swapgs + lfence movq PCPU(CURPCB),%rdi testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 4f @@ -495,7 +505,8 @@ prot_addrf: jmp alltraps_pushregs_no_rax 5: swapgs -6: movq PCPU(CURPCB),%rdi +6: lfence + movq PCPU(CURPCB),%rdi jmp 4b /* @@ -510,6 +521,7 @@ prot_addrf: SUPERALIGN_TEXT IDTVEC(fast_syscall_pti) swapgs + lfence movq %rax,PCPU(SCRATCH_RAX) cmpq $~0,PCPU(UCR3) je fast_syscall_common @@ -519,6 +531,7 @@ IDTVEC(fast_syscall_pti) SUPERALIGN_TEXT IDTVEC(fast_syscall) swapgs + lfence movq %rax,PCPU(SCRATCH_RAX) fast_syscall_common: movq %rsp,PCPU(SCRATCH_RSP) @@ -647,6 +660,7 @@ IDTVEC(dbg) popfq testb $SEL_RPL_MASK,TF_CS(%rsp) jnz dbg_fromuserspace + lfence /* * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. @@ -702,6 +716,7 @@ dbg_fromuserspace: * in trap(). */ swapgs + lfence movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f @@ -787,6 +802,7 @@ IDTVEC(nmi) * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. */ + lfence movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 @@ -812,6 +828,7 @@ IDTVEC(nmi) nmi_fromuserspace: incl %ebx swapgs + lfence movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax Modified: head/sys/amd64/include/asmacros.h ============================================================================== --- head/sys/amd64/include/asmacros.h Tue Aug 6 16:12:43 2019 (r350638) +++ head/sys/amd64/include/asmacros.h Tue Aug 6 16:53:25 2019 (r350639) @@ -196,6 +196,7 @@ .macro PTI_UENTRY has_err swapgs + lfence cmpq $~0,PCPU(UCR3) je 1f pushq %rax @@ -236,6 +237,7 @@ X\vec_name: jz .L\vec_name\()_u /* Yes, dont swapgs again */ swapgs .L\vec_name\()_u: + lfence subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201908061653.x76GrP7k031353>