From owner-svn-src-all@freebsd.org Tue Aug 6 16:53:26 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 917E3C56C6; Tue, 6 Aug 2019 16:53:26 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4631123MGLz4CTs; Tue, 6 Aug 2019 16:53:26 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 547D02EF38; Tue, 6 Aug 2019 16:53:26 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x76GrQ38031355; Tue, 6 Aug 2019 16:53:26 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x76GrP7k031353; Tue, 6 Aug 2019 16:53:25 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201908061653.x76GrP7k031353@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Tue, 6 Aug 2019 16:53:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r350639 - in head/sys/amd64: amd64 include X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: in head/sys/amd64: amd64 include X-SVN-Commit-Revision: 350639 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 06 Aug 2019 16:53:26 -0000 Author: kib Date: Tue Aug 6 16:53:25 2019 New Revision: 350639 URL: https://svnweb.freebsd.org/changeset/base/350639 Log: amd64: prevents speculations over swapgs reload of %gs base. Such speculations could use user-controlled %gs base, esp. since FreeBSD supports WRGSBASE instructions. Place LFENCEs on entry for each basic block after the test for previous kernel/user mode on the kernel entry, which prevents the speculation. Code accesses %gs-based PCPU before any serialization instructions are executed, like %cr3 reload for KPTI. With pti disabled, on haswell i7-4770S machine, "syscall_timings getppid" shows when no lfence is added to syscall path: test loop time iterations periteration getppid 0 1.040918865 4643611 0.000000224 getppid 1 1.004985962 4481816 0.000000224 getppid 2 1.005196483 4482363 0.000000224 with lfence: getppid 0 1.043701091 4554779 0.000000229 getppid 1 1.016930328 4438094 0.000000229 getppid 2 1.023223117 4466640 0.000000229 and ministat reports 'No difference proven at 95.0% confidence.' Security: CVE-2019-1125 Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/amd64/amd64/exception.S head/sys/amd64/include/asmacros.h Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Tue Aug 6 16:12:43 2019 (r350638) +++ head/sys/amd64/amd64/exception.S Tue Aug 6 16:53:25 2019 (r350639) @@ -129,6 +129,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_noen_k swapgs + lfence jmp alltraps_noen_u .endm @@ -163,6 +164,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_k swapgs + lfence jmp alltraps_u .endm @@ -198,6 +200,7 @@ X\l: testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_k swapgs + lfence jmp alltraps_u .endm @@ -227,6 +230,7 @@ alltraps_u: .globl alltraps_k .type alltraps_k,@function alltraps_k: + lfence movq %rdi,TF_RDI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) @@ -304,6 +308,7 @@ alltraps_noen_u: .globl alltraps_noen_k .type alltraps_noen_k,@function alltraps_noen_k: + lfence movq %rdi,TF_RDI(%rsp) alltraps_noen_save_segs: SAVE_SEGS @@ -343,7 +348,7 @@ IDTVEC(dblfault) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs -1: +1: lfence movq PCPU(KCR3),%rax cmpq $~0,%rax je 2f @@ -358,6 +363,7 @@ IDTVEC(page_pti) testb $SEL_RPL_MASK,PTI_CS-PTI_ERR(%rsp) jz page_k swapgs + lfence pushq %rax movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) @@ -373,6 +379,7 @@ IDTVEC(page) testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) /* Did we come from kernel? */ jnz page_u_swapgs /* already running with kernel GS.base */ page_k: + lfence subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) /* free up GP registers */ movq %rax,TF_RAX(%rsp) @@ -382,6 +389,7 @@ page_k: ALIGN_TEXT page_u_swapgs: swapgs + lfence page_u: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) @@ -419,6 +427,7 @@ page_cr2: .macro PROTF_ENTRY name,trapno \name\()_pti_doreti: swapgs + lfence cmpq $~0,PCPU(UCR3) je 1f pushq %rax @@ -441,9 +450,9 @@ IDTVEC(\name\()_pti) cmpq $doreti_iret,PTI_RIP-2*8(%rsp) je \name\()_pti_doreti testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ - jz X\name + jz X\name /* lfence is not needed until %gs: use */ PTI_UENTRY has_err=1 - swapgs + swapgs /* fence provided by PTI_UENTRY */ IDTVEC(\name) subq $TF_ERR,%rsp movl $\trapno,TF_TRAPNO(%rsp) @@ -476,6 +485,7 @@ prot_addrf: jne 2f rdgsbase %rdx 2: swapgs + lfence movq PCPU(CURPCB),%rdi testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 4f @@ -495,7 +505,8 @@ prot_addrf: jmp alltraps_pushregs_no_rax 5: swapgs -6: movq PCPU(CURPCB),%rdi +6: lfence + movq PCPU(CURPCB),%rdi jmp 4b /* @@ -510,6 +521,7 @@ prot_addrf: SUPERALIGN_TEXT IDTVEC(fast_syscall_pti) swapgs + lfence movq %rax,PCPU(SCRATCH_RAX) cmpq $~0,PCPU(UCR3) je fast_syscall_common @@ -519,6 +531,7 @@ IDTVEC(fast_syscall_pti) SUPERALIGN_TEXT IDTVEC(fast_syscall) swapgs + lfence movq %rax,PCPU(SCRATCH_RAX) fast_syscall_common: movq %rsp,PCPU(SCRATCH_RSP) @@ -647,6 +660,7 @@ IDTVEC(dbg) popfq testb $SEL_RPL_MASK,TF_CS(%rsp) jnz dbg_fromuserspace + lfence /* * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. @@ -702,6 +716,7 @@ dbg_fromuserspace: * in trap(). */ swapgs + lfence movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f @@ -787,6 +802,7 @@ IDTVEC(nmi) * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. */ + lfence movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 @@ -812,6 +828,7 @@ IDTVEC(nmi) nmi_fromuserspace: incl %ebx swapgs + lfence movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax Modified: head/sys/amd64/include/asmacros.h ============================================================================== --- head/sys/amd64/include/asmacros.h Tue Aug 6 16:12:43 2019 (r350638) +++ head/sys/amd64/include/asmacros.h Tue Aug 6 16:53:25 2019 (r350639) @@ -196,6 +196,7 @@ .macro PTI_UENTRY has_err swapgs + lfence cmpq $~0,PCPU(UCR3) je 1f pushq %rax @@ -236,6 +237,7 @@ X\vec_name: jz .L\vec_name\()_u /* Yes, dont swapgs again */ swapgs .L\vec_name\()_u: + lfence subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp)