From owner-svn-src-stable-12@freebsd.org Sun Nov 18 09:49:26 2018 Return-Path: Delivered-To: svn-src-stable-12@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5112011327C6; Sun, 18 Nov 2018 09:49:26 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id EA34E74519; Sun, 18 Nov 2018 09:49:25 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id CB32111A80; Sun, 18 Nov 2018 09:49:25 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id wAI9nPUL076104; Sun, 18 Nov 2018 09:49:25 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id wAI9nOpa076097; Sun, 18 Nov 2018 09:49:24 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201811180949.wAI9nOpa076097@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Sun, 18 Nov 2018 09:49:24 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r340552 - in stable/12/sys: amd64/amd64 amd64/ia32 amd64/include dev/cpuctl X-SVN-Group: stable-12 X-SVN-Commit-Author: kib X-SVN-Commit-Paths: in stable/12/sys: amd64/amd64 amd64/ia32 amd64/include dev/cpuctl X-SVN-Commit-Revision: 340552 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Rspamd-Queue-Id: EA34E74519 X-Spamd-Result: default: False [-0.05 / 15.00]; local_wl_from(0.00)[FreeBSD.org]; NEURAL_HAM_SHORT(-0.05)[-0.049,0]; ASN(0.00)[asn:11403, ipnet:2610:1c1:1::/48, country:US] X-Rspamd-Server: mx1.freebsd.org X-BeenThere: svn-src-stable-12@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for only the 12-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 18 Nov 2018 09:49:26 -0000 Author: kib Date: Sun Nov 18 09:49:24 2018 New Revision: 340552 URL: https://svnweb.freebsd.org/changeset/base/340552 Log: MFC r339507: amd64: flush L1 data cache on syscall return with an error. Modified: stable/12/sys/amd64/amd64/initcpu.c stable/12/sys/amd64/amd64/machdep.c stable/12/sys/amd64/amd64/support.S stable/12/sys/amd64/amd64/trap.c stable/12/sys/amd64/ia32/ia32_syscall.c stable/12/sys/amd64/include/md_var.h stable/12/sys/dev/cpuctl/cpuctl.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/amd64/amd64/initcpu.c ============================================================================== --- stable/12/sys/amd64/amd64/initcpu.c Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/amd64/initcpu.c Sun Nov 18 09:49:24 2018 (r340552) @@ -253,6 +253,7 @@ initializecpu(void) } hw_ibrs_recalculate(); hw_ssb_recalculate(false); + amd64_syscall_ret_flush_l1d_recalc(); switch (cpu_vendor_id) { case CPU_VENDOR_AMD: init_amd(); Modified: stable/12/sys/amd64/amd64/machdep.c ============================================================================== --- stable/12/sys/amd64/amd64/machdep.c Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/amd64/machdep.c Sun Nov 18 09:49:24 2018 (r340552) @@ -1728,6 +1728,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) != NULL) vty_set_preferred(VTY_VT); + TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); + TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); + TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush", + &syscall_ret_l1d_flush_mode); + finishidentcpu(); /* Final stage of CPU initialization */ initializecpu(); /* Initialize CPU registers */ @@ -1875,9 +1880,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) x86_init_fdt(); #endif thread0.td_critnest = 0; - - TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); - TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); TSEXIT(); Modified: stable/12/sys/amd64/amd64/support.S ============================================================================== --- stable/12/sys/amd64/amd64/support.S Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/amd64/support.S Sun Nov 18 09:49:24 2018 (r340552) @@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw) ret #undef L1D_FLUSH_SIZE END(flush_l1d_sw) + +ENTRY(flush_l1d_sw_abi) + pushq %rbx + call flush_l1d_sw + popq %rbx + ret +END(flush_l1d_sw_abi) Modified: stable/12/sys/amd64/amd64/trap.c ============================================================================== --- stable/12/sys/amd64/amd64/trap.c Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/amd64/trap.c Sun Nov 18 09:49:24 2018 (r340552) @@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td) #include "../../kern/subr_syscall.c" +static void (*syscall_ret_l1d_flush)(void); +int syscall_ret_l1d_flush_mode; + +static void +flush_l1d_hw(void) +{ + + wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D); +} + +static void __inline +amd64_syscall_ret_flush_l1d_inline(int error) +{ + void (*p)(void); + + if (error != 0 && error != EEXIST && error != EAGAIN && + error != EXDEV && error != ENOENT && error != ENOTCONN && + error != EINPROGRESS) { + p = syscall_ret_l1d_flush; + if (p != NULL) + p(); + } +} + +void +amd64_syscall_ret_flush_l1d(int error) +{ + + amd64_syscall_ret_flush_l1d_inline(error); +} + +void +amd64_syscall_ret_flush_l1d_recalc(void) +{ + bool l1d_hw; + + l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0; +again: + switch (syscall_ret_l1d_flush_mode) { + case 0: + syscall_ret_l1d_flush = NULL; + break; + case 1: + syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : + flush_l1d_sw_abi; + break; + case 2: + syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL; + break; + case 3: + syscall_ret_l1d_flush = flush_l1d_sw_abi; + break; + default: + syscall_ret_l1d_flush_mode = 1; + goto again; + } +} + +static int +machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS) +{ + int error, val; + + val = syscall_ret_l1d_flush_mode; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + syscall_ret_l1d_flush_mode = val; + amd64_syscall_ret_flush_l1d_recalc(); + return (0); +} +SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT | + CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0, + machdep_syscall_ret_flush_l1d, "I", + "Flush L1D on syscall return with error (0 - off, 1 - on, " + "2 - use hw only, 3 - use sw only"); + + /* * System call handler for native binaries. The trap frame is already * set up by the assembler trampoline and a pointer to it is saved in @@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced) */ if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS)) set_pcb_flags(td->td_pcb, PCB_FULL_IRET); + + amd64_syscall_ret_flush_l1d_inline(error); } Modified: stable/12/sys/amd64/ia32/ia32_syscall.c ============================================================================== --- stable/12/sys/amd64/ia32/ia32_syscall.c Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/ia32/ia32_syscall.c Sun Nov 18 09:49:24 2018 (r340552) @@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame) } syscallret(td, error); + amd64_syscall_ret_flush_l1d(error); } static void Modified: stable/12/sys/amd64/include/md_var.h ============================================================================== --- stable/12/sys/amd64/include/md_var.h Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/amd64/include/md_var.h Sun Nov 18 09:49:24 2018 (r340552) @@ -41,6 +41,7 @@ extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; extern int nmi_flush_l1d_sw; +extern int syscall_ret_l1d_flush_mode; extern vm_paddr_t intel_graphics_stolen_base; extern vm_paddr_t intel_graphics_stolen_size; @@ -58,8 +59,11 @@ void amd64_conf_fast_syscall(void); void amd64_db_resume_dbreg(void); void amd64_lower_shared_page(struct sysentvec *); void amd64_syscall(struct thread *td, int traced); +void amd64_syscall_ret_flush_l1d(int error); +void amd64_syscall_ret_flush_l1d_recalc(void); void doreti_iret(void) __asm(__STRING(doreti_iret)); void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault)); +void flush_l1d_sw_abi(void); void ld_ds(void) __asm(__STRING(ld_ds)); void ld_es(void) __asm(__STRING(ld_es)); void ld_fs(void) __asm(__STRING(ld_fs)); Modified: stable/12/sys/dev/cpuctl/cpuctl.c ============================================================================== --- stable/12/sys/dev/cpuctl/cpuctl.c Sun Nov 18 09:48:51 2018 (r340551) +++ stable/12/sys/dev/cpuctl/cpuctl.c Sun Nov 18 09:49:24 2018 (r340552) @@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td hw_ibrs_recalculate(); restore_cpu(oldcpu, is_bound, td); hw_ssb_recalculate(true); +#ifdef __amd64__ + amd64_syscall_ret_flush_l1d_recalc(); +#endif printcpuinfo(); return (0); }