Skip site navigation (1)Skip section navigation (2) 
Date:      Sun, 18 Nov 2018 09:49:24 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r340552 - in stable/12/sys: amd64/amd64 amd64/ia32 amd64/include dev/cpuctl
Message-ID:  <201811180949.wAI9nOpa076097@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: kib
Date: Sun Nov 18 09:49:24 2018
New Revision: 340552
URL: https://svnweb.freebsd.org/changeset/base/340552

Log:
  MFC r339507:
  amd64: flush L1 data cache on syscall return with an error.

Modified:
  stable/12/sys/amd64/amd64/initcpu.c
  stable/12/sys/amd64/amd64/machdep.c
  stable/12/sys/amd64/amd64/support.S
  stable/12/sys/amd64/amd64/trap.c
  stable/12/sys/amd64/ia32/ia32_syscall.c
  stable/12/sys/amd64/include/md_var.h
  stable/12/sys/dev/cpuctl/cpuctl.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/amd64/amd64/initcpu.c
==============================================================================
--- stable/12/sys/amd64/amd64/initcpu.c	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/amd64/initcpu.c	Sun Nov 18 09:49:24 2018	(r340552)
@@ -253,6 +253,7 @@ initializecpu(void)
 	}
 	hw_ibrs_recalculate();
 	hw_ssb_recalculate(false);
+	amd64_syscall_ret_flush_l1d_recalc();
 	switch (cpu_vendor_id) {
 	case CPU_VENDOR_AMD:
 		init_amd();

Modified: stable/12/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/machdep.c	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/amd64/machdep.c	Sun Nov 18 09:49:24 2018	(r340552)
@@ -1728,6 +1728,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	    != NULL)
 		vty_set_preferred(VTY_VT);
 
+	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
+	TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+	TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush",
+	    &syscall_ret_l1d_flush_mode);
+
 	finishidentcpu();	/* Final stage of CPU initialization */
 	initializecpu();	/* Initialize CPU registers */
 
@@ -1875,9 +1880,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	x86_init_fdt();
 #endif
 	thread0.td_critnest = 0;
-
-	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
-	TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
 
 	TSEXIT();
 

Modified: stable/12/sys/amd64/amd64/support.S
==============================================================================
--- stable/12/sys/amd64/amd64/support.S	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/amd64/support.S	Sun Nov 18 09:49:24 2018	(r340552)
@@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw)
 	ret
 #undef	L1D_FLUSH_SIZE
 END(flush_l1d_sw)
+
+ENTRY(flush_l1d_sw_abi)
+	pushq	%rbx
+	call	flush_l1d_sw
+	popq	%rbx
+	ret
+END(flush_l1d_sw_abi)

Modified: stable/12/sys/amd64/amd64/trap.c
==============================================================================
--- stable/12/sys/amd64/amd64/trap.c	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/amd64/trap.c	Sun Nov 18 09:49:24 2018	(r340552)
@@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td)
 
 #include "../../kern/subr_syscall.c"
 
+static void (*syscall_ret_l1d_flush)(void);
+int syscall_ret_l1d_flush_mode;
+
+static void
+flush_l1d_hw(void)
+{
+
+	wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
+}
+
+static void __inline
+amd64_syscall_ret_flush_l1d_inline(int error)
+{
+	void (*p)(void);
+
+	if (error != 0 && error != EEXIST && error != EAGAIN &&
+	    error != EXDEV && error != ENOENT && error != ENOTCONN &&
+	    error != EINPROGRESS) {
+		p = syscall_ret_l1d_flush;
+		if (p != NULL)
+			p();
+	}
+}
+
+void
+amd64_syscall_ret_flush_l1d(int error)
+{
+
+	amd64_syscall_ret_flush_l1d_inline(error);
+}
+
+void
+amd64_syscall_ret_flush_l1d_recalc(void)
+{
+	bool l1d_hw;
+
+	l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
+again:
+	switch (syscall_ret_l1d_flush_mode) {
+	case 0:
+		syscall_ret_l1d_flush = NULL;
+		break;
+	case 1:
+		syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
+		    flush_l1d_sw_abi;
+		break;
+	case 2:
+		syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
+		break;
+	case 3:
+		syscall_ret_l1d_flush = flush_l1d_sw_abi;
+		break;
+	default:
+		syscall_ret_l1d_flush_mode = 1;
+		goto again;
+	}
+}
+
+static int
+machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
+{
+	int error, val;
+
+	val = syscall_ret_l1d_flush_mode;
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	syscall_ret_l1d_flush_mode = val;
+	amd64_syscall_ret_flush_l1d_recalc();
+	return (0);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
+    CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
+    machdep_syscall_ret_flush_l1d, "I",
+    "Flush L1D on syscall return with error (0 - off, 1 - on, "
+    "2 - use hw only, 3 - use sw only");
+
+
 /*
  * System call handler for native binaries.  The trap frame is already
  * set up by the assembler trampoline and a pointer to it is saved in
@@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced)
 	 */
 	if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
 		set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+
+	amd64_syscall_ret_flush_l1d_inline(error);
 }

Modified: stable/12/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- stable/12/sys/amd64/ia32/ia32_syscall.c	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/ia32/ia32_syscall.c	Sun Nov 18 09:49:24 2018	(r340552)
@@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame)
 	}
 
 	syscallret(td, error);
+	amd64_syscall_ret_flush_l1d(error);
 }
 
 static void

Modified: stable/12/sys/amd64/include/md_var.h
==============================================================================
--- stable/12/sys/amd64/include/md_var.h	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/amd64/include/md_var.h	Sun Nov 18 09:49:24 2018	(r340552)
@@ -41,6 +41,7 @@ extern int	hw_lower_amd64_sharedpage;
 extern int	hw_ibrs_disable;
 extern int	hw_ssb_disable;
 extern int	nmi_flush_l1d_sw;
+extern int	syscall_ret_l1d_flush_mode;
 
 extern vm_paddr_t intel_graphics_stolen_base;
 extern vm_paddr_t intel_graphics_stolen_size;
@@ -58,8 +59,11 @@ void	amd64_conf_fast_syscall(void);
 void	amd64_db_resume_dbreg(void);
 void	amd64_lower_shared_page(struct sysentvec *);
 void	amd64_syscall(struct thread *td, int traced);
+void	amd64_syscall_ret_flush_l1d(int error);
+void	amd64_syscall_ret_flush_l1d_recalc(void);
 void	doreti_iret(void) __asm(__STRING(doreti_iret));
 void	doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void	flush_l1d_sw_abi(void);
 void	ld_ds(void) __asm(__STRING(ld_ds));
 void	ld_es(void) __asm(__STRING(ld_es));
 void	ld_fs(void) __asm(__STRING(ld_fs));

Modified: stable/12/sys/dev/cpuctl/cpuctl.c
==============================================================================
--- stable/12/sys/dev/cpuctl/cpuctl.c	Sun Nov 18 09:48:51 2018	(r340551)
+++ stable/12/sys/dev/cpuctl/cpuctl.c	Sun Nov 18 09:49:24 2018	(r340552)
@@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td
 	hw_ibrs_recalculate();
 	restore_cpu(oldcpu, is_bound, td);
 	hw_ssb_recalculate(true);
+#ifdef __amd64__
+	amd64_syscall_ret_flush_l1d_recalc();
+#endif
 	printcpuinfo();
 	return (0);
 }

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811180949.wAI9nOpa076097>