Date: Tue, 14 May 2019 23:28:46 +0000 (UTC) From: Enji Cooper <ngie@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r347599 - in projects/runtime-coverage-v2: share/man/man4 sys/amd64/amd64 sys/amd64/include sys/amd64/linux sys/compat/linuxkpi/common/include/linux sys/compat/linuxkpi/common/src sys/c... Message-ID: <201905142328.x4ENSkP9064450@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: ngie Date: Tue May 14 23:28:45 2019 New Revision: 347599 URL: https://svnweb.freebsd.org/changeset/base/347599 Log: MFhead@r347598 Modified: projects/runtime-coverage-v2/share/man/man4/ng_eiface.4 projects/runtime-coverage-v2/sys/amd64/amd64/exception.S projects/runtime-coverage-v2/sys/amd64/amd64/genassym.c projects/runtime-coverage-v2/sys/amd64/amd64/initcpu.c projects/runtime-coverage-v2/sys/amd64/amd64/machdep.c projects/runtime-coverage-v2/sys/amd64/amd64/support.S projects/runtime-coverage-v2/sys/amd64/amd64/trap.c projects/runtime-coverage-v2/sys/amd64/include/pcpu.h projects/runtime-coverage-v2/sys/amd64/linux/linux_ptrace.c projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/timer.h projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/ww_mutex.h projects/runtime-coverage-v2/sys/compat/linuxkpi/common/src/linux_compat.c projects/runtime-coverage-v2/sys/conf/kern.pre.mk projects/runtime-coverage-v2/sys/conf/kmod.mk projects/runtime-coverage-v2/sys/dev/cpuctl/cpuctl.c projects/runtime-coverage-v2/sys/i386/i386/exception.s projects/runtime-coverage-v2/sys/i386/i386/genassym.c projects/runtime-coverage-v2/sys/i386/i386/initcpu.c projects/runtime-coverage-v2/sys/i386/i386/support.s projects/runtime-coverage-v2/sys/i386/include/pcpu.h projects/runtime-coverage-v2/sys/kern/subr_witness.c projects/runtime-coverage-v2/sys/net/if_tuntap.c projects/runtime-coverage-v2/sys/netinet/in_mcast.c projects/runtime-coverage-v2/sys/netinet6/in6_mcast.c projects/runtime-coverage-v2/sys/sys/param.h projects/runtime-coverage-v2/sys/x86/include/specialreg.h projects/runtime-coverage-v2/sys/x86/include/x86_var.h projects/runtime-coverage-v2/sys/x86/x86/cpu_machdep.c projects/runtime-coverage-v2/usr.sbin/mountd/mountd.c Directory Properties: projects/runtime-coverage-v2/ (props changed) Modified: projects/runtime-coverage-v2/share/man/man4/ng_eiface.4 ============================================================================== --- projects/runtime-coverage-v2/share/man/man4/ng_eiface.4 Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/share/man/man4/ng_eiface.4 Tue May 14 23:28:45 2019 (r347599) @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 3, 2005 +.Dd May 14, 2019 .Dt NG_EIFACE 4 .Os .Sh NAME @@ -36,7 +36,7 @@ The .Vt eiface netgraph node implements the generic Ethernet interface. -When +When an .Vt eiface node is created, a new interface appears which is accessible via .Xr ifconfig 8 . @@ -78,7 +78,7 @@ which requires as an argument an string consisting of 6 colon-separated hex digits. .It Dv NGM_EIFACE_GET_IFNAME Pq Ic getifname Return the name of the associated interface as a -.Dv NUL Ns -terminated +.Dv NULL Ns -terminated .Tn ASCII string. .It Dv NGM_EIFACE_GET_IFADDRS @@ -97,7 +97,7 @@ Unlike most other node types, an .Vt eiface node does .Em not -go away when all hooks have been disconnected; rather, and explicit +go away when all hooks have been disconnected; rather, an explicit .Dv NGM_SHUTDOWN control message is required. .Sh SEE ALSO Modified: projects/runtime-coverage-v2/sys/amd64/amd64/exception.S ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/exception.S Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/exception.S Tue May 14 23:28:45 2019 (r347599) @@ -512,6 +512,7 @@ fast_syscall_common: testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) jne 3f call handle_ibrs_exit + callq *mds_handler /* Restore preserved registers. */ MEXITCOUNT movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ @@ -1157,6 +1158,7 @@ ld_regs: jz 2f /* keep running with kernel GS.base */ cli call handle_ibrs_exit_rs + callq *mds_handler cmpq $~0,PCPU(UCR3) je 1f pushq %rdx Modified: projects/runtime-coverage-v2/sys/amd64/amd64/genassym.c ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/genassym.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/genassym.c Tue May 14 23:28:45 2019 (r347599) @@ -233,6 +233,9 @@ ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ); ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0)); ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); +ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp)); +ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf)); +ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64)); ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL); Modified: projects/runtime-coverage-v2/sys/amd64/amd64/initcpu.c ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/initcpu.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/initcpu.c Tue May 14 23:28:45 2019 (r347599) @@ -257,6 +257,7 @@ initializecpu(void) hw_ibrs_recalculate(); hw_ssb_recalculate(false); amd64_syscall_ret_flush_l1d_recalc(); + hw_mds_recalculate(); switch (cpu_vendor_id) { case CPU_VENDOR_AMD: init_amd(); Modified: projects/runtime-coverage-v2/sys/amd64/amd64/machdep.c ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/machdep.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/machdep.c Tue May 14 23:28:45 2019 (r347599) @@ -1732,6 +1732,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush", &syscall_ret_l1d_flush_mode); + TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable); finishidentcpu(); /* Final stage of CPU initialization */ initializecpu(); /* Initialize CPU registers */ Modified: projects/runtime-coverage-v2/sys/amd64/amd64/support.S ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/support.S Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/support.S Tue May 14 23:28:45 2019 (r347599) @@ -1,8 +1,13 @@ /*- + * Copyright (c) 2018-2019 The FreeBSD Foundation * Copyright (c) 2003 Peter Wemm. * Copyright (c) 1993 The Regents of the University of California. * All rights reserved. * + * Portions of this software were developed by + * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from + * the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -1622,3 +1627,239 @@ ENTRY(flush_l1d_sw_abi) popq %rbx ret END(flush_l1d_sw_abi) + +ENTRY(mds_handler_void) + retq +END(mds_handler_void) + +ENTRY(mds_handler_verw) + subq $8, %rsp + movw %ds, (%rsp) + verw (%rsp) + addq $8, %rsp + retq +END(mds_handler_verw) + +ENTRY(mds_handler_ivb) + pushq %rax + pushq %rdx + pushq %rcx + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%rdx), %xmm0 + orpd (%rdx), %xmm0 + mfence + movl $40, %ecx + addq $16, %rdx +2: movntdq %xmm0, (%rdx) + addq $16, %rdx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_ivb) + +ENTRY(mds_handler_bdw) + pushq %rax + pushq %rbx + pushq %rcx + pushq %rdi + pushq %rsi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rbx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movq %rbx, %rdi + movq %rbx, %rsi + movl $40, %ecx +2: movntdq %xmm0, (%rbx) + addq $16, %rbx + decl %ecx + jnz 2b + mfence + movl $1536, %ecx + rep; movsb + lfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rsi + popq %rdi + popq %rcx + popq %rbx + popq %rax + retq +END(mds_handler_bdw) + +ENTRY(mds_handler_skl_sse) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%rdx), %xmm0 + orpd (%rdx), %xmm0 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + movdqa PCPU(MDS_TMP), %xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_sse) + +ENTRY(mds_handler_skl_avx) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx + vmovdqa %ymm0, PCPU(MDS_TMP) + vpxor %ymm0, %ymm0, %ymm0 + + lfence + vorpd (%rdx), %ymm0, %ymm0 + vorpd (%rdx), %ymm0, %ymm0 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa PCPU(MDS_TMP), %ymm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_avx) + +ENTRY(mds_handler_skl_avx512) + pushq %rax + pushq %rdx + pushq %rcx + pushq %rdi + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdi + movq PCPU(MDS_BUF64), %rdx + vmovdqa64 %zmm0, PCPU(MDS_TMP) + vpxor %zmm0, %zmm0, %zmm0 + + lfence + vorpd (%rdx), %zmm0, %zmm0 + vorpd (%rdx), %zmm0, %zmm0 + xorl %eax, %eax +2: clflushopt 5376(%rdi, %rax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa64 PCPU(MDS_TMP), %zmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rdi + popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_skl_avx512) + +ENTRY(mds_handler_silvermont) + pushq %rax + pushq %rdx + pushq %rcx + + movq %cr0, %rax + testb $CR0_TS, %al + je 1f + clts +1: movq PCPU(MDS_BUF), %rdx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl $16, %ecx +2: movntdq %xmm0, (%rdx) + addq $16, %rdx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movq %rax, %cr0 +3: popq %rcx + popq %rdx + popq %rax + retq +END(mds_handler_silvermont) Modified: projects/runtime-coverage-v2/sys/amd64/amd64/trap.c ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/amd64/trap.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/amd64/trap.c Tue May 14 23:28:45 2019 (r347599) @@ -904,8 +904,8 @@ trap_fatal(frame, eva) code & PGEX_U ? "user" : "supervisor", code & PGEX_W ? "write" : "read", code & PGEX_I ? "instruction" : "data", - code & PGEX_PK ? " prot key" : " ", - code & PGEX_SGX ? " SGX" : " ", + code & PGEX_PK ? " prot key" : "", + code & PGEX_SGX ? " SGX" : "", code & PGEX_RSV ? "reserved bits in PTE" : code & PGEX_P ? "protection violation" : "page not present"); } Modified: projects/runtime-coverage-v2/sys/amd64/include/pcpu.h ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/include/pcpu.h Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/include/pcpu.h Tue May 14 23:28:45 2019 (r347599) @@ -84,8 +84,12 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ + void *pc_mds_buf; \ + void *pc_mds_buf64; \ + uint32_t pc_pad[2]; \ + uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ - char __pad[3284] /* pad to UMA_PCPU_ALLOC_SIZE */ + char __pad[3172] /* pad to UMA_PCPU_ALLOC_SIZE */ #define PC_DBREG_CMD_NONE 0 #define PC_DBREG_CMD_LOAD 1 Modified: projects/runtime-coverage-v2/sys/amd64/linux/linux_ptrace.c ============================================================================== --- projects/runtime-coverage-v2/sys/amd64/linux/linux_ptrace.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/amd64/linux/linux_ptrace.c Tue May 14 23:28:45 2019 (r347599) @@ -338,18 +338,27 @@ linux_ptrace_getregs(struct thread *td, pid_t pid, voi map_regs_to_linux(&b_reg, &l_reg); - /* - * The strace(1) utility depends on RAX being set to -ENOSYS - * on syscall entry. - */ error = kern_ptrace(td, PT_LWPINFO, pid, &lwpinfo, sizeof(lwpinfo)); if (error != 0) { printf("%s: PT_LWPINFO failed with error %d\n", __func__, error); return (error); } - if (lwpinfo.pl_flags & PL_FLAG_SCE) - l_reg.rax = -38; // XXX: Don't hardcode? + if (lwpinfo.pl_flags & PL_FLAG_SCE) { + /* + * The strace(1) utility depends on RAX being set to -ENOSYS + * on syscall entry; otherwise it loops printing those: + * + * [ Process PID=928 runs in 64 bit mode. ] + * [ Process PID=928 runs in x32 mode. ] + */ + l_reg.rax = -38; /* -ENOSYS */ + /* + * Undo the mangling done in exception.S:fast_syscall_common(). + */ + l_reg.r10 = l_reg.rcx; + } + error = copyout(&l_reg, (void *)data, sizeof(l_reg)); return (error); } @@ -399,21 +408,27 @@ linux_ptrace_getregset_prstatus(struct thread *td, pid map_regs_to_linux_regset(&b_reg, fsbase, gsbase, &l_regset); - /* - * The strace(1) utility depends on RAX being set to -ENOSYS - * on syscall entry; otherwise it loops printing those: - * - * [ Process PID=928 runs in 64 bit mode. ] - * [ Process PID=928 runs in x32 mode. ] - */ error = kern_ptrace(td, PT_LWPINFO, pid, &lwpinfo, sizeof(lwpinfo)); if (error != 0) { printf("%s: PT_LWPINFO failed with error %d\n", __func__, error); return (error); } - if (lwpinfo.pl_flags & PL_FLAG_SCE) - l_regset.rax = -38; // XXX: Don't hardcode? + if (lwpinfo.pl_flags & PL_FLAG_SCE) { + /* + * The strace(1) utility depends on RAX being set to -ENOSYS + * on syscall entry; otherwise it loops printing those: + * + * [ Process PID=928 runs in 64 bit mode. ] + * [ Process PID=928 runs in x32 mode. ] + */ + l_regset.rax = -38; /* -ENOSYS */ + + /* + * Undo the mangling done in exception.S:fast_syscall_common(). + */ + l_regset.r10 = l_regset.rcx; + } len = MIN(iov.iov_len, sizeof(l_regset)); error = copyout(&l_regset, (void *)iov.iov_base, len); Modified: projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/timer.h ============================================================================== --- projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/timer.h Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/timer.h Tue May 14 23:28:45 2019 (r347599) @@ -81,8 +81,8 @@ extern unsigned long linux_timer_hz_mask; extern void mod_timer(struct timer_list *, int); extern void add_timer(struct timer_list *); extern void add_timer_on(struct timer_list *, int cpu); +extern int del_timer(struct timer_list *); -#define del_timer(timer) (void)callout_stop(&(timer)->callout) #define del_timer_sync(timer) (void)callout_drain(&(timer)->callout) #define timer_pending(timer) callout_pending(&(timer)->callout) #define round_jiffies(j) \ Modified: projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/ww_mutex.h ============================================================================== --- projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/ww_mutex.h Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/compat/linuxkpi/common/include/linux/ww_mutex.h Tue May 14 23:28:45 2019 (r347599) @@ -45,6 +45,7 @@ struct ww_acquire_ctx { struct ww_mutex { struct mutex base; struct cv condvar; + struct ww_acquire_ctx *ctx; }; #define DEFINE_WW_CLASS(name) \ Modified: projects/runtime-coverage-v2/sys/compat/linuxkpi/common/src/linux_compat.c ============================================================================== --- projects/runtime-coverage-v2/sys/compat/linuxkpi/common/src/linux_compat.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/compat/linuxkpi/common/src/linux_compat.c Tue May 14 23:28:45 2019 (r347599) @@ -1906,6 +1906,15 @@ add_timer_on(struct timer_list *timer, int cpu) &linux_timer_callback_wrapper, timer, cpu); } +int +del_timer(struct timer_list *timer) +{ + + if (callout_stop(&(timer)->callout) == -1) + return (0); + return (1); +} + static void linux_timer_init(void *arg) { Modified: projects/runtime-coverage-v2/sys/conf/kern.pre.mk ============================================================================== --- projects/runtime-coverage-v2/sys/conf/kern.pre.mk Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/conf/kern.pre.mk Tue May 14 23:28:45 2019 (r347599) @@ -144,7 +144,7 @@ CFLAGS+= ${GCOV_CFLAGS} CFLAGS+= ${CONF_CFLAGS} .if defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mbuild-id} -LDFLAGS+= -Wl,--build-id=sha1 +LDFLAGS+= --build-id=sha1 .endif .if (${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \ @@ -153,11 +153,11 @@ LDFLAGS+= -Wl,--build-id=sha1 .error amd64/arm64/i386 kernel requires linker ifunc support .endif .if ${MACHINE_CPUARCH} == "amd64" -LDFLAGS+= -Wl,-z max-page-size=2097152 +LDFLAGS+= -z max-page-size=2097152 .if ${LINKER_TYPE} != "lld" -LDFLAGS+= -Wl,-z common-page-size=4096 +LDFLAGS+= -z common-page-size=4096 .else -LDFLAGS+= -Wl,-z -Wl,ifunc-noplt +LDFLAGS+= -z notext -z ifunc-noplt .endif .endif Modified: projects/runtime-coverage-v2/sys/conf/kmod.mk ============================================================================== --- projects/runtime-coverage-v2/sys/conf/kmod.mk Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/conf/kmod.mk Tue May 14 23:28:45 2019 (r347599) @@ -138,7 +138,7 @@ CFLAGS+= -fno-common LDFLAGS+= -d -warn-common .if defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mbuild-id} -LDFLAGS+= -Wl,--build-id=sha1 +LDFLAGS+= --build-id=sha1 .endif CFLAGS+= ${DEBUG_FLAGS} Modified: projects/runtime-coverage-v2/sys/dev/cpuctl/cpuctl.c ============================================================================== --- projects/runtime-coverage-v2/sys/dev/cpuctl/cpuctl.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/dev/cpuctl/cpuctl.c Tue May 14 23:28:45 2019 (r347599) @@ -540,6 +540,7 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td #ifdef __amd64__ amd64_syscall_ret_flush_l1d_recalc(); #endif + hw_mds_recalculate(); printcpuinfo(); return (0); } Modified: projects/runtime-coverage-v2/sys/i386/i386/exception.s ============================================================================== --- projects/runtime-coverage-v2/sys/i386/i386/exception.s Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/i386/i386/exception.s Tue May 14 23:28:45 2019 (r347599) @@ -522,6 +522,8 @@ doreti_exit: 2: movl $handle_ibrs_exit,%eax pushl %ecx /* preserve enough call-used regs */ call *%eax + movl mds_handler,%eax + call *%eax popl %ecx movl %esp, %esi movl PCPU(TRAMPSTK), %edx Modified: projects/runtime-coverage-v2/sys/i386/i386/genassym.c ============================================================================== --- projects/runtime-coverage-v2/sys/i386/i386/genassym.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/i386/i386/genassym.c Tue May 14 23:28:45 2019 (r347599) @@ -209,6 +209,9 @@ ASSYM(PC_KESP0, offsetof(struct pcpu, pc_kesp0)); ASSYM(PC_TRAMPSTK, offsetof(struct pcpu, pc_trampstk)); ASSYM(PC_COPYOUT_BUF, offsetof(struct pcpu, pc_copyout_buf)); ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); +ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp)); +ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf)); +ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64)); ASSYM(PMAP_TRM_MIN_ADDRESS, PMAP_TRM_MIN_ADDRESS); ASSYM(KERNLOAD, KERNLOAD); ASSYM(KERNBASE, KERNBASE); Modified: projects/runtime-coverage-v2/sys/i386/i386/initcpu.c ============================================================================== --- projects/runtime-coverage-v2/sys/i386/i386/initcpu.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/i386/i386/initcpu.c Tue May 14 23:28:45 2019 (r347599) @@ -749,6 +749,7 @@ initializecpu(void) msr = rdmsr(MSR_EFER) | EFER_NXE; wrmsr(MSR_EFER, msr); } + hw_mds_recalculate(); if ((amd_feature & AMDID_RDTSCP) != 0 || (cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0) wrmsr(MSR_TSC_AUX, PCPU_GET(cpuid)); Modified: projects/runtime-coverage-v2/sys/i386/i386/support.s ============================================================================== --- projects/runtime-coverage-v2/sys/i386/i386/support.s Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/i386/i386/support.s Tue May 14 23:28:45 2019 (r347599) @@ -472,3 +472,187 @@ ENTRY(handle_ibrs_exit) movb $0,PCPU(IBPB_SET) 1: ret END(handle_ibrs_exit) + +ENTRY(mds_handler_void) + ret +END(mds_handler_void) + +ENTRY(mds_handler_verw) + subl $4, %esp + movw %ds, (%esp) + verw (%esp) + addl $4, %esp + ret +END(mds_handler_verw) + +ENTRY(mds_handler_ivb) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%edx), %xmm0 + orpd (%edx), %xmm0 + mfence + movl $40, %ecx + addl $16, %edx +2: movntdq %xmm0, (%edx) + addl $16, %edx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_ivb) + +ENTRY(mds_handler_bdw) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %ebx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl %ebx, %edi + movl %ebx, %esi + movl $40, %ecx +2: movntdq %xmm0, (%ebx) + addl $16, %ebx + decl %ecx + jnz 2b + mfence + movl $1536, %ecx + rep; movsb + lfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_bdw) + +ENTRY(mds_handler_skl_sse) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + lfence + orpd (%edx), %xmm0 + orpd (%edx), %xmm0 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + movdqa PCPU(MDS_TMP), %xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_sse) + +ENTRY(mds_handler_skl_avx) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx + vmovdqa %ymm0, PCPU(MDS_TMP) + vpxor %ymm0, %ymm0, %ymm0 + + lfence + vorpd (%edx), %ymm0, %ymm0 + vorpd (%edx), %ymm0, %ymm0 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa PCPU(MDS_TMP), %ymm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_avx) + +ENTRY(mds_handler_skl_avx512) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edi + movl PCPU(MDS_BUF64), %edx + vmovdqa64 %zmm0, PCPU(MDS_TMP) + vpxor %zmm0, %zmm0, %zmm0 + + lfence + vorpd (%edx), %zmm0, %zmm0 + vorpd (%edx), %zmm0, %zmm0 + xorl %eax, %eax +2: clflushopt 5376(%edi, %eax, 8) + addl $8, %eax + cmpl $8 * 12, %eax + jb 2b + sfence + movl $6144, %ecx + xorl %eax, %eax + rep; stosb + mfence + + vmovdqa64 PCPU(MDS_TMP), %zmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_skl_avx512) + +ENTRY(mds_handler_silvermont) + movl %cr0, %eax + testb $CR0_TS, %al + je 1f + clts +1: movl PCPU(MDS_BUF), %edx + movdqa %xmm0, PCPU(MDS_TMP) + pxor %xmm0, %xmm0 + + movl $16, %ecx +2: movntdq %xmm0, (%edx) + addl $16, %edx + decl %ecx + jnz 2b + mfence + + movdqa PCPU(MDS_TMP),%xmm0 + testb $CR0_TS, %al + je 3f + movl %eax, %cr0 +3: ret +END(mds_handler_silvermont) Modified: projects/runtime-coverage-v2/sys/i386/include/pcpu.h ============================================================================== --- projects/runtime-coverage-v2/sys/i386/include/pcpu.h Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/i386/include/pcpu.h Tue May 14 23:28:45 2019 (r347599) @@ -84,11 +84,15 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c struct sx pc_copyout_slock; \ char *pc_copyout_buf; \ vm_offset_t pc_pmap_eh_va; \ - caddr_t pc_pmap_eh_ptep; \ + caddr_t pc_pmap_eh_ptep; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ + void *pc_mds_buf; \ + void *pc_mds_buf64; \ + uint32_t pc_pad[4]; \ + uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ - char __pad[3606] + char __pad[3518] #ifdef _KERNEL Modified: projects/runtime-coverage-v2/sys/kern/subr_witness.c ============================================================================== --- projects/runtime-coverage-v2/sys/kern/subr_witness.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/kern/subr_witness.c Tue May 14 23:28:45 2019 (r347599) @@ -576,7 +576,6 @@ static struct witness_order_list_entry order_lists[] = * BPF */ { "bpf global lock", &lock_class_sx }, - { "bpf interface lock", &lock_class_rw }, { "bpf cdev lock", &lock_class_mtx_sleep }, { NULL, NULL }, /* Modified: projects/runtime-coverage-v2/sys/net/if_tuntap.c ============================================================================== --- projects/runtime-coverage-v2/sys/net/if_tuntap.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/net/if_tuntap.c Tue May 14 23:28:45 2019 (r347599) @@ -537,9 +537,6 @@ tun_destroy(struct tuntap_softc *tp) TUN_UNLOCK(tp); CURVNET_SET(TUN2IFP(tp)->if_vnet); - sx_xlock(&tun_ioctl_sx); - TUN2IFP(tp)->if_softc = NULL; - sx_xunlock(&tun_ioctl_sx); destroy_dev(tp->tun_dev); seldrain(&tp->tun_rsel); @@ -551,6 +548,9 @@ tun_destroy(struct tuntap_softc *tp) bpfdetach(TUN2IFP(tp)); if_detach(TUN2IFP(tp)); } + sx_xlock(&tun_ioctl_sx); + TUN2IFP(tp)->if_softc = NULL; + sx_xunlock(&tun_ioctl_sx); free_unr(tp->tun_drv->unrhdr, TUN2IFP(tp)->if_dunit); if_free(TUN2IFP(tp)); mtx_destroy(&tp->tun_mtx); Modified: projects/runtime-coverage-v2/sys/netinet/in_mcast.c ============================================================================== --- projects/runtime-coverage-v2/sys/netinet/in_mcast.c Tue May 14 23:28:04 2019 (r347598) +++ projects/runtime-coverage-v2/sys/netinet/in_mcast.c Tue May 14 23:28:45 2019 (r347599) @@ -1534,6 +1534,7 @@ inp_block_unblock_source(struct inpcb *inp, struct soc /* * Check if we are actually a member of this group. */ + IN_MULTI_LOCK(); imo = inp_findmoptions(inp); idx = imo_match_group(imo, ifp, &gsa->sa); if (idx == -1 || imo->imo_mfilters == NULL) { @@ -1593,14 +1594,13 @@ inp_block_unblock_source(struct inpcb *inp, struct soc /* * Begin state merge transaction at IGMP layer. */ - IN_MULTI_LOCK(); CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); IN_MULTI_LIST_LOCK(); error = inm_merge(inm, imf); if (error) { CTR1(KTR_IGMPV3, "%s: failed to merge inm state", __func__); IN_MULTI_LIST_UNLOCK(); - goto out_in_multi_locked; + goto out_imf_rollback; } CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); @@ -1609,9 +1609,6 @@ inp_block_unblock_source(struct inpcb *inp, struct soc if (error) CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); -out_in_multi_locked: - - IN_MULTI_UNLOCK(); out_imf_rollback: if (error) imf_rollback(imf); @@ -1622,6 +1619,7 @@ out_imf_rollback: out_inp_locked: INP_WUNLOCK(inp); + IN_MULTI_UNLOCK(); return (error); } @@ -1680,10 +1678,10 @@ inp_findmoptions(struct inpcb *inp) static void inp_gcmoptions(struct ip_moptions *imo) { - struct in_mfilter *imf; + struct in_mfilter *imf; struct in_multi *inm; struct ifnet *ifp; - size_t idx, nmships; + size_t idx, nmships; nmships = imo->imo_num_memberships; for (idx = 0; idx < nmships; ++idx) { @@ -2142,12 +2140,12 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt CTR2(KTR_IGMPV3, "%s: unknown sopt_name %d", __func__, sopt->sopt_name); return (EOPNOTSUPP); - break; } if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0) return (EADDRNOTAVAIL); + IN_MULTI_LOCK(); imo = inp_findmoptions(inp); idx = imo_match_group(imo, ifp, &gsa->sa); if (idx == -1) { @@ -2272,10 +2270,6 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt /* * Begin state merge transaction at IGMP layer. */ - in_pcbref(inp); - INP_WUNLOCK(inp); - IN_MULTI_LOCK(); - if (is_new) { error = in_joingroup_locked(ifp, &gsa->sin.sin_addr, imf, &inm); @@ -2286,6 +2280,8 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt goto out_imo_free; } inm_acquire(inm); + KASSERT(imo->imo_membership[idx] == NULL, + ("%s: imo_membership already allocated", __func__)); imo->imo_membership[idx] = inm; } else { CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); @@ -2295,7 +2291,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt CTR1(KTR_IGMPV3, "%s: failed to merge inm state", __func__); IN_MULTI_LIST_UNLOCK(); - goto out_in_multi_locked; + goto out_imf_rollback; } CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); error = igmp_change_state(inm); @@ -2303,16 +2299,11 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt if (error) { CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); - goto out_in_multi_locked; + goto out_imf_rollback; } } -out_in_multi_locked: - - IN_MULTI_UNLOCK(); - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (ENXIO); +out_imf_rollback: if (error) { imf_rollback(imf); if (is_new) @@ -2337,6 +2328,7 @@ out_imo_free: out_inp_locked: INP_WUNLOCK(inp); + IN_MULTI_UNLOCK(); return (error); } @@ -2463,6 +2455,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sop /* * Find the membership in the membership array. */ + IN_MULTI_LOCK(); imo = inp_findmoptions(inp); idx = imo_match_group(imo, ifp, &gsa->sa); if (idx == -1) { @@ -2510,9 +2503,6 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sop /* * Begin state merge transaction at IGMP layer. */ - in_pcbref(inp); - INP_WUNLOCK(inp); - IN_MULTI_LOCK(); if (is_final) { /* @@ -2528,7 +2518,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sop CTR1(KTR_IGMPV3, "%s: failed to merge inm state", __func__); IN_MULTI_LIST_UNLOCK(); - goto out_in_multi_locked; + goto out_imf_rollback; } CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); @@ -2540,13 +2530,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sop } } -out_in_multi_locked: - - IN_MULTI_UNLOCK(); - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (ENXIO); - +out_imf_rollback: if (error) imf_rollback(imf); else @@ -2557,7 +2541,7 @@ out_in_multi_locked: if (is_final) { /* Remove the gap in the membership and filter array. */ KASSERT(RB_EMPTY(&imf->imf_sources), - ("%s: imf_sources not empty", __func__)); + ("%s: imf_sources (%p %p %zu) not empty", __func__, imf, imo, idx)); for (++idx; idx < imo->imo_num_memberships; ++idx) { imo->imo_membership[idx - 1] = imo->imo_membership[idx]; imo->imo_mfilters[idx - 1] = imo->imo_mfilters[idx]; @@ -2569,6 +2553,7 @@ out_in_multi_locked: out_inp_locked: INP_WUNLOCK(inp); + IN_MULTI_UNLOCK(); return (error); } @@ -2646,8 +2631,6 @@ inp_set_multicast_if(struct inpcb *inp, struct sockopt /* * Atomically set source filters on a socket for an IPv4 multicast group. - * - * SMPng: NOTE: Potentially calls malloc(M_WAITOK) with Giant held. *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201905142328.x4ENSkP9064450>