Date: Sat, 10 Aug 2019 08:27:36 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r350832 - in stable/12/sys/amd64: amd64 include Message-ID: <201908100827.x7A8Radu065152@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Sat Aug 10 08:27:36 2019 New Revision: 350832 URL: https://svnweb.freebsd.org/changeset/base/350832 Log: MFC r350561: amd64: Streamline exceptions and interrupts handlers. Modified: stable/12/sys/amd64/amd64/exception.S stable/12/sys/amd64/include/asmacros.h Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/amd64/amd64/exception.S ============================================================================== --- stable/12/sys/amd64/amd64/exception.S Sat Aug 10 08:26:10 2019 (r350831) +++ stable/12/sys/amd64/amd64/exception.S Sat Aug 10 08:27:36 2019 (r350832) @@ -105,14 +105,31 @@ MCOUNT_LABEL(btrap) /* Traps that we leave interrupts disabled for. */ .macro TRAP_NOEN l, trapno - PTI_ENTRY \l,X\l + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u +\l\()_pti_k: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_noen_k +\l\()_pti_u: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_noen_u + .globl X\l .type X\l,@function -X\l: subq $TF_RIP,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - movq $0,TF_ERR(%rsp) - jmp alltraps_noen +X\l: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_noen_k + swapgs + jmp alltraps_noen_u .endm TRAP_NOEN bpt, T_BPTFLT @@ -122,15 +139,31 @@ X\l: subq $TF_RIP,%rsp /* Regular traps; The cpu does not supply tf_err for these. */ .macro TRAP l, trapno - PTI_ENTRY \l,X\l + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u +\l\()_pti_k: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_k +\l\()_pti_u: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_u + .globl X\l .type X\l,@function X\l: - subq $TF_RIP,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - movq $0,TF_ERR(%rsp) - jmp alltraps + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_k + swapgs + jmp alltraps_u .endm TRAP div, T_DIVIDE @@ -145,42 +178,62 @@ X\l: /* This group of traps have tf_err already pushed by the cpu. */ .macro TRAP_ERR l, trapno - PTI_ENTRY \l,X\l,has_err=1 + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u,has_err=1 +\l\()_pti_k: + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + jmp alltraps_k +\l\()_pti_u: + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + jmp alltraps_u .globl X\l .type X\l,@function X\l: - subq $TF_ERR,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - jmp alltraps + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_k + swapgs + jmp alltraps_u .endm TRAP_ERR tss, T_TSSFLT TRAP_ERR align, T_ALIGNFLT /* - * alltraps entry point. Use swapgs if this is the first time in the - * kernel from userland. Reenable interrupts if they were enabled - * before the trap. This approximates SDT_SYS386TGT on the i386 port. + * alltraps_u/k entry points. + * SWAPGS must be already performed by prologue, + * if this is the first time in the kernel from userland. + * Reenable interrupts if they were enabled before the trap. + * This approximates SDT_SYS386TGT on the i386 port. */ SUPERALIGN_TEXT - .globl alltraps - .type alltraps,@function -alltraps: + .globl alltraps_u + .type alltraps_u,@function +alltraps_u: movq %rdi,TF_RDI(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz 1f /* already running with kernel GS.base */ - swapgs + movq %rdx,TF_RDX(%rsp) + movq %rax,TF_RAX(%rsp) + movq %rcx,TF_RCX(%rsp) movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) -1: SAVE_SEGS + call handle_ibrs_entry + jmp alltraps_save_segs + SUPERALIGN_TEXT + .globl alltraps_k + .type alltraps_k,@function +alltraps_k: + movq %rdi,TF_RDI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) - jz 2f - call handle_ibrs_entry -2: testl $PSL_I,TF_RFLAGS(%rsp) +alltraps_save_segs: + SAVE_SEGS + testl $PSL_I,TF_RFLAGS(%rsp) jz alltraps_pushregs_no_rax sti alltraps_pushregs_no_rax: @@ -234,21 +287,26 @@ calltrap: jmp doreti /* Handle any pending ASTs */ /* - * alltraps_noen entry point. Unlike alltraps above, we want to - * leave the interrupts disabled. This corresponds to - * SDT_SYS386IGT on the i386 port. + * alltraps_noen_u/k entry points. + * Again, SWAPGS must be already performed by prologue, if needed. + * Unlike alltraps above, we want to leave the interrupts disabled. + * This corresponds to SDT_SYS386IGT on the i386 port. */ SUPERALIGN_TEXT - .globl alltraps_noen - .type alltraps_noen,@function -alltraps_noen: + .globl alltraps_noen_u + .type alltraps_noen_u,@function +alltraps_noen_u: movq %rdi,TF_RDI(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz 1f /* already running with kernel GS.base */ - swapgs movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) -1: SAVE_SEGS + jmp alltraps_noen_save_segs + SUPERALIGN_TEXT + .globl alltraps_noen_k + .type alltraps_noen_k,@function +alltraps_noen_k: + movq %rdi,TF_RDI(%rsp) +alltraps_noen_save_segs: + SAVE_SEGS movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) @@ -297,8 +355,8 @@ IDTVEC(dblfault) ALIGN_TEXT IDTVEC(page_pti) - testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) - jz Xpage + testb $SEL_RPL_MASK,PTI_CS-PTI_ERR(%rsp) + jz page_k swapgs pushq %rax movq %cr3,%rax @@ -306,25 +364,31 @@ IDTVEC(page_pti) cmpq $~0,PCPU(UCR3) jne 1f popq %rax - jmp 2f + jmp page_u 1: pushq %rdx PTI_UUENTRY has_err=1 -2: subq $TF_ERR,%rsp - movq %rdi,TF_RDI(%rsp) - movq %rax,TF_RAX(%rsp) - movq %rdx,TF_RDX(%rsp) - movq %rcx,TF_RCX(%rsp) jmp page_u + ALIGN_TEXT IDTVEC(page) + testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) /* Did we come from kernel? */ + jnz page_u_swapgs /* already running with kernel GS.base */ +page_k: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) /* free up GP registers */ movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz page_cr2 /* already running with kernel GS.base */ + jmp page_cr2 + ALIGN_TEXT +page_u_swapgs: swapgs -page_u: movq PCPU(CURPCB),%rdi +page_u: + subq $TF_ERR,%rsp + movq %rdi,TF_RDI(%rsp) + movq %rax,TF_RAX(%rsp) + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) movq PCPU(SAVED_UCR3),%rax movq %rax,PCB_SAVED_UCR3(%rdi) Modified: stable/12/sys/amd64/include/asmacros.h ============================================================================== --- stable/12/sys/amd64/include/asmacros.h Sat Aug 10 08:26:10 2019 (r350831) +++ stable/12/sys/amd64/include/asmacros.h Sat Aug 10 08:27:36 2019 (r350832) @@ -204,17 +204,16 @@ 1: .endm - .macro PTI_ENTRY name, cont, has_err=0 + .macro PTI_ENTRY name, contk, contu, has_err=0 ALIGN_TEXT .globl X\name\()_pti .type X\name\()_pti,@function X\name\()_pti: - /* %rax, %rdx and possibly err not yet pushed */ - testb $SEL_RPL_MASK,PTI_CS-(2+1-\has_err)*8(%rsp) - jz \cont + /* %rax, %rdx, and possibly err are not yet pushed */ + testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp) + jz \contk PTI_UENTRY \has_err - swapgs - jmp \cont + jmp \contu .endm .macro PTI_INTRENTRY vec_name
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201908100827.x7A8Radu065152>