Date: Mon, 23 Jun 2014 07:37:54 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r267767 - in head/sys: amd64/amd64 amd64/include crypto/aesni crypto/via i386/include i386/isa Message-ID: <201406230737.s5N7bsNU067408@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Mon Jun 23 07:37:54 2014 New Revision: 267767 URL: http://svnweb.freebsd.org/changeset/base/267767 Log: Add FPU_KERN_KTHR flag to fpu_kern_enter(9), which avoids saving FPU context into memory for the kernel threads which called fpu_kern_thread(9). This allows the fpu_kern_enter() callers to not check for is_fpu_kern_thread() to get the optimization. Apply the flag to padlock(4) and aesni(4). In aesni_cipher_process(), do not leak FPU context state on error. Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/amd64/amd64/fpu.c head/sys/amd64/include/fpu.h head/sys/crypto/aesni/aesni_wrap.c head/sys/crypto/via/padlock.c head/sys/crypto/via/padlock_cipher.c head/sys/crypto/via/padlock_hash.c head/sys/i386/include/npx.h head/sys/i386/isa/npx.c Modified: head/sys/amd64/amd64/fpu.c ============================================================================== --- head/sys/amd64/amd64/fpu.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/amd64/amd64/fpu.c Mon Jun 23 07:37:54 2014 (r267767) @@ -890,6 +890,7 @@ static MALLOC_DEFINE(M_FPUKERN_CTX, "fpu "Kernel contexts for FPU state"); #define FPU_KERN_CTX_FPUINITDONE 0x01 +#define FPU_KERN_CTX_DUMMY 0x02 /* avoided save for the kern thread */ struct fpu_kern_ctx { struct savefpu *prev; @@ -933,6 +934,10 @@ fpu_kern_enter(struct thread *td, struct { struct pcb *pcb; + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY; + return (0); + } pcb = td->td_pcb; KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); @@ -952,6 +957,9 @@ fpu_kern_leave(struct thread *td, struct { struct pcb *pcb; + if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); + KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); pcb = td->td_pcb; critical_enter(); if (curthread == PCPU_GET(fpcurthread)) Modified: head/sys/amd64/include/fpu.h ============================================================================== --- head/sys/amd64/include/fpu.h Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/amd64/include/fpu.h Mon Jun 23 07:37:54 2014 (r267767) @@ -84,6 +84,7 @@ void fpu_save_area_reset(struct savefpu */ #define FPU_KERN_NORMAL 0x0000 #define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 #endif Modified: head/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- head/sys/crypto/aesni/aesni_wrap.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/aesni/aesni_wrap.c Mon Jun 23 07:37:54 2014 (r267767) @@ -382,22 +382,16 @@ int aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) { struct thread *td; - int error, saved_ctx; + int error; td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } - if (error == 0) { - error = aesni_cipher_setup_common(ses, encini->cri_key, - encini->cri_klen); - if (saved_ctx) - fpu_kern_leave(td, ses->fpu_ctx); - } + error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); + if (error != 0) + return (error); + error = aesni_cipher_setup_common(ses, encini->cri_key, + encini->cri_klen); + fpu_kern_leave(td, ses->fpu_ctx); return (error); } @@ -407,22 +401,17 @@ aesni_cipher_process(struct aesni_sessio { struct thread *td; uint8_t *buf; - int error, allocated, saved_ctx; + int error, allocated; buf = aesni_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) return (ENOMEM); td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); - if (error != 0) - goto out; - saved_ctx = 1; - } else { - saved_ctx = 0; - error = 0; - } + error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); + if (error != 0) + goto out1; if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { error = aesni_cipher_setup_common(ses, enccrd->crd_key, @@ -460,8 +449,6 @@ aesni_cipher_process(struct aesni_sessio ses->iv); } } - if (saved_ctx) - fpu_kern_leave(td, ses->fpu_ctx); if (allocated) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); @@ -469,7 +456,9 @@ aesni_cipher_process(struct aesni_sessio crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, AES_BLOCK_LEN, ses->iv); - out: +out: + fpu_kern_leave(td, ses->fpu_ctx); +out1: if (allocated) { bzero(buf, enccrd->crd_len); free(buf, M_AESNI); Modified: head/sys/crypto/via/padlock.c ============================================================================== --- head/sys/crypto/via/padlock.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock.c Mon Jun 23 07:37:54 2014 (r267767) @@ -171,7 +171,7 @@ padlock_newsession(device_t dev, uint32_ struct padlock_session *ses = NULL; struct cryptoini *encini, *macini; struct thread *td; - int error, saved_ctx; + int error; if (sidp == NULL || cri == NULL) return (EINVAL); @@ -246,18 +246,11 @@ padlock_newsession(device_t dev, uint32_ if (macini != NULL) { td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, - FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error == 0) { error = padlock_hash_setup(ses, macini); - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); } if (error != 0) { padlock_freesession_one(sc, ses, 0); Modified: head/sys/crypto/via/padlock_cipher.c ============================================================================== --- head/sys/crypto/via/padlock_cipher.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock_cipher.c Mon Jun 23 07:37:54 2014 (r267767) @@ -205,7 +205,7 @@ padlock_cipher_process(struct padlock_se struct thread *td; u_char *buf, *abuf; uint32_t *key; - int allocated, error, saved_ctx; + int allocated, error; buf = padlock_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) @@ -250,21 +250,13 @@ padlock_cipher_process(struct padlock_se } td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error != 0) goto out; - padlock_cbc(abuf, abuf, enccrd->crd_len / AES_BLOCK_LEN, key, cw, ses->ses_iv); - - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); if (allocated) { crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, Modified: head/sys/crypto/via/padlock_hash.c ============================================================================== --- head/sys/crypto/via/padlock_hash.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock_hash.c Mon Jun 23 07:37:54 2014 (r267767) @@ -366,24 +366,18 @@ padlock_hash_process(struct padlock_sess struct cryptop *crp) { struct thread *td; - int error, saved_ctx; + int error; td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error != 0) return (error); if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen); error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags); - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); return (error); } Modified: head/sys/i386/include/npx.h ============================================================================== --- head/sys/i386/include/npx.h Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/i386/include/npx.h Mon Jun 23 07:37:54 2014 (r267767) @@ -71,6 +71,7 @@ int is_fpu_kern_thread(u_int flags); */ #define FPU_KERN_NORMAL 0x0000 #define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 #endif Modified: head/sys/i386/isa/npx.c ============================================================================== --- head/sys/i386/isa/npx.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/i386/isa/npx.c Mon Jun 23 07:37:54 2014 (r267767) @@ -1008,6 +1008,7 @@ static MALLOC_DEFINE(M_FPUKERN_CTX, "fpu #define XSAVE_AREA_ALIGN 64 #define FPU_KERN_CTX_NPXINITDONE 0x01 +#define FPU_KERN_CTX_DUMMY 0x02 struct fpu_kern_ctx { union savefpu *prev; @@ -1051,6 +1052,10 @@ fpu_kern_enter(struct thread *td, struct { struct pcb *pcb; + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY; + return (0); + } pcb = td->td_pcb; KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == &pcb->pcb_user_save, ("mangled pcb_save")); @@ -1070,6 +1075,8 @@ fpu_kern_leave(struct thread *td, struct { struct pcb *pcb; + if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); pcb = td->td_pcb; critical_enter(); if (curthread == PCPU_GET(fpcurthread))
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201406230737.s5N7bsNU067408>