From owner-svn-src-all@FreeBSD.ORG Mon Jun 23 07:37:56 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 14BA4C19; Mon, 23 Jun 2014 07:37:56 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 00883276F; Mon, 23 Jun 2014 07:37:56 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s5N7btXa067420; Mon, 23 Jun 2014 07:37:55 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s5N7bsNU067408; Mon, 23 Jun 2014 07:37:54 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201406230737.s5N7bsNU067408@svn.freebsd.org> From: Konstantin Belousov Date: Mon, 23 Jun 2014 07:37:54 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r267767 - in head/sys: amd64/amd64 amd64/include crypto/aesni crypto/via i386/include i386/isa X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 23 Jun 2014 07:37:56 -0000 Author: kib Date: Mon Jun 23 07:37:54 2014 New Revision: 267767 URL: http://svnweb.freebsd.org/changeset/base/267767 Log: Add FPU_KERN_KTHR flag to fpu_kern_enter(9), which avoids saving FPU context into memory for the kernel threads which called fpu_kern_thread(9). This allows the fpu_kern_enter() callers to not check for is_fpu_kern_thread() to get the optimization. Apply the flag to padlock(4) and aesni(4). In aesni_cipher_process(), do not leak FPU context state on error. Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/amd64/amd64/fpu.c head/sys/amd64/include/fpu.h head/sys/crypto/aesni/aesni_wrap.c head/sys/crypto/via/padlock.c head/sys/crypto/via/padlock_cipher.c head/sys/crypto/via/padlock_hash.c head/sys/i386/include/npx.h head/sys/i386/isa/npx.c Modified: head/sys/amd64/amd64/fpu.c ============================================================================== --- head/sys/amd64/amd64/fpu.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/amd64/amd64/fpu.c Mon Jun 23 07:37:54 2014 (r267767) @@ -890,6 +890,7 @@ static MALLOC_DEFINE(M_FPUKERN_CTX, "fpu "Kernel contexts for FPU state"); #define FPU_KERN_CTX_FPUINITDONE 0x01 +#define FPU_KERN_CTX_DUMMY 0x02 /* avoided save for the kern thread */ struct fpu_kern_ctx { struct savefpu *prev; @@ -933,6 +934,10 @@ fpu_kern_enter(struct thread *td, struct { struct pcb *pcb; + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY; + return (0); + } pcb = td->td_pcb; KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); @@ -952,6 +957,9 @@ fpu_kern_leave(struct thread *td, struct { struct pcb *pcb; + if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); + KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); pcb = td->td_pcb; critical_enter(); if (curthread == PCPU_GET(fpcurthread)) Modified: head/sys/amd64/include/fpu.h ============================================================================== --- head/sys/amd64/include/fpu.h Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/amd64/include/fpu.h Mon Jun 23 07:37:54 2014 (r267767) @@ -84,6 +84,7 @@ void fpu_save_area_reset(struct savefpu */ #define FPU_KERN_NORMAL 0x0000 #define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 #endif Modified: head/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- head/sys/crypto/aesni/aesni_wrap.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/aesni/aesni_wrap.c Mon Jun 23 07:37:54 2014 (r267767) @@ -382,22 +382,16 @@ int aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) { struct thread *td; - int error, saved_ctx; + int error; td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } - if (error == 0) { - error = aesni_cipher_setup_common(ses, encini->cri_key, - encini->cri_klen); - if (saved_ctx) - fpu_kern_leave(td, ses->fpu_ctx); - } + error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); + if (error != 0) + return (error); + error = aesni_cipher_setup_common(ses, encini->cri_key, + encini->cri_klen); + fpu_kern_leave(td, ses->fpu_ctx); return (error); } @@ -407,22 +401,17 @@ aesni_cipher_process(struct aesni_sessio { struct thread *td; uint8_t *buf; - int error, allocated, saved_ctx; + int error, allocated; buf = aesni_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) return (ENOMEM); td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); - if (error != 0) - goto out; - saved_ctx = 1; - } else { - saved_ctx = 0; - error = 0; - } + error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); + if (error != 0) + goto out1; if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { error = aesni_cipher_setup_common(ses, enccrd->crd_key, @@ -460,8 +449,6 @@ aesni_cipher_process(struct aesni_sessio ses->iv); } } - if (saved_ctx) - fpu_kern_leave(td, ses->fpu_ctx); if (allocated) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); @@ -469,7 +456,9 @@ aesni_cipher_process(struct aesni_sessio crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, AES_BLOCK_LEN, ses->iv); - out: +out: + fpu_kern_leave(td, ses->fpu_ctx); +out1: if (allocated) { bzero(buf, enccrd->crd_len); free(buf, M_AESNI); Modified: head/sys/crypto/via/padlock.c ============================================================================== --- head/sys/crypto/via/padlock.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock.c Mon Jun 23 07:37:54 2014 (r267767) @@ -171,7 +171,7 @@ padlock_newsession(device_t dev, uint32_ struct padlock_session *ses = NULL; struct cryptoini *encini, *macini; struct thread *td; - int error, saved_ctx; + int error; if (sidp == NULL || cri == NULL) return (EINVAL); @@ -246,18 +246,11 @@ padlock_newsession(device_t dev, uint32_ if (macini != NULL) { td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, - FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error == 0) { error = padlock_hash_setup(ses, macini); - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); } if (error != 0) { padlock_freesession_one(sc, ses, 0); Modified: head/sys/crypto/via/padlock_cipher.c ============================================================================== --- head/sys/crypto/via/padlock_cipher.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock_cipher.c Mon Jun 23 07:37:54 2014 (r267767) @@ -205,7 +205,7 @@ padlock_cipher_process(struct padlock_se struct thread *td; u_char *buf, *abuf; uint32_t *key; - int allocated, error, saved_ctx; + int allocated, error; buf = padlock_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) @@ -250,21 +250,13 @@ padlock_cipher_process(struct padlock_se } td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error != 0) goto out; - padlock_cbc(abuf, abuf, enccrd->crd_len / AES_BLOCK_LEN, key, cw, ses->ses_iv); - - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); if (allocated) { crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, Modified: head/sys/crypto/via/padlock_hash.c ============================================================================== --- head/sys/crypto/via/padlock_hash.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/crypto/via/padlock_hash.c Mon Jun 23 07:37:54 2014 (r267767) @@ -366,24 +366,18 @@ padlock_hash_process(struct padlock_sess struct cryptop *crp) { struct thread *td; - int error, saved_ctx; + int error; td = curthread; - if (!is_fpu_kern_thread(0)) { - error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL); - saved_ctx = 1; - } else { - error = 0; - saved_ctx = 0; - } + error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | + FPU_KERN_KTHR); if (error != 0) return (error); if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen); error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags); - if (saved_ctx) - fpu_kern_leave(td, ses->ses_fpu_ctx); + fpu_kern_leave(td, ses->ses_fpu_ctx); return (error); } Modified: head/sys/i386/include/npx.h ============================================================================== --- head/sys/i386/include/npx.h Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/i386/include/npx.h Mon Jun 23 07:37:54 2014 (r267767) @@ -71,6 +71,7 @@ int is_fpu_kern_thread(u_int flags); */ #define FPU_KERN_NORMAL 0x0000 #define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 #endif Modified: head/sys/i386/isa/npx.c ============================================================================== --- head/sys/i386/isa/npx.c Mon Jun 23 07:03:47 2014 (r267766) +++ head/sys/i386/isa/npx.c Mon Jun 23 07:37:54 2014 (r267767) @@ -1008,6 +1008,7 @@ static MALLOC_DEFINE(M_FPUKERN_CTX, "fpu #define XSAVE_AREA_ALIGN 64 #define FPU_KERN_CTX_NPXINITDONE 0x01 +#define FPU_KERN_CTX_DUMMY 0x02 struct fpu_kern_ctx { union savefpu *prev; @@ -1051,6 +1052,10 @@ fpu_kern_enter(struct thread *td, struct { struct pcb *pcb; + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY; + return (0); + } pcb = td->td_pcb; KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == &pcb->pcb_user_save, ("mangled pcb_save")); @@ -1070,6 +1075,8 @@ fpu_kern_leave(struct thread *td, struct { struct pcb *pcb; + if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); pcb = td->td_pcb; critical_enter(); if (curthread == PCPU_GET(fpcurthread))