From owner-svn-src-stable-8@FreeBSD.ORG Sun Nov 21 13:20:14 2010 Return-Path: Delivered-To: svn-src-stable-8@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 980F1106566B; Sun, 21 Nov 2010 13:20:14 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 84ED08FC19; Sun, 21 Nov 2010 13:20:14 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id oALDKENl058040; Sun, 21 Nov 2010 13:20:14 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id oALDKEKc058031; Sun, 21 Nov 2010 13:20:14 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201011211320.oALDKEKc058031@svn.freebsd.org> From: Konstantin Belousov Date: Sun, 21 Nov 2010 13:20:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-8@freebsd.org X-SVN-Group: stable-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r215633 - in stable/8/sys: conf crypto/aesni modules modules/aesni X-BeenThere: svn-src-stable-8@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for only the 8-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 21 Nov 2010 13:20:14 -0000 Author: kib Date: Sun Nov 21 13:20:14 2010 New Revision: 215633 URL: http://svn.freebsd.org/changeset/base/215633 Log: Bring aesni(4) driver from HEAD. MFC r210409: Crypto(4) driver for AESNI. MFC r213064 (by pjd): Simplify code a bit. MFC r213066 (by pjd): Add support for CRD_F_KEY_EXPLICIT flag. MFC r213069 (by pjd): Add support for CRYPTO_AES_XTS. MFC r213166 (by pjd): Fix two copy&paste bugs. MFC r213797 (by dim): Change two missed instances of 'retq' in aeskeys_i386.S to 'retl'. MFC r215427: Only save FPU context when not executing in the context of the crypto thread. Added: stable/8/sys/crypto/aesni/ - copied from r210409, head/sys/crypto/aesni/ stable/8/sys/modules/aesni/ - copied from r210409, head/sys/modules/aesni/ Modified: stable/8/sys/conf/files.amd64 stable/8/sys/conf/files.i386 stable/8/sys/crypto/aesni/aeskeys_i386.S stable/8/sys/crypto/aesni/aesni.c stable/8/sys/crypto/aesni/aesni.h stable/8/sys/crypto/aesni/aesni_wrap.c stable/8/sys/modules/Makefile stable/8/sys/modules/aesni/Makefile Directory Properties: stable/8/sys/ (props changed) stable/8/sys/amd64/include/xen/ (props changed) stable/8/sys/cddl/contrib/opensolaris/ (props changed) stable/8/sys/contrib/dev/acpica/ (props changed) stable/8/sys/contrib/pf/ (props changed) stable/8/sys/dev/xen/xenpci/ (props changed) Modified: stable/8/sys/conf/files.amd64 ============================================================================== --- stable/8/sys/conf/files.amd64 Sun Nov 21 13:11:50 2010 (r215632) +++ stable/8/sys/conf/files.amd64 Sun Nov 21 13:20:14 2010 (r215633) @@ -136,6 +136,10 @@ amd64/amd64/uma_machdep.c standard amd64/amd64/vm_machdep.c standard amd64/pci/pci_bus.c optional pci amd64/pci/pci_cfgreg.c optional pci +crypto/aesni/aesencdec_amd64.S optional aesni +crypto/aesni/aeskeys_amd64.S optional aesni +crypto/aesni/aesni.c optional aesni +crypto/aesni/aesni_wrap.c optional aesni crypto/blowfish/bf_enc.c optional crypto | ipsec crypto/des/des_enc.c optional crypto | ipsec | netsmb crypto/via/padlock.c optional padlock Modified: stable/8/sys/conf/files.i386 ============================================================================== --- stable/8/sys/conf/files.i386 Sun Nov 21 13:11:50 2010 (r215632) +++ stable/8/sys/conf/files.i386 Sun Nov 21 13:20:14 2010 (r215633) @@ -112,6 +112,10 @@ bf_enc.o optional crypto | ipsec \ dependency "$S/crypto/blowfish/arch/i386/bf_enc.S $S/crypto/blowfish/arch/i386/bf_enc_586.S $S/crypto/blowfish/arch/i386/bf_enc_686.S" \ compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS} ${WERROR} ${.IMPSRC}" \ no-implicit-rule +crypto/aesni/aesencdec_i386.S optional aesni +crypto/aesni/aeskeys_i386.S optional aesni +crypto/aesni/aesni.c optional aesni +crypto/aesni/aesni_wrap.c optional aesni crypto/des/arch/i386/des_enc.S optional crypto | ipsec | netsmb crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock Modified: stable/8/sys/crypto/aesni/aeskeys_i386.S ============================================================================== --- head/sys/crypto/aesni/aeskeys_i386.S Fri Jul 23 11:00:46 2010 (r210409) +++ stable/8/sys/crypto/aesni/aeskeys_i386.S Sun Nov 21 13:20:14 2010 (r215633) @@ -52,7 +52,7 @@ _key_expansion_256a: pxor %xmm1,%xmm0 movaps %xmm0,(%edx) addl $0x10,%edx - retq + retl .cfi_endproc END(_key_expansion_128) @@ -76,7 +76,7 @@ ENTRY(_key_expansion_192a) shufps $0b01001110,%xmm2,%xmm1 movaps %xmm1,0x10(%edx) addl $0x20,%edx - retq + retl .cfi_endproc END(_key_expansion_192a) Modified: stable/8/sys/crypto/aesni/aesni.c ============================================================================== --- head/sys/crypto/aesni/aesni.c Fri Jul 23 11:00:46 2010 (r210409) +++ stable/8/sys/crypto/aesni/aesni.c Sun Nov 21 13:20:14 2010 (r215633) @@ -69,14 +69,12 @@ aesni_identify(driver_t *drv, device_t p static int aesni_probe(device_t dev) { - char capp[32]; if ((cpu_feature2 & CPUID2_AESNI) == 0) { device_printf(dev, "No AESNI support.\n"); return (EINVAL); } - strlcpy(capp, "AES-CBC", sizeof(capp)); - device_set_desc_copy(dev, capp); + device_set_desc_copy(dev, "AES-CBC,AES-XTS"); return (0); } @@ -96,6 +94,7 @@ aesni_attach(device_t dev) rw_init(&sc->lock, "aesni_lock"); crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); return (0); } @@ -142,6 +141,7 @@ aesni_newsession(device_t dev, uint32_t for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_XTS: if (encini != NULL) return (EINVAL); encini = cri; @@ -174,6 +174,7 @@ aesni_newsession(device_t dev, uint32_t ses->used = 1; TAILQ_INSERT_TAIL(&sc->sessions, ses, next); rw_wunlock(&sc->lock); + ses->algo = encini->cri_alg; error = aesni_cipher_setup(ses, encini); if (error != 0) { @@ -245,6 +246,7 @@ aesni_process(device_t dev, struct crypt for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_XTS: if (enccrd != NULL) { error = EINVAL; goto out; Modified: stable/8/sys/crypto/aesni/aesni.h ============================================================================== --- head/sys/crypto/aesni/aesni.h Fri Jul 23 11:00:46 2010 (r210409) +++ stable/8/sys/crypto/aesni/aesni.h Sun Nov 21 13:20:14 2010 (r215633) @@ -55,7 +55,9 @@ struct aesni_session { uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16); uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16); + uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16); uint8_t iv[AES_BLOCK_LEN]; + int algo; int rounds; /* uint8_t *ses_ictx; */ /* uint8_t *ses_octx; */ Modified: stable/8/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- head/sys/crypto/aesni/aesni_wrap.c Fri Jul 23 11:00:46 2010 (r210409) +++ stable/8/sys/crypto/aesni/aesni_wrap.c Sun Nov 21 13:20:14 2010 (r215633) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2010 Konstantin Belousov + * Copyright (c) 2010 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -105,35 +106,161 @@ aesni_decrypt_ecb(int rounds, const void } } -int -aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) +#define AES_XTS_BLOCKSIZE 16 +#define AES_XTS_IVSIZE 8 +#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ + +static void +aesni_crypt_xts_block(int rounds, const void *key_schedule, uint8_t *tweak, + const uint8_t *from, uint8_t *to, int do_encrypt) { - struct thread *td; - int error; + uint8_t block[AES_XTS_BLOCKSIZE]; + u_int i, carry_in, carry_out; - switch (encini->cri_klen) { - case 128: - ses->rounds = AES128_ROUNDS; - break; - case 192: - ses->rounds = AES192_ROUNDS; + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + block[i] = from[i] ^ tweak[i]; + + if (do_encrypt) + aesni_enc(rounds - 1, key_schedule, block, to, NULL); + else + aesni_dec(rounds - 1, key_schedule, block, to, NULL); + + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + to[i] ^= tweak[i]; + + /* Exponentiate tweak. */ + carry_in = 0; + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { + carry_out = tweak[i] & 0x80; + tweak[i] = (tweak[i] << 1) | (carry_in ? 1 : 0); + carry_in = carry_out; + } + if (carry_in) + tweak[0] ^= AES_XTS_ALPHA; + bzero(block, sizeof(block)); +} + +static void +aesni_crypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) +{ + uint8_t tweak[AES_XTS_BLOCKSIZE]; + uint64_t blocknum; + size_t i; + + /* + * Prepare tweak as E_k2(IV). IV is specified as LE representation + * of a 64-bit block number which we allow to be passed in directly. + */ + bcopy(iv, &blocknum, AES_XTS_IVSIZE); + for (i = 0; i < AES_XTS_IVSIZE; i++) { + tweak[i] = blocknum & 0xff; + blocknum >>= 8; + } + /* Last 64 bits of IV are always zero. */ + bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); + aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL); + + len /= AES_XTS_BLOCKSIZE; + for (i = 0; i < len; i++) { + aesni_crypt_xts_block(rounds, data_schedule, tweak, from, to, + do_encrypt); + from += AES_XTS_BLOCKSIZE; + to += AES_XTS_BLOCKSIZE; + } + + bzero(tweak, sizeof(tweak)); +} + +static void +aesni_encrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN]) +{ + + aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 1); +} + +static void +aesni_decrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN]) +{ + + aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 0); +} + +static int +aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, + int keylen) +{ + + switch (ses->algo) { + case CRYPTO_AES_CBC: + switch (keylen) { + case 128: + ses->rounds = AES128_ROUNDS; + break; + case 192: + ses->rounds = AES192_ROUNDS; + break; + case 256: + ses->rounds = AES256_ROUNDS; + break; + default: + return (EINVAL); + } break; - case 256: - ses->rounds = AES256_ROUNDS; + case CRYPTO_AES_XTS: + switch (keylen) { + case 256: + ses->rounds = AES128_ROUNDS; + break; + case 512: + ses->rounds = AES256_ROUNDS; + break; + default: + return (EINVAL); + } break; default: return (EINVAL); } + aesni_set_enckey(key, ses->enc_schedule, ses->rounds); + aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); + if (ses->algo == CRYPTO_AES_CBC) + arc4rand(ses->iv, sizeof(ses->iv), 0); + else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_set_enckey(key + keylen / 16, ses->xts_schedule, + ses->rounds); + } + + return (0); +} + +int +aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) +{ + struct thread *td; + int error, saved_ctx; + td = curthread; - error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); + if (!is_fpu_kern_thread(0)) { + error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); + saved_ctx = 1; + } else { + error = 0; + saved_ctx = 0; + } if (error == 0) { - aesni_set_enckey(encini->cri_key, ses->enc_schedule, - ses->rounds); - aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, - ses->rounds); - arc4rand(ses->iv, sizeof(ses->iv), 0); - fpu_kern_leave(td, &ses->fpu_ctx); + error = aesni_cipher_setup_common(ses, encini->cri_key, + encini->cri_klen); + if (saved_ctx) + fpu_kern_leave(td, &ses->fpu_ctx); } return (error); } @@ -144,39 +271,61 @@ aesni_cipher_process(struct aesni_sessio { struct thread *td; uint8_t *buf; - int error, allocated; + int error, allocated, saved_ctx; buf = aesni_cipher_alloc(enccrd, crp, &allocated); - if (buf == NULL) { - error = ENOMEM; - goto out; - } + if (buf == NULL) + return (ENOMEM); td = curthread; - error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); - if (error != 0) - goto out1; + if (!is_fpu_kern_thread(0)) { + error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); + if (error != 0) + goto out; + saved_ctx = 1; + } else { + saved_ctx = 0; + error = 0; + } + + if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { + error = aesni_cipher_setup_common(ses, enccrd->crd_key, + enccrd->crd_klen); + if (error != 0) + goto out; + } if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - - aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, - enccrd->crd_len, buf, buf, ses->iv); + if (ses->algo == CRYPTO_AES_CBC) { + aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, + enccrd->crd_len, buf, buf, ses->iv); + } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_encrypt_xts(ses->rounds, ses->enc_schedule, + ses->xts_schedule, enccrd->crd_len, buf, buf, + ses->iv); + } } else { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); else crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, - enccrd->crd_len, buf, ses->iv); + if (ses->algo == CRYPTO_AES_CBC) { + aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, + enccrd->crd_len, buf, ses->iv); + } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_decrypt_xts(ses->rounds, ses->dec_schedule, + ses->xts_schedule, enccrd->crd_len, buf, buf, + ses->iv); + } } - fpu_kern_leave(td, &ses->fpu_ctx); + if (saved_ctx) + fpu_kern_leave(td, &ses->fpu_ctx); if (allocated) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); @@ -184,11 +333,10 @@ aesni_cipher_process(struct aesni_sessio crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, AES_BLOCK_LEN, ses->iv); - out1: + out: if (allocated) { bzero(buf, enccrd->crd_len); free(buf, M_AESNI); } - out: return (error); } Modified: stable/8/sys/modules/Makefile ============================================================================== --- stable/8/sys/modules/Makefile Sun Nov 21 13:11:50 2010 (r215632) +++ stable/8/sys/modules/Makefile Sun Nov 21 13:20:14 2010 (r215633) @@ -10,6 +10,7 @@ SUBDIR= ${_3dfx} \ accf_http \ ${_acpi} \ ae \ + ${_aesni} \ age \ ${_agp} \ aha \ @@ -433,6 +434,9 @@ _zfs= zfs .if ${MACHINE} == "i386" _aac= aac _acpi= acpi +.if ${MK_CRYPT} != "no" || defined(ALL_MODULES) +_aesni= aesni +.endif _ahb= ahb _amdsbwd= amdsbwd _amdtemp= amdtemp @@ -489,6 +493,9 @@ _snc= snc .if ${MACHINE_ARCH} == "amd64" _aac= aac _acpi= acpi +.if ${MK_CRYPT} != "no" || defined(ALL_MODULES) +_aesni= aesni +.endif _agp= agp _an= an _amdsbwd= amdsbwd Modified: stable/8/sys/modules/aesni/Makefile ============================================================================== --- head/sys/modules/aesni/Makefile Fri Jul 23 11:00:46 2010 (r210409) +++ stable/8/sys/modules/aesni/Makefile Sun Nov 21 13:20:14 2010 (r215633) @@ -4,7 +4,7 @@ KMOD= aesni SRCS= aesni.c aesni_wrap.c -SRCS+= aesencdec_$(MACHINE_CPUARCH).S aeskeys_$(MACHINE_CPUARCH).S +SRCS+= aesencdec_$(MACHINE_ARCH).S aeskeys_$(MACHINE_ARCH).S SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h .include