Date: Thu, 21 Aug 2014 20:36:23 +0000 (UTC) From: Randall Stewart <rrs@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r270302 - in projects/rrs_socrypto_tls: share/man/man4 share/man/man9 sys/amd64/conf sys/conf sys/crypto/aesni sys/crypto/via sys/modules/aesni sys/modules/crypto sys/opencrypto sys/sys Message-ID: <201408212036.s7LKaNEu073980@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rrs Date: Thu Aug 21 20:36:22 2014 New Revision: 270302 URL: http://svnweb.freebsd.org/changeset/base/270302 Log: This sync's down to my play branch JMG's crypto changes. He will be committing these through other channels some day so this branch will have to wait (if it ever goes anywhere) until those are sync'd down. Modified: projects/rrs_socrypto_tls/share/man/man4/crypto.4 projects/rrs_socrypto_tls/share/man/man9/crypto.9 projects/rrs_socrypto_tls/sys/amd64/conf/GENERIC projects/rrs_socrypto_tls/sys/conf/files projects/rrs_socrypto_tls/sys/conf/files.amd64 projects/rrs_socrypto_tls/sys/conf/files.i386 projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.c projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.h projects/rrs_socrypto_tls/sys/crypto/aesni/aesni_wrap.c projects/rrs_socrypto_tls/sys/crypto/via/padlock_hash.c projects/rrs_socrypto_tls/sys/modules/aesni/Makefile projects/rrs_socrypto_tls/sys/modules/crypto/Makefile projects/rrs_socrypto_tls/sys/opencrypto/crypto.c projects/rrs_socrypto_tls/sys/opencrypto/cryptodev.c projects/rrs_socrypto_tls/sys/opencrypto/cryptodev.h projects/rrs_socrypto_tls/sys/opencrypto/cryptosoft.c projects/rrs_socrypto_tls/sys/opencrypto/xform.c projects/rrs_socrypto_tls/sys/opencrypto/xform.h projects/rrs_socrypto_tls/sys/sys/systm.h Modified: projects/rrs_socrypto_tls/share/man/man4/crypto.4 ============================================================================== --- projects/rrs_socrypto_tls/share/man/man4/crypto.4 Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/share/man/man4/crypto.4 Thu Aug 21 20:36:22 2014 (r270302) @@ -1,8 +1,15 @@ -.\" $OpenBSD: crypto.4,v 1.4 2002/09/12 07:15:03 deraadt Exp $ +.\" $NetBSD: crypto.4,v 1.24 2014/01/27 21:23:59 pgoyette Exp $ .\" -.\" Copyright (c) 2001 Theo de Raadt +.\" Copyright (c) 2008 The NetBSD Foundation, Inc. +.\" Copyright (c) 2014 The FreeBSD Foundation .\" All rights reserved. .\" +.\" Portions of this documentation were writen by John-Mark Gurney +.\" under sponsorship from the FreeBSD Foundation. +.\" +.\" This code is derived from software contributed to The NetBSD Foundation +.\" by Coyote Point Systems, Inc. +.\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: @@ -123,3 +130,24 @@ The .Nm driver was imported to .Fx 5.0 . +.Sh BUGS +Error checking and reporting is weak. +.Pp +The values specified for symmetric-key key sizes to +.Dv CIOCGSESSION +must exactly match the values expected by +.Xr opencrypto 9 . +The output buffer and MAC buffers supplied to +.Dv CIOCCRYPT +must follow whether privacy or integrity algorithms were specified for +session: if you request a +.No non- Ns Dv NULL +algorithm, you must supply a suitably-sized buffer. +.Pp +The scheme for passing arguments for asymmetric requests is baroque. +.Pp +The naming inconsistency between +.Dv CRIOGET +and the various +.Dv CIOC Ns \&* +names is an unfortunate historical artifact. Modified: projects/rrs_socrypto_tls/share/man/man9/crypto.9 ============================================================================== --- projects/rrs_socrypto_tls/share/man/man9/crypto.9 Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/share/man/man9/crypto.9 Thu Aug 21 20:36:22 2014 (r270302) @@ -17,7 +17,7 @@ .\" .\" $FreeBSD$ .\" -.Dd April 18, 2014 +.Dd August 12, 2014 .Dt CRYPTO 9 .Os .Sh NAME @@ -175,17 +175,26 @@ Contains an algorithm identifier. Currently supported algorithms are: .Pp .Bl -tag -width ".Dv CRYPTO_RIPEMD160_HMAC" -compact +.It Dv CRYPTO_AES_128_NIST_GMAC +.It Dv CRYPTO_AES_192_NIST_GMAC +.It Dv CRYPTO_AES_256_NIST_GMAC .It Dv CRYPTO_AES_CBC +.It Dv CRYPTO_AES_ICM +.It Dv CRYPTO_AES_NIST_GCM_16 +.It Dv CRYPTO_AES_NIST_GMAC +.It Dv CRYPTO_AES_XTS .It Dv CRYPTO_ARC4 .It Dv CRYPTO_BLF_CBC .It Dv CRYPTO_CAMELLIA_CBC .It Dv CRYPTO_CAST_CBC +.It Dv CRYPTO_DEFLATE_COMP .It Dv CRYPTO_DES_CBC .It Dv CRYPTO_3DES_CBC -.It Dv CRYPTO_SKIPJACK_CBC .It Dv CRYPTO_MD5 .It Dv CRYPTO_MD5_HMAC .It Dv CRYPTO_MD5_KPDK +.It Dv CRYPTO_NULL_HMAC +.It Dv CRYPTO_NULL_CBC .It Dv CRYPTO_RIPEMD160_HMAC .It Dv CRYPTO_SHA1 .It Dv CRYPTO_SHA1_HMAC @@ -193,8 +202,7 @@ Currently supported algorithms are: .It Dv CRYPTO_SHA2_256_HMAC .It Dv CRYPTO_SHA2_384_HMAC .It Dv CRYPTO_SHA2_512_HMAC -.It Dv CRYPTO_NULL_HMAC -.It Dv CRYPTO_NULL_CBC +.It Dv CRYPTO_SKIPJACK_CBC .El .It Va cri_klen Specifies the length of the key in bits, for variable-size key Modified: projects/rrs_socrypto_tls/sys/amd64/conf/GENERIC ============================================================================== --- projects/rrs_socrypto_tls/sys/amd64/conf/GENERIC Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/amd64/conf/GENERIC Thu Aug 21 20:36:22 2014 (r270302) @@ -130,6 +130,11 @@ device aic # Adaptec 15[012]x SCSI ad device bt # Buslogic/Mylex MultiMaster SCSI adapters device isci # Intel C600 SAS controller +# crypto devices +device crypto +device cryptodev +device aesni + # ATA/SCSI peripherals device scbus # SCSI bus (required for ATA/SCSI) device ch # SCSI media changers Modified: projects/rrs_socrypto_tls/sys/conf/files ============================================================================== --- projects/rrs_socrypto_tls/sys/conf/files Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/conf/files Thu Aug 21 20:36:22 2014 (r270302) @@ -3141,6 +3141,7 @@ libkern/arc4random.c standard libkern/bcd.c standard libkern/bsearch.c standard libkern/crc32.c standard +libkern/explicit_bzero.c standard libkern/fnmatch.c standard libkern/iconv.c optional libiconv libkern/iconv_converter_if.m optional libiconv @@ -3833,6 +3834,8 @@ opencrypto/cryptodev.c optional cryptod opencrypto/cryptodev_if.m optional crypto opencrypto/cryptosoft.c optional crypto opencrypto/cryptodeflate.c optional crypto +opencrypto/gmac.c optional crypto +opencrypto/gfmult.c optional crypto opencrypto/rmd160.c optional crypto | ipsec opencrypto/skipjack.c optional crypto opencrypto/xform.c optional crypto Modified: projects/rrs_socrypto_tls/sys/conf/files.amd64 ============================================================================== --- projects/rrs_socrypto_tls/sys/conf/files.amd64 Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/conf/files.amd64 Thu Aug 21 20:36:22 2014 (r270302) @@ -130,6 +130,11 @@ amd64/pci/pci_cfgreg.c optional pci cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S optional zfs compile-with "${ZFS_S}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/aesni/aesni.c optional aesni +aesni_ghash.o optional aesni \ + dependency "$S/crypto/aesni/aesni_ghash.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mpclmul -msse4 -mmmx -msse -maes ${.IMPSRC}" \ + no-implicit-rule \ + clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \ Modified: projects/rrs_socrypto_tls/sys/conf/files.i386 ============================================================================== --- projects/rrs_socrypto_tls/sys/conf/files.i386 Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/conf/files.i386 Thu Aug 21 20:36:22 2014 (r270302) @@ -115,6 +115,11 @@ bf_enc.o optional crypto | ipsec \ no-implicit-rule crypto/aesni/aeskeys_i386.S optional aesni crypto/aesni/aesni.c optional aesni +aesni_ghash.o optional aesni \ + dependency "$S/crypto/aesni/aesni_ghash.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \ + no-implicit-rule \ + clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \ Modified: projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.c ============================================================================== --- projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.c Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.c Thu Aug 21 20:36:22 2014 (r270302) @@ -1,8 +1,12 @@ /*- * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> + * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship from the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -41,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include <sys/uio.h> #include <crypto/aesni/aesni.h> #include <cryptodev_if.h> +#include <opencrypto/gmac.h> struct aesni_softc { int32_t cid; @@ -56,7 +61,7 @@ static void aesni_freesession_locked(str static int aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini); static int aesni_cipher_process(struct aesni_session *ses, - struct cryptodesc *enccrd, struct cryptop *crp); + struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp); MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data"); @@ -79,12 +84,12 @@ aesni_probe(device_t dev) return (EINVAL); } - if ((cpu_feature & CPUID_SSE2) == 0) { - device_printf(dev, "No SSE2 support but AESNI!?!\n"); + if ((cpu_feature & CPUID2_SSE41) == 0) { + device_printf(dev, "No SSE4.1 support.\n"); return (EINVAL); } - device_set_desc_copy(dev, "AES-CBC,AES-XTS"); + device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM"); return (0); } @@ -106,6 +111,10 @@ aesni_attach(device_t dev) rw_init(&sc->lock, "aesni_lock"); crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); return (0); } @@ -144,8 +153,10 @@ aesni_newsession(device_t dev, uint32_t struct cryptoini *encini; int error; - if (sidp == NULL || cri == NULL) + if (sidp == NULL || cri == NULL) { + CRYPTDEB("no sidp or cri"); return (EINVAL); + } sc = device_get_softc(dev); ses = NULL; @@ -154,16 +165,30 @@ aesni_newsession(device_t dev, uint32_t switch (cri->cri_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_XTS: - if (encini != NULL) + case CRYPTO_AES_NIST_GCM_16: + if (encini != NULL) { + CRYPTDEB("encini already set"); return (EINVAL); + } encini = cri; break; + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + /* + * nothing to do here, maybe in the future cache some + * values for GHASH + */ + break; default: + CRYPTDEB("unhandled algorithm"); return (EINVAL); } } - if (encini == NULL) + if (encini == NULL) { + CRYPTDEB("no cipher"); return (EINVAL); + } rw_wlock(&sc->lock); /* @@ -195,6 +220,7 @@ aesni_newsession(device_t dev, uint32_t error = aesni_cipher_setup(ses, encini); if (error != 0) { + CRYPTDEB("setup failed"); rw_wlock(&sc->lock); aesni_freesession_locked(sc, ses); rw_wunlock(&sc->lock); @@ -248,11 +274,13 @@ aesni_process(device_t dev, struct crypt { struct aesni_softc *sc = device_get_softc(dev); struct aesni_session *ses = NULL; - struct cryptodesc *crd, *enccrd; - int error; + struct cryptodesc *crd, *enccrd, *authcrd; + int error, needauth; error = 0; enccrd = NULL; + authcrd = NULL; + needauth = 0; /* Sanity check. */ if (crp == NULL) @@ -273,11 +301,40 @@ aesni_process(device_t dev, struct crypt } enccrd = crd; break; + + case CRYPTO_AES_NIST_GCM_16: + if (enccrd != NULL) { + error = EINVAL; + goto out; + } + enccrd = crd; + needauth = 1; + break; + + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + if (authcrd != NULL) { + error = EINVAL; + goto out; + } + authcrd = crd; + needauth = 1; + break; + default: return (EINVAL); } } - if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { + + /* CBC & XTS can only handle full blocks for now */ + if ((enccrd->crd_alg == CRYPTO_AES_CBC || enccrd->crd_alg == + CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) { + error = EINVAL; + goto out; + } + + if (enccrd == NULL || (needauth && authcrd == NULL)) { error = EINVAL; goto out; } @@ -293,7 +350,7 @@ aesni_process(device_t dev, struct crypt goto out; } - error = aesni_cipher_process(ses, enccrd, crp); + error = aesni_cipher_process(ses, enccrd, authcrd, crp); if (error != 0) goto out; @@ -376,22 +433,49 @@ aesni_cipher_setup(struct aesni_session return (error); } +static void +printhexstr(uint8_t *ptr, int len) +{ + int i; + + for (i = 0; i < len; i++) + printf("%02hhx", ptr[i]); +} + +/* + * authcrd contains the associated date. + */ static int aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, - struct cryptop *crp) + struct cryptodesc *authcrd, struct cryptop *crp) { + uint8_t tag[GMAC_DIGEST_LEN]; struct thread *td; - uint8_t *buf; - int error, allocated; + uint8_t *buf, *authbuf; + int error, allocated, authallocated; + int ivlen, encflag; + int r; + + encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; buf = aesni_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) return (ENOMEM); + authbuf = NULL; + authallocated = 0; + if (authcrd != NULL) { + authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); + if (authbuf == NULL) { + error = ENOMEM; + goto out1; + } + } + td = curthread; error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); - if (error != 0) + if (error != 0) goto out1; if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { @@ -401,42 +485,109 @@ aesni_cipher_process(struct aesni_sessio goto out; } + /* XXX - validate that enccrd and authcrd have/use same key? */ + switch (enccrd->crd_alg) { + case CRYPTO_AES_CBC: + ivlen = 16; + break; + case CRYPTO_AES_XTS: + ivlen = 8; + break; + case CRYPTO_AES_NIST_GCM_16: + ivlen = 12; /* should support arbitarily larger */ + break; + } + + /* Setup ses->iv */ + bzero(ses->iv, sizeof ses->iv); + /*printf("crd_flags: %#x, ivlen: %d, iv: ", enccrd->crd_flags, ivlen);*/ if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) - bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); + bcopy(enccrd->crd_iv, ses->iv, ivlen); if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - if (ses->algo == CRYPTO_AES_CBC) { - aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, - enccrd->crd_len, buf, buf, ses->iv); - } else /* if (ses->algo == CRYPTO_AES_XTS) */ { - aesni_encrypt_xts(ses->rounds, ses->enc_schedule, - ses->xts_schedule, enccrd->crd_len, buf, buf, - ses->iv); - } + enccrd->crd_inject, ivlen, ses->iv); } else { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) - bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); + bcopy(enccrd->crd_iv, ses->iv, ivlen); else crypto_copydata(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - if (ses->algo == CRYPTO_AES_CBC) { + enccrd->crd_inject, ivlen, ses->iv); + } + /*printhexstr(ses->iv, ivlen); + printf("\n");*/ + + if (authcrd != NULL && !encflag) { + crypto_copydata(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + } else { + /*printf("ptag: "); + printhexstr(tag, sizeof tag); + printf("\n");*/ + bzero(tag, sizeof tag); + } + + /* Do work */ + switch (ses->algo) { + case CRYPTO_AES_CBC: + if (encflag) + aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, + enccrd->crd_len, buf, buf, ses->iv); + else aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, enccrd->crd_len, buf, ses->iv); - } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + break; + case CRYPTO_AES_XTS: + if (encflag) + aesni_encrypt_xts(ses->rounds, ses->enc_schedule, + ses->xts_schedule, enccrd->crd_len, buf, buf, + ses->iv); + else aesni_decrypt_xts(ses->rounds, ses->dec_schedule, ses->xts_schedule, enccrd->crd_len, buf, buf, ses->iv); + break; + case CRYPTO_AES_NIST_GCM_16: + /*printf("GCM: %d\n", encflag); + printf("buf(%d): ", enccrd->crd_len); + printhexstr(buf, enccrd->crd_len); + printf("\nauthbuf(%d): ", authcrd->crd_len); + printhexstr(authbuf, authcrd->crd_len); + printf("\niv: "); + printhexstr(ses->iv, ivlen); + printf("\ntag: "); + printhexstr(tag, 16); + printf("\nsched: "); + printhexstr(ses->enc_schedule, 16 * (ses->rounds + 1)); + printf("\n");*/ + if (encflag) + AES_GCM_encrypt(buf, buf, authbuf, ses->iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds); + else { + r = AES_GCM_decrypt(buf, buf, authbuf, ses->iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds); + /*printf("dec r: %d\n", r);*/ } + break; } + if (allocated) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); + + /* OpenBSD doesn't copy this back. Why not? */ if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, AES_BLOCK_LEN, ses->iv); + + if (authcrd != NULL) { + crypto_copyback(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + } + out: fpu_kern_leave(td, ses->fpu_ctx); out1: @@ -444,5 +595,7 @@ out1: bzero(buf, enccrd->crd_len); free(buf, M_AESNI); } + if (authallocated) + free(authbuf, M_AESNI); return (error); } Modified: projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.h ============================================================================== --- projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.h Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/crypto/aesni/aesni.h Thu Aug 21 20:36:22 2014 (r270302) @@ -96,6 +96,16 @@ void aesni_decrypt_xts(int rounds, const const void *tweak_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]); +/* GCM & GHASH functions */ +void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, + const unsigned char *addt, const unsigned char *ivec, + unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, + const unsigned char *key, int nr); +int AES_GCM_decrypt(const unsigned char *in, unsigned char *out, + const unsigned char *addt, const unsigned char *ivec, + unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, + const unsigned char *key, int nr); + int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, int keylen); uint8_t *aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, Modified: projects/rrs_socrypto_tls/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- projects/rrs_socrypto_tls/sys/crypto/aesni/aesni_wrap.c Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/crypto/aesni/aesni_wrap.c Thu Aug 21 20:36:22 2014 (r270302) @@ -3,8 +3,12 @@ * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> * Copyright 2012-2013 John-Mark Gurney <jmg@FreeBSD.org> + * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship from the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -29,14 +33,15 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/libkern.h> #include <sys/malloc.h> #include <sys/proc.h> #include <sys/systm.h> #include <crypto/aesni/aesni.h> - + +#include <opencrypto/gmac.h> + #include "aesencdec.h" MALLOC_DECLARE(M_AESNI); @@ -336,6 +341,7 @@ aesni_cipher_setup_common(struct aesni_s switch (ses->algo) { case CRYPTO_AES_CBC: + case CRYPTO_AES_NIST_GCM_16: switch (keylen) { case 128: ses->rounds = AES128_ROUNDS; @@ -347,6 +353,7 @@ aesni_cipher_setup_common(struct aesni_s ses->rounds = AES256_ROUNDS; break; default: + CRYPTDEB("invalid CBC/GCM key length"); return (EINVAL); } break; @@ -359,6 +366,7 @@ aesni_cipher_setup_common(struct aesni_s ses->rounds = AES256_ROUNDS; break; default: + CRYPTDEB("invalid XTS key length"); return (EINVAL); } break; @@ -368,7 +376,9 @@ aesni_cipher_setup_common(struct aesni_s aesni_set_enckey(key, ses->enc_schedule, ses->rounds); aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); - if (ses->algo == CRYPTO_AES_CBC) + + /* setup IV */ + if (ses->algo == CRYPTO_AES_CBC || ses->algo == CRYPTO_AES_NIST_GCM_16) arc4rand(ses->iv, sizeof(ses->iv), 0); else /* if (ses->algo == CRYPTO_AES_XTS) */ { aesni_set_enckey(key + keylen / 16, ses->xts_schedule, Modified: projects/rrs_socrypto_tls/sys/crypto/via/padlock_hash.c ============================================================================== --- projects/rrs_socrypto_tls/sys/crypto/via/padlock_hash.c Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/crypto/via/padlock_hash.c Thu Aug 21 20:36:22 2014 (r270302) @@ -75,7 +75,7 @@ struct padlock_sha_ctx { CTASSERT(sizeof(struct padlock_sha_ctx) <= sizeof(union authctx)); static void padlock_sha_init(struct padlock_sha_ctx *ctx); -static int padlock_sha_update(struct padlock_sha_ctx *ctx, uint8_t *buf, +static int padlock_sha_update(struct padlock_sha_ctx *ctx, const uint8_t *buf, uint16_t bufsize); static void padlock_sha1_final(uint8_t *hash, struct padlock_sha_ctx *ctx); static void padlock_sha256_final(uint8_t *hash, struct padlock_sha_ctx *ctx); @@ -83,16 +83,16 @@ static void padlock_sha256_final(uint8_t static struct auth_hash padlock_hmac_sha1 = { CRYPTO_SHA1_HMAC, "HMAC-SHA1", 20, SHA1_HASH_LEN, SHA1_HMAC_BLOCK_LEN, sizeof(struct padlock_sha_ctx), - (void (*)(void *))padlock_sha_init, - (int (*)(void *, uint8_t *, uint16_t))padlock_sha_update, + (void (*)(void *))padlock_sha_init, NULL, NULL, + (int (*)(void *, const uint8_t *, uint16_t))padlock_sha_update, (void (*)(uint8_t *, void *))padlock_sha1_final }; static struct auth_hash padlock_hmac_sha256 = { CRYPTO_SHA2_256_HMAC, "HMAC-SHA2-256", 32, SHA2_256_HASH_LEN, SHA2_256_HMAC_BLOCK_LEN, sizeof(struct padlock_sha_ctx), - (void (*)(void *))padlock_sha_init, - (int (*)(void *, uint8_t *, uint16_t))padlock_sha_update, + (void (*)(void *))padlock_sha_init, NULL, NULL, + (int (*)(void *, const uint8_t *, uint16_t))padlock_sha_update, (void (*)(uint8_t *, void *))padlock_sha256_final }; @@ -167,7 +167,7 @@ padlock_sha_init(struct padlock_sha_ctx } static int -padlock_sha_update(struct padlock_sha_ctx *ctx, uint8_t *buf, uint16_t bufsize) +padlock_sha_update(struct padlock_sha_ctx *ctx, const uint8_t *buf, uint16_t bufsize) { if (ctx->psc_size - ctx->psc_offset < bufsize) { Modified: projects/rrs_socrypto_tls/sys/modules/aesni/Makefile ============================================================================== --- projects/rrs_socrypto_tls/sys/modules/aesni/Makefile Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/modules/aesni/Makefile Thu Aug 21 20:36:22 2014 (r270302) @@ -7,13 +7,18 @@ SRCS= aesni.c SRCS+= aeskeys_${MACHINE_CPUARCH}.S SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h -OBJS+= aesni_wrap.o +OBJS+= aesni_ghash.o aesni_wrap.o # Remove -nostdinc so we can get the intrinsics. +aesni_ghash.o: aesni_ghash.c + # XXX - gcc won't understand -mpclmul + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ + -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC} + ${CTFCONVERT_CMD} + aesni_wrap.o: aesni_wrap.c ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ -mmmx -msse -maes ${.IMPSRC} ${CTFCONVERT_CMD} .include <bsd.kmod.mk> - Modified: projects/rrs_socrypto_tls/sys/modules/crypto/Makefile ============================================================================== --- projects/rrs_socrypto_tls/sys/modules/crypto/Makefile Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/modules/crypto/Makefile Thu Aug 21 20:36:22 2014 (r270302) @@ -18,6 +18,7 @@ SRCS += camellia.c camellia-api.c SRCS += des_ecb.c des_enc.c des_setkey.c SRCS += sha1.c sha2.c sha256c.c SRCS += siphash.c +SRCS += gmac.c gfmult.c SRCS += opt_param.h cryptodev_if.h bus_if.h device_if.h SRCS += opt_ddb.h Modified: projects/rrs_socrypto_tls/sys/opencrypto/crypto.c ============================================================================== --- projects/rrs_socrypto_tls/sys/opencrypto/crypto.c Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/opencrypto/crypto.c Thu Aug 21 20:36:22 2014 (r270302) @@ -316,8 +316,12 @@ driver_suitable(const struct cryptocap * /* See if all the algorithms are supported. */ for (cr = cri; cr; cr = cr->cri_next) - if (cap->cc_alg[cr->cri_alg] == 0) + if (cap->cc_alg[cr->cri_alg] == 0) { +#ifdef DEBUG + printf("cr->cri_alg: %d\n", cr->cri_alg); +#endif return 0; + } return 1; } @@ -421,9 +425,12 @@ crypto_newsession(u_int64_t *sid, struct (*sid) <<= 32; (*sid) |= (lid & 0xffffffff); cap->cc_sessions++; - } - } else + } else + CRYPTDEB("dev newsession failed"); + } else { + CRYPTDEB("no driver"); err = EINVAL; + } CRYPTO_DRIVER_UNLOCK(); return err; } Modified: projects/rrs_socrypto_tls/sys/opencrypto/cryptodev.c ============================================================================== --- projects/rrs_socrypto_tls/sys/opencrypto/cryptodev.c Thu Aug 21 20:35:39 2014 (r270301) +++ projects/rrs_socrypto_tls/sys/opencrypto/cryptodev.c Thu Aug 21 20:36:22 2014 (r270302) @@ -3,6 +3,11 @@ /*- * Copyright (c) 2001 Theo de Raadt * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -36,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include "opt_compat.h" +#include "opt_kdtrace.h" #include <sys/param.h> #include <sys/systm.h> @@ -54,10 +60,15 @@ __FBSDID("$FreeBSD$"); #include <sys/module.h> #include <sys/fcntl.h> #include <sys/bus.h> +#include <sys/sdt.h> #include <opencrypto/cryptodev.h> #include <opencrypto/xform.h> +SDT_PROVIDER_DECLARE(opencrypto); + +SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); + #ifdef COMPAT_FREEBSD32 #include <sys/mount.h> #include <compat/freebsd32/freebsd32.h> @@ -317,6 +328,8 @@ static int csefree(struct csession *); static int cryptodev_op(struct csession *, struct crypt_op *, struct ucred *, struct thread *td); +static int cryptodev_aead(struct csession *, struct crypt_aead *, + struct ucred *, struct thread *); static int cryptodev_key(struct crypt_kop *); static int cryptodev_find(struct crypt_find_op *); @@ -377,6 +390,7 @@ cryptof_ioctl( struct csession *cse; struct session_op *sop; struct crypt_op *cop; + struct crypt_aead *caead; struct enc_xform *txform = NULL; struct auth_hash *thash = NULL; struct crypt_kop *kop; @@ -437,7 +451,15 @@ cryptof_ioctl( case CRYPTO_CAMELLIA_CBC: txform = &enc_xform_camellia; break; + case CRYPTO_AES_ICM: + txform = &enc_xform_aes_icm; + break; + case CRYPTO_AES_NIST_GCM_16: + txform = &enc_xform_aes_nist_gcm; + break; + default: + CRYPTDEB("invalid cipher"); return (EINVAL); } @@ -462,6 +484,16 @@ cryptof_ioctl( case CRYPTO_RIPEMD160_HMAC: thash = &auth_hash_hmac_ripemd_160; break; + case CRYPTO_AES_128_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_128; + break; + case CRYPTO_AES_192_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_192; + break; + case CRYPTO_AES_256_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_256; + break; + #ifdef notdef case CRYPTO_MD5: thash = &auth_hash_md5; @@ -474,6 +506,7 @@ cryptof_ioctl( thash = &auth_hash_null; break; default: + CRYPTDEB("invalid mac"); return (EINVAL); } @@ -485,6 +518,7 @@ cryptof_ioctl( crie.cri_klen = sop->keylen * 8; if (sop->keylen > txform->maxkey || sop->keylen < txform->minkey) { + CRYPTDEB("invalid cipher parameters"); error = EINVAL; goto bail; } @@ -492,8 +526,10 @@ cryptof_ioctl( crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK); if ((error = copyin(sop->key, crie.cri_key, - crie.cri_klen / 8))) + crie.cri_klen / 8))) { + CRYPTDEB("invalid key"); goto bail; + } if (thash) crie.cri_next = &cria; } @@ -502,6 +538,7 @@ cryptof_ioctl( cria.cri_alg = thash->type; cria.cri_klen = sop->mackeylen * 8; if (sop->mackeylen != thash->keysize) { + CRYPTDEB("invalid mac key length"); error = EINVAL; goto bail; } @@ -510,8 +547,10 @@ cryptof_ioctl( cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA, M_WAITOK); if ((error = copyin(sop->mackey, cria.cri_key, - cria.cri_klen / 8))) + cria.cri_klen / 8))) { + CRYPTDEB("invalid mac key"); goto bail; + } } } @@ -523,13 +562,17 @@ cryptof_ioctl( ) { crid = SES2(sop)->crid; error = checkforsoftware(crid); - if (error) + if (error) { + CRYPTDEB("checkforsoftware"); goto bail; + } } else crid = CRYPTOCAP_F_HARDWARE; error = crypto_newsession(&sid, (txform ? &crie : &cria), crid); - if (error) + if (error) { + CRYPTDEB("crypto_newsession"); goto bail; + } cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen, cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform, @@ -538,6 +581,7 @@ cryptof_ioctl( if (cse == NULL) { crypto_freesession(sid); error = EINVAL; + CRYPTDEB("csecreate"); goto bail; } sop->ses = cse->ses; @@ -584,8 +628,10 @@ bail: #endif cop = (struct crypt_op *)data; cse = csefind(fcr, cop->ses); - if (cse == NULL) + if (cse == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); + } error = cryptodev_op(cse, cop, active_cred, td); #ifdef COMPAT_FREEBSD32 if (error == 0 && cmd == CIOCCRYPT32) @@ -639,6 +685,13 @@ bail: case CIOCFINDDEV: error = cryptodev_find((struct crypt_find_op *)data); break; + case CIOCCRYPTAEAD: + caead = (struct crypt_aead *)data; + cse = csefind(fcr, caead->ses); + if (cse == NULL) + return (EINVAL); + error = cryptodev_aead(cse, caead, active_cred, td); + break; default: error = EINVAL; break; @@ -661,12 +714,16 @@ cryptodev_op( struct cryptodesc *crde = NULL, *crda = NULL; int error; - if (cop->len > 256*1024-4) + if (cop->len > 256*1024-4) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (E2BIG); + } if (cse->txform) { - if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) + if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); + } } cse->uio.uio_iov = &cse->iovec; @@ -686,6 +743,7 @@ cryptodev_op( crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL)); if (crp == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = ENOMEM; goto bail; } @@ -698,13 +756,17 @@ cryptodev_op( if (cse->txform) crde = crp->crp_desc; else { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } } - if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len))) + if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, + cop->len))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } if (crda) { crda->crd_skip = 0; @@ -739,15 +801,20 @@ cryptodev_op( if (cop->iv) { if (crde == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } - if ((error = copyin(cop->iv, cse->tmp_iv, cse->txform->blocksize))) + if ((error = copyin(cop->iv, cse->tmp_iv, + cse->txform->blocksize))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } bcopy(cse->tmp_iv, crde->crd_iv, cse->txform->blocksize); crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; crde->crd_skip = 0; @@ -760,6 +827,7 @@ cryptodev_op( } if (cop->mac && crda == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } @@ -778,8 +846,10 @@ again: error = msleep(crp, &cse->lock, PWAIT, "crydev", 0); mtx_unlock(&cse->lock); - if (error != 0) + if (error != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } if (crp->crp_etype == EAGAIN) { crp->crp_etype = 0; @@ -788,23 +858,30 @@ again: } if (crp->crp_etype != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = crp->crp_etype; goto bail; } if (cse->error) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201408212036.s7LKaNEu073980>