Date: Wed, 11 Sep 2019 23:45:58 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r352235 - in stable/12/sys: conf crypto/aesni modules/aesni Message-ID: <201909112345.x8BNjwFD034170@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mav Date: Wed Sep 11 23:45:58 2019 New Revision: 352235 URL: https://svnweb.freebsd.org/changeset/base/352235 Log: MFC r348268 (by sef), r348293 (by cem): Add an AESNI-optimized version of the CCM/CBC cryptographic and authentication code. The primary client of this is probably going to be ZFS encryption. Added: stable/12/sys/crypto/aesni/aesni_ccm.c - copied, changed from r348268, head/sys/crypto/aesni/aesni_ccm.c Modified: stable/12/sys/conf/files.amd64 stable/12/sys/conf/files.i386 stable/12/sys/crypto/aesni/aesni.c stable/12/sys/crypto/aesni/aesni.h stable/12/sys/crypto/aesni/aesni_wrap.c stable/12/sys/modules/aesni/Makefile Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/conf/files.amd64 ============================================================================== --- stable/12/sys/conf/files.amd64 Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/conf/files.amd64 Wed Sep 11 23:45:58 2019 (r352235) @@ -175,6 +175,11 @@ aesni_ghash.o optional aesni \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" +aesni_ccm.o optional aesni \ + dependency "$S/crypto/aesni/aesni_ccm.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ + no-implicit-rule \ + clean "aesni_ccm.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ Modified: stable/12/sys/conf/files.i386 ============================================================================== --- stable/12/sys/conf/files.i386 Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/conf/files.i386 Wed Sep 11 23:45:58 2019 (r352235) @@ -127,6 +127,11 @@ aesni_ghash.o optional aesni \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" +aesni_ccm.o optional aesni \ + dependency "$S/crypto/aesni/aesni_ccm.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ + no-implicit-rule \ + clean "aesni_ccm.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ Modified: stable/12/sys/crypto/aesni/aesni.c ============================================================================== --- stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 23:45:58 2019 (r352235) @@ -131,9 +131,10 @@ aesni_probe(device_t dev) return (EINVAL); } else if (has_aes && has_sha) device_set_desc(dev, - "AES-CBC,AES-XTS,AES-GCM,AES-ICM,SHA1,SHA256"); + "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256"); else if (has_aes) - device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM"); + device_set_desc(dev, + "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS"); else device_set_desc(dev, "SHA1,SHA256"); @@ -193,6 +194,8 @@ aesni_attach(device_t dev) crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0); } if (sc->has_sha) { crypto_register(sc->cid, CRYPTO_SHA1, 0, 0); @@ -226,6 +229,7 @@ aesni_newsession(device_t dev, crypto_session_t cses, struct aesni_session *ses; struct cryptoini *encini, *authini; bool gcm_hash, gcm; + bool cbc_hash, ccm; int error; KASSERT(cses != NULL, ("EDOOFUS")); @@ -242,10 +246,17 @@ aesni_newsession(device_t dev, crypto_session_t cses, encini = NULL; gcm = false; gcm_hash = false; + ccm = cbc_hash = false; + for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { case CRYPTO_AES_NIST_GCM_16: - gcm = true; + case CRYPTO_AES_CCM_16: + if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) { + gcm = true; + } else if (cri->cri_alg == CRYPTO_AES_CCM_16) { + ccm = true; + } /* FALLTHROUGH */ case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: @@ -258,6 +269,10 @@ aesni_newsession(device_t dev, crypto_session_t cses, } encini = cri; break; + case CRYPTO_AES_CCM_CBC_MAC: + cbc_hash = true; + authini = cri; + break; case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: @@ -265,7 +280,12 @@ aesni_newsession(device_t dev, crypto_session_t cses, * nothing to do here, maybe in the future cache some * values for GHASH */ + if (authini != NULL) { + CRYPTDEB("authini already set"); + return (EINVAL); + } gcm_hash = true; + authini = cri; break; case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: @@ -295,9 +315,16 @@ unhandled: * GMAC algorithms are only supported with simultaneous GCM. Likewise * GCM is not supported without GMAC. */ - if (gcm_hash != gcm) + if (gcm_hash != gcm) { + CRYPTDEB("gcm_hash != gcm"); return (EINVAL); + } + if (cbc_hash != ccm) { + CRYPTDEB("cbc_hash != ccm"); + return (EINVAL); + } + if (encini != NULL) ses->algo = encini->cri_alg; if (authini != NULL) @@ -338,6 +365,7 @@ aesni_process(device_t dev, struct cryptop *crp, int h for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: needauth = 1; /* FALLTHROUGH */ case CRYPTO_AES_CBC: @@ -353,6 +381,7 @@ aesni_process(device_t dev, struct cryptop *crp, int h case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: + case CRYPTO_AES_CCM_CBC_MAC: case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: @@ -647,6 +676,7 @@ aesni_cipher_process(struct aesni_session *ses, struct if (enccrd != NULL) { if ((enccrd->crd_alg == CRYPTO_AES_ICM || + enccrd->crd_alg == CRYPTO_AES_CCM_16 || enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) return (EINVAL); @@ -700,8 +730,9 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c int error, ivlen; bool encflag, allocated, authallocated; - KASSERT(ses->algo != CRYPTO_AES_NIST_GCM_16 || authcrd != NULL, - ("AES_NIST_GCM_16 must include MAC descriptor")); + KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 && + ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL, + ("AES_NIST_GCM_16/AES_CCM_16 must include MAC descriptor")); ivlen = 0; authbuf = NULL; @@ -711,7 +742,8 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c return (ENOMEM); authallocated = false; - if (ses->algo == CRYPTO_AES_NIST_GCM_16) { + if (ses->algo == CRYPTO_AES_NIST_GCM_16 || + ses->algo == CRYPTO_AES_CCM_16) { authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); if (authbuf == NULL) { error = ENOMEM; @@ -737,6 +769,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c ivlen = 8; break; case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: ivlen = 12; /* should support arbitarily larger */ break; } @@ -786,7 +819,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c case CRYPTO_AES_NIST_GCM_16: if (!encflag) crypto_copydata(crp->crp_flags, crp->crp_buf, - authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + authcrd->crd_inject, sizeof(tag), tag); else bzero(tag, sizeof tag); @@ -797,7 +830,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c if (authcrd != NULL) crypto_copyback(crp->crp_flags, crp->crp_buf, - authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + authcrd->crd_inject, sizeof(tag), tag); } else { if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, enccrd->crd_len, authcrd->crd_len, ivlen, @@ -805,9 +838,28 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c error = EBADMSG; } break; + case CRYPTO_AES_CCM_16: + if (!encflag) + crypto_copydata(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, sizeof(tag), tag); + else + bzero(tag, sizeof tag); + if (encflag) { + AES_CCM_encrypt(buf, buf, authbuf, iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds); + if (authcrd != NULL) + crypto_copyback(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, sizeof(tag), tag); + } else { + if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds)) + error = EBADMSG; + } + break; } - - if (allocated) + if (allocated && error == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); Modified: stable/12/sys/crypto/aesni/aesni.h ============================================================================== --- stable/12/sys/crypto/aesni/aesni.h Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/crypto/aesni/aesni.h Wed Sep 11 23:45:58 2019 (r352235) @@ -111,6 +111,15 @@ int AES_GCM_decrypt(const unsigned char *in, unsigned const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, const unsigned char *key, int nr); +/* CCM + CBC-MAC functions */ +void AES_CCM_encrypt(const unsigned char *in, unsigned char *out, + const unsigned char *addt, const unsigned char *ivec, + unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, + const unsigned char *key, int nr); +int AES_CCM_decrypt(const unsigned char *in, unsigned char *out, + const unsigned char *addt, const unsigned char *ivec, + const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, + const unsigned char *key, int nr); int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, int keylen); Copied and modified: stable/12/sys/crypto/aesni/aesni_ccm.c (from r348268, head/sys/crypto/aesni/aesni_ccm.c) ============================================================================== --- head/sys/crypto/aesni/aesni_ccm.c Sat May 25 07:26:30 2019 (r348268, copy source) +++ stable/12/sys/crypto/aesni/aesni_ccm.c Wed Sep 11 23:45:58 2019 (r352235) @@ -58,7 +58,7 @@ static inline __m128i xor_and_encrypt(__m128i a, __m128i b, const unsigned char *k, int nr) { - __m128 retval = _mm_xor_si128(a, b); + __m128i retval = _mm_xor_si128(a, b); retval = AESNI_ENC(retval, k, nr); return (retval); Modified: stable/12/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- stable/12/sys/crypto/aesni/aesni_wrap.c Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/crypto/aesni/aesni_wrap.c Wed Sep 11 23:45:58 2019 (r352235) @@ -446,6 +446,7 @@ aesni_cipher_setup_common(struct aesni_session *ses, c switch (ses->algo) { case CRYPTO_AES_ICM: case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: decsched = 0; /* FALLTHROUGH */ case CRYPTO_AES_CBC: Modified: stable/12/sys/modules/aesni/Makefile ============================================================================== --- stable/12/sys/modules/aesni/Makefile Wed Sep 11 23:41:09 2019 (r352234) +++ stable/12/sys/modules/aesni/Makefile Wed Sep 11 23:45:58 2019 (r352235) @@ -8,16 +8,20 @@ SRCS= aesni.c SRCS+= aeskeys_${MACHINE_CPUARCH}.S SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h -OBJS+= aesni_ghash.o aesni_wrap.o +OBJS+= aesni_ghash.o aesni_wrap.o aesni_ccm.o OBJS+= intel_sha1.o intel_sha256.o # Remove -nostdinc so we can get the intrinsics. aesni_ghash.o: aesni_ghash.c - # XXX - gcc won't understand -mpclmul ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC} ${CTFCONVERT_CMD} +aesni_ccm.o: aesni_ccm.c + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ + -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC} + ${CTFCONVERT_CMD} + aesni_wrap.o: aesni_wrap.c ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ -mmmx -msse -msse4 -maes ${.IMPSRC} @@ -35,6 +39,7 @@ intel_sha256.o: intel_sha256.c aesni_ghash.o: aesni.h aesni_wrap.o: aesni.h +aesni_ccm.o: aesni.h intel_sha1.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h intel_sha256.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201909112345.x8BNjwFD034170>