From owner-dev-commits-src-branches@freebsd.org Mon Feb 15 19:24:02 2021 Return-Path: Delivered-To: dev-commits-src-branches@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 9DC7A536CEA; Mon, 15 Feb 2021 19:24:01 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4DfYtm2mwsz3FTq; Mon, 15 Feb 2021 19:24:00 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 8DADB231A3; Mon, 15 Feb 2021 19:23:59 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 11FJNxrE085338; Mon, 15 Feb 2021 19:23:59 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 11FJNxTQ085337; Mon, 15 Feb 2021 19:23:59 GMT (envelope-from git) Date: Mon, 15 Feb 2021 19:23:59 GMT Message-Id: <202102151923.11FJNxTQ085337@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mark Johnston Subject: git: 313cd76a2aac - stable/13 - armv8crypto: Extract GCM state into a structure MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: markj X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 313cd76a2aacdc37667cb9df34e027ba08a74f97 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-branches@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commits to the stable branches of the FreeBSD src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 15 Feb 2021 19:24:04 -0000 The branch stable/13 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=313cd76a2aacdc37667cb9df34e027ba08a74f97 commit 313cd76a2aacdc37667cb9df34e027ba08a74f97 Author: Mark Johnston AuthorDate: 2021-02-08 14:19:10 +0000 Commit: Mark Johnston CommitDate: 2021-02-15 19:13:06 +0000 armv8crypto: Extract GCM state into a structure This makes it easier to refactor the GCM code to operate on crypto_buffer_cursors rather than plain contiguous buffers, with the aim of minimizing the amount of copying and zeroing done today. No functional change intended. Reviewed by: jhb Sponsored by: Ampere Computing Submitted by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D28500 (cherry picked from commit 7509b677b413b9551c15b483ec2ed9ce655d2455) --- sys/crypto/armv8/armv8_crypto_wrap.c | 135 +++++++++++++++++------------------ 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c index 2f880258bf46..eb4a431d33e9 100644 --- a/sys/crypto/armv8/armv8_crypto_wrap.c +++ b/sys/crypto/armv8/armv8_crypto_wrap.c @@ -234,6 +234,14 @@ armv8_aes_decrypt_xts(AES_key_t *data_schedule, break; \ } while (0) +struct armv8_gcm_state { + __uint128_val_t EK0; + __uint128_val_t EKi; + __uint128_val_t Xi; + __uint128_val_t lenblock; + uint8_t aes_counter[AES_BLOCK_LEN]; +}; + void armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, const uint8_t *from, uint8_t *to, @@ -242,36 +250,34 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable) { - size_t i; + struct armv8_gcm_state s; const uint64_t *from64; uint64_t *to64; - uint8_t aes_counter[AES_BLOCK_LEN]; uint8_t block[AES_BLOCK_LEN]; - size_t trailer; - __uint128_val_t EK0, EKi, Xi, lenblock; + size_t i, trailer; - bzero(&aes_counter, AES_BLOCK_LEN); - memcpy(aes_counter, iv, AES_GCM_IV_LEN); + bzero(&s.aes_counter, AES_BLOCK_LEN); + memcpy(s.aes_counter, iv, AES_GCM_IV_LEN); /* Setup the counter */ - aes_counter[AES_BLOCK_LEN - 1] = 1; + s.aes_counter[AES_BLOCK_LEN - 1] = 1; /* EK0 for a final GMAC round */ - aes_v8_encrypt(aes_counter, EK0.c, aes_key); + aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key); /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ - aes_counter[AES_BLOCK_LEN - 1] = 2; + s.aes_counter[AES_BLOCK_LEN - 1] = 2; - memset(Xi.c, 0, sizeof(Xi.c)); + memset(s.Xi.c, 0, sizeof(s.Xi.c)); trailer = authdatalen % AES_BLOCK_LEN; if (authdatalen - trailer > 0) { - gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer); + gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer); authdata += authdatalen - trailer; } if (trailer > 0 || authdatalen == 0) { memset(block, 0, sizeof(block)); memcpy(block, authdata, trailer); - gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } from64 = (const uint64_t*)from; @@ -279,11 +285,11 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, trailer = len % AES_BLOCK_LEN; for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { - aes_v8_encrypt(aes_counter, EKi.c, aes_key); - AES_INC_COUNTER(aes_counter); - to64[0] = from64[0] ^ EKi.u[0]; - to64[1] = from64[1] ^ EKi.u[1]; - gcm_ghash_v8(Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN); + aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); + AES_INC_COUNTER(s.aes_counter); + to64[0] = from64[0] ^ s.EKi.u[0]; + to64[1] = from64[1] ^ s.EKi.u[1]; + gcm_ghash_v8(s.Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN); to64 += 2; from64 += 2; @@ -293,31 +299,27 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, from += (len - trailer); if (trailer) { - aes_v8_encrypt(aes_counter, EKi.c, aes_key); - AES_INC_COUNTER(aes_counter); + aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); + AES_INC_COUNTER(s.aes_counter); memset(block, 0, sizeof(block)); for (i = 0; i < trailer; i++) { - block[i] = to[i] = from[i] ^ EKi.c[i]; + block[i] = to[i] = from[i] ^ s.EKi.c[i]; } - gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } /* Lengths block */ - lenblock.u[0] = lenblock.u[1] = 0; - lenblock.d[1] = htobe32(authdatalen * 8); - lenblock.d[3] = htobe32(len * 8); - gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); - - Xi.u[0] ^= EK0.u[0]; - Xi.u[1] ^= EK0.u[1]; - memcpy(tag, Xi.c, GMAC_DIGEST_LEN); - - explicit_bzero(aes_counter, sizeof(aes_counter)); - explicit_bzero(Xi.c, sizeof(Xi.c)); - explicit_bzero(EK0.c, sizeof(EK0.c)); - explicit_bzero(EKi.c, sizeof(EKi.c)); - explicit_bzero(lenblock.c, sizeof(lenblock.c)); + s.lenblock.u[0] = s.lenblock.u[1] = 0; + s.lenblock.d[1] = htobe32(authdatalen * 8); + s.lenblock.d[3] = htobe32(len * 8); + gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN); + + s.Xi.u[0] ^= s.EK0.u[0]; + s.Xi.u[1] ^= s.EK0.u[1]; + memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN); + + explicit_bzero(&s, sizeof(s)); } int @@ -328,70 +330,68 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable) { - size_t i; + struct armv8_gcm_state s; const uint64_t *from64; uint64_t *to64; - uint8_t aes_counter[AES_BLOCK_LEN]; uint8_t block[AES_BLOCK_LEN]; - size_t trailer; - __uint128_val_t EK0, EKi, Xi, lenblock; + size_t i, trailer; int error; error = 0; - bzero(&aes_counter, AES_BLOCK_LEN); - memcpy(aes_counter, iv, AES_GCM_IV_LEN); + bzero(&s.aes_counter, AES_BLOCK_LEN); + memcpy(s.aes_counter, iv, AES_GCM_IV_LEN); /* Setup the counter */ - aes_counter[AES_BLOCK_LEN - 1] = 1; + s.aes_counter[AES_BLOCK_LEN - 1] = 1; /* EK0 for a final GMAC round */ - aes_v8_encrypt(aes_counter, EK0.c, aes_key); + aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key); - memset(Xi.c, 0, sizeof(Xi.c)); + memset(s.Xi.c, 0, sizeof(s.Xi.c)); trailer = authdatalen % AES_BLOCK_LEN; if (authdatalen - trailer > 0) { - gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer); + gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer); authdata += authdatalen - trailer; } if (trailer > 0 || authdatalen == 0) { memset(block, 0, sizeof(block)); memcpy(block, authdata, trailer); - gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } trailer = len % AES_BLOCK_LEN; if (len - trailer > 0) - gcm_ghash_v8(Xi.u, Htable, from, len - trailer); + gcm_ghash_v8(s.Xi.u, Htable, from, len - trailer); if (trailer > 0) { memset(block, 0, sizeof(block)); memcpy(block, from + len - trailer, trailer); - gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } /* Lengths block */ - lenblock.u[0] = lenblock.u[1] = 0; - lenblock.d[1] = htobe32(authdatalen * 8); - lenblock.d[3] = htobe32(len * 8); - gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); - - Xi.u[0] ^= EK0.u[0]; - Xi.u[1] ^= EK0.u[1]; - if (timingsafe_bcmp(tag, Xi.c, GMAC_DIGEST_LEN) != 0) { + s.lenblock.u[0] = s.lenblock.u[1] = 0; + s.lenblock.d[1] = htobe32(authdatalen * 8); + s.lenblock.d[3] = htobe32(len * 8); + gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN); + + s.Xi.u[0] ^= s.EK0.u[0]; + s.Xi.u[1] ^= s.EK0.u[1]; + if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) { error = EBADMSG; goto out; } /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ - aes_counter[AES_BLOCK_LEN - 1] = 2; + s.aes_counter[AES_BLOCK_LEN - 1] = 2; from64 = (const uint64_t*)from; to64 = (uint64_t*)to; for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { - aes_v8_encrypt(aes_counter, EKi.c, aes_key); - AES_INC_COUNTER(aes_counter); - to64[0] = from64[0] ^ EKi.u[0]; - to64[1] = from64[1] ^ EKi.u[1]; + aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); + AES_INC_COUNTER(s.aes_counter); + to64[0] = from64[0] ^ s.EKi.u[0]; + to64[1] = from64[1] ^ s.EKi.u[1]; to64 += 2; from64 += 2; } @@ -400,18 +400,13 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, from += (len - trailer); if (trailer) { - aes_v8_encrypt(aes_counter, EKi.c, aes_key); - AES_INC_COUNTER(aes_counter); + aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); + AES_INC_COUNTER(s.aes_counter); for (i = 0; i < trailer; i++) - to[i] = from[i] ^ EKi.c[i]; + to[i] = from[i] ^ s.EKi.c[i]; } out: - explicit_bzero(aes_counter, sizeof(aes_counter)); - explicit_bzero(Xi.c, sizeof(Xi.c)); - explicit_bzero(EK0.c, sizeof(EK0.c)); - explicit_bzero(EKi.c, sizeof(EKi.c)); - explicit_bzero(lenblock.c, sizeof(lenblock.c)); - + explicit_bzero(&s, sizeof(s)); return (error); }