From owner-svn-src-projects@FreeBSD.ORG Tue Dec 23 20:45:40 2008 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 7B56C1065674; Tue, 23 Dec 2008 20:45:40 +0000 (UTC) (envelope-from sam@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 67F4C8FC23; Tue, 23 Dec 2008 20:45:40 +0000 (UTC) (envelope-from sam@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id mBNKjeLs013228; Tue, 23 Dec 2008 20:45:40 GMT (envelope-from sam@svn.freebsd.org) Received: (from sam@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id mBNKjei3013222; Tue, 23 Dec 2008 20:45:40 GMT (envelope-from sam@svn.freebsd.org) Message-Id: <200812232045.mBNKjei3013222@svn.freebsd.org> From: Sam Leffler Date: Tue, 23 Dec 2008 20:45:40 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r186459 - in projects/cambria/sys/arm: conf xscale/ixp425 X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 23 Dec 2008 20:45:40 -0000 Author: sam Date: Tue Dec 23 20:45:40 2008 New Revision: 186459 URL: http://svn.freebsd.org/changeset/base/186459 Log: checkpoint NPE crypto support; this is just a shell of the driver but compiles and doesn't affect the system so commit it to simplify merges Added: projects/cambria/sys/arm/xscale/ixp425/ixp4xx_crypto.c (contents, props changed) Modified: projects/cambria/sys/arm/conf/AVILA projects/cambria/sys/arm/conf/AVILA.hints projects/cambria/sys/arm/conf/CAMBRIA (contents, props changed) projects/cambria/sys/arm/conf/CAMBRIA.hints (contents, props changed) projects/cambria/sys/arm/xscale/ixp425/files.ixp425 Modified: projects/cambria/sys/arm/conf/AVILA ============================================================================== --- projects/cambria/sys/arm/conf/AVILA Tue Dec 23 20:43:42 2008 (r186458) +++ projects/cambria/sys/arm/conf/AVILA Tue Dec 23 20:45:40 2008 (r186459) @@ -83,6 +83,10 @@ device ata device atadisk # ATA disk drives device avila_ata # Gateworks CF/IDE support +device ixpcrypto # requires npe + qmgr +device crypto +device cryptodev + device npe # Network Processing Engine device npe_fw device firmware Modified: projects/cambria/sys/arm/conf/AVILA.hints ============================================================================== --- projects/cambria/sys/arm/conf/AVILA.hints Tue Dec 23 20:43:42 2008 (r186458) +++ projects/cambria/sys/arm/conf/AVILA.hints Tue Dec 23 20:45:40 2008 (r186459) @@ -29,6 +29,9 @@ hint.npe.1.mac="C" hint.npe.1.mii="B" hint.npe.1.phy=1 +# NPE crypto acceleration +hint.ixpcrypto.0.at="ixp0" + # CF IDE controller hint.ata_avila.0.at="ixp0" Modified: projects/cambria/sys/arm/conf/CAMBRIA ============================================================================== --- projects/cambria/sys/arm/conf/CAMBRIA Tue Dec 23 20:43:42 2008 (r186458) +++ projects/cambria/sys/arm/conf/CAMBRIA Tue Dec 23 20:45:40 2008 (r186459) @@ -84,6 +84,10 @@ device ata device atadisk # ATA disk drives device avila_ata # Gateworks CF/IDE support +device ixpcrypto # requires npe + qmgr +device crypto +device cryptodev + device npe # Network Processing Engine device npe_fw device firmware @@ -102,7 +106,7 @@ device random # Entrop # NB: 2 USB 2.0 ports standard device usb options USB_EHCI_BIG_ENDIAN_DESC # handle big-endian byte order -#options USB_DEBUG +options USB_DEBUG device ehci device ugen device umass Modified: projects/cambria/sys/arm/conf/CAMBRIA.hints ============================================================================== --- projects/cambria/sys/arm/conf/CAMBRIA.hints Tue Dec 23 20:43:42 2008 (r186458) +++ projects/cambria/sys/arm/conf/CAMBRIA.hints Tue Dec 23 20:45:40 2008 (r186459) @@ -27,6 +27,9 @@ hint.npe.0.phy=1 #hint.npe.1.mii="C" #hint.npe.1.phy=2 +# NPE crypto acceleration +hint.ixpcrypto.0.at="ixp0" + # CF IDE controller hint.ata_avila.0.at="ixp0" Modified: projects/cambria/sys/arm/xscale/ixp425/files.ixp425 ============================================================================== --- projects/cambria/sys/arm/xscale/ixp425/files.ixp425 Tue Dec 23 20:43:42 2008 (r186458) +++ projects/cambria/sys/arm/xscale/ixp425/files.ixp425 Tue Dec 23 20:45:40 2008 (r186459) @@ -44,5 +44,6 @@ IxNpeMicrocode.dat optional npe_fw \ # Q-Manager support # arm/xscale/ixp425/ixp425_qmgr.c optional qmgr +arm/xscale/ixp425/ixp4xx_crypto.c optional ixpcrypto # -arm/xscale/ixp425/ixp435_ehci.c optional ehci +dev/usb/ehci_ixp4xx.c optional ehci Added: projects/cambria/sys/arm/xscale/ixp425/ixp4xx_crypto.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/cambria/sys/arm/xscale/ixp425/ixp4xx_crypto.c Tue Dec 23 20:45:40 2008 (r186459) @@ -0,0 +1,722 @@ +/*- + * Copyright (c) 2008 Sam Leffler. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include "cryptodev_if.h" + +#include +#include +#include +#include + +/* + * NPE crypto hw accelration Operation Opcode definition, by default + * the transfer mode = 0 (in-place) and Crypt direction = 0 (decrypt) + * + * NPE Crypto Hw Acceleration Operation Opcode bit field definition (pls + * refer to IxCryptoNpeOperationMode) for details : + * + * DATH CM0Vb + * D : Cryption Direction + * A : Hmac Enable + * T : Transfer Mode + * H : Hash Enable + * C : Crypt Enable + * M : CCM Enable + * V : Verification enable / Key Generation enable + */ +#define NPE_OP_HASH_GEN_ICV 0x50 /* hash */ +#define NPE_OP_HASH_VER_ICV 0x51 /* hash + ICV verification */ +#define NPE_OP_HMAC_GEN_ICV 0x10 /* HMAC + ICV generation */ +#define NPE_OP_HMAC_VER_ICV 0x11 /* HMAC + ICV verification */ +#define NPE_OP_CRYPT 0x48 /* Cryption operation */ +#define NPE_OP_CCM_GEN_MIC 0xCC /* CCM Encrypt with MIC Gen */ +#define NPE_OP_CCM_VER_MIC 0x4D /* CCM Decrypt with MIC Ver */ +#define NPE_OP_ENC_GEN_KEY 0xC9 /* Reverse AES key generation */ +#define NPE_OP_ENC_HMAC_GEN_ICV 0x98 /* Encrypt + HMAC */ +#define NPE_OP_HMAC_VER_ICV_DEC 0x19 /* reverse HMAC + Decrypt */ +/* modifiers or'd in */ +#define NPE_OP_CCM_ENA 0x04 /* CCM Enable */ +#define NPE_OP_CRYPT_ENA 0x08 /* Crypt enable mask */ +#define NPE_OP_TRANSFER_MODE 0x20 /* Transfer mode mask */ +#define NPE_OP_HMAC_DISABLE 0x40 /* HMAC enable mask */ +#define NPE_OP_CRYPT_DIR 0x80 /* Crypt direction mask */ + +#define MOD_ECB 0x0000 +#define MOD_CTR 0x1000 +#define MOD_CBC_ENC 0x2000 +#define MOD_CBC_DEC 0x3000 +#define MOD_CCM_ENC 0x4000 +#define MOD_CCM_DEC 0x5000 + +#define CIPH_DECR 0x0000 +#define CIPH_ENCR 0x0400 + +#define MOD_DES 0x0000 +#define MOD_TDEA2 0x0100 +#define MOD_3DES 0x0200 +#define MOD_AES 0x0800 +#define MOD_AES128 (0x0800 | 4) /* 128-bit key = 4 words */ +#define MOD_AES192 (0x0900 | 6) /* 192-bit key = 6 words */ +#define MOD_AES256 (0x0a00 | 8) /* 256-bit key = 8 words */ + +#define CTL_FLAG_UNUSED 0x0000 +#define CTL_FLAG_USED 0x1000 +#define CTL_FLAG_PERFORM_ABLK 0x0001 +#define CTL_FLAG_GEN_ICV 0x0002 +#define CTL_FLAG_GEN_REVAES 0x0004 +#define CTL_FLAG_PERFORM_AEAD 0x0008 +#define CTL_FLAG_MASK 0x000f + +#define NPE_MAXSEG 3 /* empirically selected */ + +struct npehwbuf { + struct { /* NPE shared area, cacheline aligned */ + uint32_t next; /* phys addr of next segment */ + uint32_t len; /* buffer/segment length (bytes) */ + uint32_t data; /* phys addr of data segment */ + uint32_t pad[5]; /* pad to cacheline */ + } ne[NPE_MAXSEG]; +}; + +#define NPE_QM_Q_ADDR(e) ((e)&0xffffffff8) /* phys address */ +#define NPE_QM_Q_OK(e) (((e)&1) == 0) /* cmd status */ + +struct npehwctx { /* h/w crypto context */ + uint8_t op; /* npe operation */ + uint8_t init_len; + uint16_t pad; + uint8_t iv[16]; /* IV for CBC or CTR IV for CTR */ + union { + uint32_t icvAddr; /* address for ICV */ + uint32_t revAesKeyaddr; /* address for Rev AES key */ + } u; +#define icv u.icvAddr +#define rev_aes u.revAesKeyaddr + uint32_t src; /* phys addr of src data */ + uint32_t dst; /* phys addr of dst data */ + uint16_t hash_off; /* authentication start offset */ + uint16_t hash_len; /* authentication data length */ + uint16_t cipher_off; /* cipher start offset */ + uint16_t cipher_len; /* cipher data length */ + uint32_t aad_addr; /* Additional Auth Data addr for CCM */ + uint32_t ctx; /* phys addr of NPE crypto context */ +}; + +struct ixpcrypto_session { + TAILQ_ENTRY(ixpcrypto_session) next; + uint32_t id; + int inuse; + uint8_t authkey[32]; + int authkey_len; + uint8_t cipherkey[32]; + int cipherkey_len; + uint8_t iv[16]; +}; + +struct npebuf { + struct npebuf *next; /* chain to next buffer */ + bus_dmamap_t map; /* bus dma map for associated data */ + struct npehwbuf *hw; /* associated h/w block */ + uint32_t neaddr; /* phys address of hw->ne */ + struct cryptop *crp; /* associated crypto operation */ +}; + +struct ixpcrypto_softc { + device_t dev; + int debug; /* debug msg flags */ + int32_t cid; /* crypto driver id */ + uint32_t sid; /* next available session id */ + struct rwlock sessions_lock; /* lock over session table */ + TAILQ_HEAD(ixpcrypto_sessions_head, ixpcrypto_session) sessions; + struct ixpnpe_softc *npe; /* handle on NPE engine */ + bus_dma_tag_t dtag; /* bus dma tag for mapped data */ + struct npehwbuf *hwbuf; /* NPE h/w buffers */ + bus_dma_tag_t buf_tag; /* tag+map for NPE cmd buffers */ + bus_dmamap_t buf_map; + bus_addr_t buf_phys; /* phys addr of h/w buffers */ + struct npebuf *buf; /* cmd buffers (1-1 w/ h/w) */ + struct npebuf *free; /* list of free cmd buffers */ + struct mtx mtx; /* lock over cmd buffer list */ + int cmd_qid; /* qid for submitting cmds */ + int cmddone_qid; /* qid cmds return on */ +}; + +SYSCTL_NODE(_hw, OID_AUTO, ixpcrypto, CTLFLAG_RD, 0, + "IXP4XX Crypto driver parameters"); + +static int ixpcrypto_debug = 0; +SYSCTL_INT(_hw_ixpcrypto, OID_AUTO, debug, CTLFLAG_RW, &ixpcrypto_debug, + 0, "IXP4XX Crypto debug msgs"); +TUNABLE_INT("hw.ixpcrypto.npe", &ixpcrypto_debug); +#define DPRINTF(sc, fmt, ...) do { \ + if (sc->debug) device_printf(sc->dev, fmt, __VA_ARGS__); \ +} while (0) +#define DPRINTFn(n, sc, fmt, ...) do { \ + if (sc->debug >= n) device_printf(sc->dev, fmt, __VA_ARGS__); \ +} while (0) + +static int ixpcrypto_cmdbuf = 64; /* # cmd buffers to allocate */ +SYSCTL_INT(_hw_ixpcrypto, OID_AUTO, cmdbuf, CTLFLAG_RD, &ixpcrypto_cmdbuf, + 0, "cmd buffers allocated"); +TUNABLE_INT("hw.ixpcrypto.cmdbuf", &ixpcrypto_cmdbuf); + +static int ixpcrypto_dma_setup(struct ixpcrypto_softc *); +static void ixpcrypto_dma_destroy(struct ixpcrypto_softc *); +static int ixpcrypto_newsession(device_t, uint32_t *, struct cryptoini *); +static int ixpcrypto_freesession(device_t, uint64_t); +static void ixpcrypto_freesession_locked(struct ixpcrypto_softc *, + struct ixpcrypto_session *); +static int ixpcrypto_process(device_t, struct cryptop *, int hint __unused); +static void ixpcrypto_cmddone(int qid, void *arg); + +MALLOC_DEFINE(M_IXPCRYPTO, "ixpcrypto_data", "IXP Crypto Data"); + +static int +ixpcrypto_probe(device_t dev) +{ + int unit = device_get_unit(dev); + + /* NB: this assumes we'll load firmware w/ crypto support */ + if (unit != 0 || (ixp4xx_read_feature_bits() & EXP_FCTRL_NPEC) == 0) + return EINVAL; + device_set_desc_copy(dev, "IXP4XX Crypto"); + return 0; +} + +static int +ixpcrypto_attach(device_t dev) +{ + struct ixpcrypto_softc *sc = device_get_softc(dev); + int error; + + sc->dev = dev; + sc->debug = ixpcrypto_debug; + + error = ixpcrypto_dma_setup(sc); + if (error != 0) { + device_printf(dev, "cannot setup dma (error %d)\n", error); + return error; + } + + sc->npe = ixpnpe_attach(dev, NPE_C); + if (sc->npe == NULL) { + device_printf(dev, "cannot attach ixpnpe\n"); + error = EIO; /* XXX */ + goto bad; + } + error = ixpnpe_init(sc->npe); + if (error != 0) { + device_printf(dev, "cannot init NPE (error %d)\n", error); + goto bad; + } + + sc->cmd_qid = 29; + ixpqmgr_qconfig(sc->cmd_qid, ixpcrypto_cmdbuf, 0, + ixpcrypto_cmdbuf, 0, NULL, sc); + + sc->cmddone_qid = 30; + KASSERT(ixpcrypto_cmdbuf > 2*4, ("%d cmd buffers", ixpcrypto_cmdbuf)); + ixpqmgr_qconfig(sc->cmddone_qid, ixpcrypto_cmdbuf/4, 0, 2, + IX_QMGR_Q_SOURCE_ID_NOT_E, ixpcrypto_cmddone, sc); + + sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); + if (sc->cid < 0) { + device_printf(dev, "Could not get crypto driver id.\n"); + error = ENOMEM; + goto bad; + } + + rw_init(&sc->sessions_lock, "ixpcrypto_lock"); + TAILQ_INIT(&sc->sessions); + sc->sid = 1; + + if (ixp4xx_read_feature_bits() & EXP_FCTRL_DES) { + crypto_register(sc->cid, CRYPTO_DES_CBC, 0, 0); + crypto_register(sc->cid, CRYPTO_3DES_CBC, 0, 0); + } + if (ixp4xx_read_feature_bits() & EXP_FCTRL_AES) { + crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); +#ifdef CRYPTO_AES_CTR_CBC + crypto_register(sc->cid, CRYPTO_AES_CTR_CBC, 0, 0); +#endif + } + if (ixp4xx_read_feature_bits() & EXP_FCTRL_HASH) { + crypto_register(sc->cid, CRYPTO_MD5_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); + } +#if 0 + /* XXX needs s/w assist */ + crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0); +#endif + return 0; +bad: + if (sc->npe != NULL) + ixpnpe_detach(sc->npe); + ixpcrypto_dma_destroy(sc); + return error; +} + +static int +ixpcrypto_detach(device_t dev) +{ + struct ixpcrypto_softc *sc = device_get_softc(dev); + struct ixpcrypto_session *ses; + + rw_wlock(&sc->sessions_lock); + TAILQ_FOREACH(ses, &sc->sessions, next) { + if (ses->inuse) { + rw_wunlock(&sc->sessions_lock); + device_printf(dev, + "Cannot detach, sessions still active.\n"); + return EBUSY; + } + } + while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) { + TAILQ_REMOVE(&sc->sessions, ses, next); + free(ses, M_IXPCRYPTO); + } + rw_destroy(&sc->sessions_lock); + + crypto_unregister_all(sc->cid); +#if 0 + ixpnpe_stop(sc->npe); +#endif + ixpnpe_detach(sc->npe); + ixpcrypto_dma_destroy(sc); + return 0; +} + +static void +npe_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + if (error == 0) + ((struct ixpcrypto_softc *)arg)->buf_phys = segs[0].ds_addr; +} + +static int +ixpcrypto_dma_setup(struct ixpcrypto_softc *sc) +{ + int error, i; + + mtx_init(&sc->mtx, "ixpcrypto", NULL, MTX_DEF); + + /* DMA tag for mapped mbufs */ + error = bus_dma_tag_create(bus_get_dma_tag(device_get_parent(sc->dev)), + 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, + /* XXX 64K */ + 64*1024, NPE_MAXSEG, 64*1024, 0, NULL, NULL, &sc->dtag); + if (error != 0) { + device_printf(sc->dev, "unable to create mbuf dma tag, " + "error %u\n", error); + return error; + } + + /* DMA tag and map for the NPE buffers */ + error = bus_dma_tag_create(bus_get_dma_tag(device_get_parent(sc->dev)), + sizeof(uint32_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, + NULL, NULL, + ixpcrypto_cmdbuf * sizeof(struct npehwbuf), 1, + ixpcrypto_cmdbuf * sizeof(struct npehwbuf), 0, + NULL, NULL, &sc->buf_tag); + if (error != 0) { + device_printf(sc->dev, + "unable to create npebuf dma tag, error %u\n", error); + return error; + } + /* XXX COHERENT for now */ + if (bus_dmamem_alloc(sc->buf_tag, (void **)&sc->hwbuf, + BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &sc->buf_map) != 0) { + device_printf(sc->dev, + "unable to allocate memory for h/w buffers, error %u\n", + error); + return error; + } + sc->buf = malloc(ixpcrypto_cmdbuf * sizeof(struct npebuf), + M_IXPCRYPTO, M_NOWAIT | M_ZERO); + if (sc->buf == NULL) { + device_printf(sc->dev, + "unable to allocate memory for s/w buffers\n"); + return error; + } + if (bus_dmamap_load(sc->buf_tag, sc->buf_map, + sc->hwbuf, ixpcrypto_cmdbuf*sizeof(struct npehwbuf), npe_getaddr, sc, 0) != 0) { + device_printf(sc->dev, + "unable to map memory for h/w buffers, error %u\n", error); + return error; + } + /* NB: sc->buf_phys set by npe_getaddr */ + for (i = 0; i < ixpcrypto_cmdbuf; i++) { + struct npebuf *npe = &sc->buf[i]; + struct npehwbuf *hw = &sc->hwbuf[i]; + + /* calculate offset to shared area */ + npe->neaddr = sc->buf_phys + + ((uintptr_t)hw - (uintptr_t)sc->hwbuf); + KASSERT((npe->neaddr & 0x1f) == 0, + ("ixpbuf misaligned, PA 0x%x", npe->neaddr)); + error = bus_dmamap_create(sc->dtag, BUS_DMA_NOWAIT, &npe->map); + if (error != 0) { + device_printf(sc->dev, + "unable to create dmamap for buffer %u, " + "error %u\n", i, error); + return error; + } + npe->hw = hw; + + npe->next = sc->free; + sc->free = npe; + } + bus_dmamap_sync(sc->buf_tag, sc->buf_map, BUS_DMASYNC_PREWRITE); + return 0; +} + +static void +ixpcrypto_dma_destroy(struct ixpcrypto_softc *sc) +{ + int i; + + if (sc->hwbuf != NULL) { + for (i = 0; i < ixpcrypto_cmdbuf; i++) { + struct npebuf *npe = &sc->buf[i]; + bus_dmamap_destroy(sc->dtag, npe->map); + } + bus_dmamap_unload(sc->buf_tag, sc->buf_map); + bus_dmamem_free(sc->buf_tag, sc->hwbuf, sc->buf_map); + } + if (sc->buf != NULL) + free(sc->buf, M_IXPCRYPTO); + if (sc->buf_tag) + bus_dma_tag_destroy(sc->buf_tag); + if (sc->dtag) + bus_dma_tag_destroy(sc->dtag); + mtx_destroy(&sc->mtx); +} + +static int +ixpcrypto_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) +{ + struct ixpcrypto_softc *sc = device_get_softc(dev); + struct ixpcrypto_session *ses = NULL; + struct cryptoini *encini, *macini; + + if (sidp == NULL || cri == NULL) + return EINVAL; + + encini = macini = NULL; + for (; cri != NULL; cri = cri->cri_next) { + switch (cri->cri_alg) { + case CRYPTO_NULL_HMAC: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + if (macini != NULL) + return EINVAL; + macini = cri; + break; + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + case CRYPTO_AES_CBC: +#ifdef CRYPTO_AES_CTR_CBC + case CRYPTO_AES_CTR_CBC: +#endif + if (encini != NULL) + return EINVAL; + encini = cri; + break; + default: + return EINVAL; + } + } + + /* + * Let's look for a free session structure. + */ + rw_wlock(&sc->sessions_lock); + /* + * Free sessions goes first, so if first session is used, we need to + * allocate one. + */ + ses = TAILQ_FIRST(&sc->sessions); + if (ses == NULL || ses->inuse) { + ses = malloc(sizeof(*ses), M_IXPCRYPTO, M_NOWAIT | M_ZERO); + if (ses == NULL) { + rw_wunlock(&sc->sessions_lock); + return ENOMEM; + } + ses->id = sc->sid++; + } else { + TAILQ_REMOVE(&sc->sessions, ses, next); + } + ses->inuse = 1; + TAILQ_INSERT_TAIL(&sc->sessions, ses, next); + rw_wunlock(&sc->sessions_lock); + + *sidp = ses->id; + return 0; +} + +static void +ixpcrypto_freesession_locked(struct ixpcrypto_softc *sc, + struct ixpcrypto_session *ses) +{ + uint32_t sid = ses->id; + + TAILQ_REMOVE(&sc->sessions, ses, next); + bzero(ses, sizeof(*ses)); + ses->inuse = 0; + ses->id = sid; + TAILQ_INSERT_HEAD(&sc->sessions, ses, next); +} + +static int +ixpcrypto_freesession(device_t dev, uint64_t tid) +{ + struct ixpcrypto_softc *sc = device_get_softc(dev); + struct ixpcrypto_session *ses; + uint32_t sid = ((uint32_t)tid) & 0xffffffff; + + rw_wlock(&sc->sessions_lock); + TAILQ_FOREACH_REVERSE(ses, &sc->sessions, ixpcrypto_sessions_head, next) { + if (ses->id == sid) { + ixpcrypto_freesession_locked(sc, ses); + rw_wunlock(&sc->sessions_lock); + return 0; + } + } + rw_wunlock(&sc->sessions_lock); + return EINVAL; +} + +static void +ixpcrypto_cb(void *arg, + bus_dma_segment_t *segs, int nsegs, bus_size_t len, int error) +{ + struct npebuf *npe = arg; + struct npehwbuf *hw; + uint32_t next; + int i; + + if (error != 0) + return; + hw = npe->hw; + next = npe->neaddr + sizeof(hw->ne[0]); + for (i = 0; i < nsegs; i++) { + hw->ne[i].data = htobe32(segs[i].ds_addr); + hw->ne[i].len = htobe32((segs[i].ds_len<<16) | len); + hw->ne[i].next = htobe32(next); + + len = 0; /* zero for segments > 1 */ + next += sizeof(hw->ne[0]); + } + hw->ne[i-1].next = 0; /* zero last in chain */ +} + +static int +ixpcrypto_process(device_t dev, struct cryptop *crp, int hint __unused) +{ + struct ixpcrypto_softc *sc = device_get_softc(dev); + struct ixpcrypto_session *ses = NULL; + struct cryptodesc *crd, *enccrd, *maccrd; + struct npebuf *npe; + int error = 0; + + enccrd = maccrd = NULL; + + /* Sanity check. */ + if (crp == NULL) + return EINVAL; + + if (crp->crp_callback == NULL || crp->crp_desc == NULL) { + error = EINVAL; + goto out; + } + + for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { + switch (crd->crd_alg) { + case CRYPTO_NULL_HMAC: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: +#if 0 + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: +#endif + if (maccrd != NULL) { + error = EINVAL; + goto out; + } + maccrd = crd; + break; + case CRYPTO_AES_CBC: + if (enccrd != NULL) { + error = EINVAL; + goto out; + } + enccrd = crd; + break; + default: + return EINVAL; + } + } + if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { + error = EINVAL; + goto out; + } + + rw_rlock(&sc->sessions_lock); + TAILQ_FOREACH_REVERSE(ses, &sc->sessions, ixpcrypto_sessions_head, next) { + if (ses->id == (crp->crp_sid & 0xffffffff)) + break; + } + rw_runlock(&sc->sessions_lock); + if (ses == NULL) { + error = EINVAL; + goto out; + } + + mtx_lock(&sc->mtx); + npe = sc->free; + if (npe != NULL) + sc->free = npe->next; + mtx_unlock(&sc->mtx); + if (npe == NULL) { + error = ENOBUFS; + goto out; + } + + npe->crp = crp; + if (crp->crp_flags & CRYPTO_F_IMBUF) { + error = bus_dmamap_load_mbuf(sc->dtag, npe->map, + (struct mbuf *) crp->crp_buf, + ixpcrypto_cb, npe, BUS_DMA_NOWAIT); + } else if (crp->crp_flags & CRYPTO_F_IOV) { + error = bus_dmamap_load_uio(sc->dtag, npe->map, + (struct uio *) crp->crp_buf, + ixpcrypto_cb, npe, BUS_DMA_NOWAIT); + } else + error = EINVAL; + if (error != 0) { + device_printf(sc->dev, "%s: error %u\n", __func__, error); + mtx_lock(&sc->mtx); + npe->next = sc->free; + sc->free = npe; + mtx_unlock(&sc->mtx); + goto out; + } + + bus_dmamap_sync(sc->dtag, npe->map, BUS_DMASYNC_PREWRITE); + /* XXX flush descriptor instead of using uncached memory */ + + DPRINTF(sc, "%s: qwrite(%u, 0x%x) data %x len 0x%x\n", + __func__, sc->cmd_qid, npe->neaddr, + npe->hw->ne[0].data, npe->hw->ne[0].len); + /* stick it on the cmd q */ + ixpqmgr_qwrite(sc->cmd_qid, npe->neaddr); + return 0; +out: + crp->crp_etype = error; + crypto_done(crp); + return error; +} + +static void +ixpcrypto_cmddone(int qid, void *arg) +{ +#define P2V(a, sc) \ + &(sc)->buf[((a) - (sc)->buf_phys) / sizeof(struct npehwbuf)] + struct ixpcrypto_softc *sc = arg; + uint32_t entry; + struct npebuf *head; + struct npebuf **tail; + struct npebuf *npe; + + head = NULL; + tail = &head; + while (ixpqmgr_qread(qid, &entry) == 0) { + npe = P2V(NPE_QM_Q_ADDR(entry), sc); + + /* XXX optimize based on request */ + bus_dmamap_sync(sc->dtag, npe->map, + BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); + + /* XXX copyback iv */ + crypto_done(npe->crp); + + *tail = npe; + tail = &npe->next; + } + mtx_lock(&sc->mtx); + *tail = sc->free; + sc->free = head; + mtx_unlock(&sc->mtx); +#undef P2V +} + +static device_method_t ixpcrypto_methods[] = { + DEVMETHOD(device_probe, ixpcrypto_probe), + DEVMETHOD(device_attach, ixpcrypto_attach), + DEVMETHOD(device_detach, ixpcrypto_detach), + + DEVMETHOD(cryptodev_newsession, ixpcrypto_newsession), + DEVMETHOD(cryptodev_freesession,ixpcrypto_freesession), + DEVMETHOD(cryptodev_process, ixpcrypto_process), + + {0, 0}, +}; + +static driver_t ixpcrypto_driver = { + "ixpcrypto", + ixpcrypto_methods, + sizeof(struct ixpcrypto_softc), +}; +static devclass_t ixpcrypto_devclass; + +DRIVER_MODULE(ixpcrypto, ixp, ixpcrypto_driver, ixpcrypto_devclass, 0, 0); +MODULE_VERSION(ixpcrypto, 1); +MODULE_DEPEND(ixpcrypto, ixpqmgr, 1, 1, 1); +MODULE_DEPEND(ixpcrypto, crypto, 1, 1, 1);