Date: Sat, 17 Dec 2011 15:18:58 +0100 From: Matthieu Kraus <matthieu.kraus@s2008.tu-chemnitz.de> To: freebsd-arm@freebsd.org Subject: cesa patch issues Message-ID: <20111217151858.182523ch5m4bclw2@mail.tu-chemnitz.de>
next in thread | raw e-mail | index | archive | help
This message is in MIME format.
--=_bx6zcmqegq
Content-Type: text/plain;
charset=UTF-8;
DelSp="Yes";
format="flowed"
Content-Disposition: inline
Content-Transfer-Encoding: 7bit
Greetings,
as I had issues getting CESA as it is working for bigger requests (as
those issued by geli) because it either fails to complete the
requeusts at all or (after increasing SA and TDMA pools quite a bit)
suffers from bad performance I tried to rework it to go with the flow
suggested in the functional spec, i.e. preparing the data in regular
buffers and upon enqueueing filling out the TDMA/SA descriptors. (see
attached patch)
However I can't seem to get my modifications to work as I always get:
vm_fault(0xc35789c8, 0, 2, 0) -> 1
Fatal kernel mode data abort: 'Translation Fault (P)'
trapframe: 0xd2069bb0
FSR=00000017, FAR=00000010, spsr=80000013
r0 =0000001f, r1 =00000000, r2 =00000000, r3 =00000004
r4 =00000000, r5 =00000000, r6 =00000000, r7 =c373d898
r8 =c3736000, r9 =00000000, r10=c3736000, r11=d2069c38
r12=00000000, ssp=d2069bfc, slr=c0c02f40, pc =c0c22d58
[ thread pid 1439 tid 100057 ]
Stopped at cpu_initclocks+0x3340: str r4, [r3, #0x00c]
I'm completely new to kernel development, so any hints on how to
further debug the issue (traces in ddb didn't seem very informational
to me) would be highly appreciated.
kind regards,
Matthieu Kraus
--=_bx6zcmqegq
Content-Type: text/plain;
charset=ISO-8859-1;
name="cesa-patch.txt"
Content-Disposition: attachment;
filename="cesa-patch.txt"
Content-Transfer-Encoding: 7bit
Index: sys/dev/cesa/cesa.h
===================================================================
--- sys/dev/cesa/cesa.h (revision 228127)
+++ sys/dev/cesa/cesa.h (working copy)
@@ -61,11 +61,11 @@
*/
/* Values below are optimized for requests containing about 1.5 kB of data */
-#define CESA_SA_DESC_PER_REQ 2
-#define CESA_TDMA_DESC_PER_REQ 8
+#define CESA_PACKETS_PER_REQ 2
+#define CESA_TDMAS_PER_PACKET 4
-#define CESA_SA_DESCRIPTORS (CESA_SA_DESC_PER_REQ * CESA_REQUESTS)
-#define CESA_TDMA_DESCRIPTORS (CESA_TDMA_DESC_PER_REQ * CESA_REQUESTS)
+#define CESA_SA_DESCRIPTORS (CESA_PACKETS_PER_REQ * CESA_REQUESTS)
+#define CESA_TDMA_DESCRIPTORS ((CESA_PACKETS_PER_REQ*CESA_TDMAS_PER_PACKET+2) * CESA_REQUESTS)
/* Useful constants */
#define CESA_HMAC_HASH_LENGTH 12
@@ -99,9 +99,7 @@
bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
/* Generic allocator for objects */
-#define CESA_GENERIC_ALLOC_LOCKED(sc, obj, pool) do { \
- CESA_LOCK(sc, pool); \
- \
+#define CESA_GENERIC_ALLOC(sc, obj, pool) do { \
if (STAILQ_EMPTY(&(sc)->sc_free_ ## pool)) \
obj = NULL; \
else { \
@@ -109,14 +107,23 @@
STAILQ_REMOVE_HEAD(&(sc)->sc_free_ ## pool, \
obj ## _stq); \
} \
- \
+} while(0)
+
+#define CESA_GENERIC_FREE(sc, obj, pool) do { \
+ STAILQ_INSERT_TAIL(&(sc)->sc_free_ ## pool, obj, \
+ obj ## _stq); \
+} while (0)
+
+/* Generic locked allocator for objects */
+#define CESA_GENERIC_ALLOC_LOCKED(sc, obj, pool) do { \
+ CESA_LOCK(sc, pool); \
+ CESA_GENERIC_ALLOC(sc, obj, pool); \
CESA_UNLOCK(sc, pool); \
} while (0)
#define CESA_GENERIC_FREE_LOCKED(sc, obj, pool) do { \
CESA_LOCK(sc, pool); \
- STAILQ_INSERT_TAIL(&(sc)->sc_free_ ## pool, obj, \
- obj ## _stq); \
+ CESA_GENERIC_FREE(sc, obj, pool); \
CESA_UNLOCK(sc, pool); \
} while (0)
@@ -194,29 +201,40 @@
STAILQ_ENTRY(cesa_session) cs_stq;
};
+struct cesa_packet {
+ uint32_t cp_data;
+ uint16_t cp_size;
+
+ uint32_t cp_config;
+ uint16_t cp_enc_src;
+ uint16_t cp_enc_dst;
+ uint32_t cp_enc_dlen;
+ uint16_t cp_mac_src;
+ uint16_t cp_mac_total_dlen;
+ uint16_t cp_mac_dlen;
+
+ STAILQ_ENTRY(cesa_packet) cp_stq;
+};
+
struct cesa_request {
- struct cesa_sa_data *cr_csd;
- bus_addr_t cr_csd_paddr;
struct cryptop *cr_crp;
struct cryptodesc *cr_enc;
struct cryptodesc *cr_mac;
struct cesa_session *cr_cs;
+
bus_dmamap_t cr_dmap;
int cr_dmap_loaded;
- STAILQ_HEAD(, cesa_tdma_desc) cr_tdesc;
+ struct cesa_sa_data *cr_csd;
+ bus_addr_t cr_csd_paddr;
+
+ unsigned int cr_processing;
+ STAILQ_HEAD(, cesa_packet) cr_packets;
STAILQ_HEAD(, cesa_sa_desc) cr_sdesc;
STAILQ_ENTRY(cesa_request) cr_stq;
};
-struct cesa_packet {
- STAILQ_HEAD(, cesa_tdma_desc) cp_copyin;
- STAILQ_HEAD(, cesa_tdma_desc) cp_copyout;
- unsigned int cp_size;
- unsigned int cp_offset;
-};
-
struct cesa_softc {
device_t sc_dev;
int32_t sc_cid;
@@ -230,12 +248,12 @@
struct mtx sc_sc_lock;
int sc_blocked;
+ int sc_busy;
/* TDMA descriptors pool */
- struct mtx sc_tdesc_lock;
struct cesa_tdma_desc sc_tdesc[CESA_TDMA_DESCRIPTORS];
struct cesa_dma_mem sc_tdesc_cdm;
- STAILQ_HEAD(, cesa_tdma_desc) sc_free_tdesc;
+ unsigned int sc_tdesc_curr;
/* SA descriptors pool */
struct mtx sc_sdesc_lock;
@@ -263,9 +281,16 @@
struct cesa_chain_info {
struct cesa_softc *cci_sc;
struct cesa_request *cci_cr;
+
struct cryptodesc *cci_enc;
struct cryptodesc *cci_mac;
+
uint32_t cci_config;
+
+ unsigned int cci_mmask;
+ unsigned int cci_emask;
+ unsigned int cci_mpsize;
+
int cci_error;
};
Index: sys/dev/cesa/cesa.c
===================================================================
--- sys/dev/cesa/cesa.c (revision 228127)
+++ sys/dev/cesa/cesa.c (working copy)
@@ -71,7 +71,7 @@
#include <arm/mv/mvvar.h>
#include "cesa.h"
-#undef DEBUG
+static MALLOC_DEFINE(M_CESA, "cesa packet data", "CESA packet data");
static int cesa_probe(device_t);
static int cesa_attach(device_t);
@@ -112,30 +112,6 @@
MODULE_DEPEND(cesa, crypto, 1, 1, 1);
static void
-cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
-{
-#ifdef DEBUG
- device_t dev;
-
- dev = sc->sc_dev;
- device_printf(dev, "CESA SA Hardware Descriptor:\n");
- device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
- device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src);
- device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst);
- device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
- device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key);
- device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
- device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
- device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src);
- device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst);
- device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
- device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
- device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
- device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
-#endif
-}
-
-static void
cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
struct cesa_dma_mem *cdm;
@@ -220,15 +196,6 @@
bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
}
-static void
-cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
-{
-
- cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
- cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
- cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
-}
-
static struct cesa_session *
cesa_alloc_session(struct cesa_softc *sc)
{
@@ -252,7 +219,6 @@
static void
cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs)
{
-
CESA_GENERIC_FREE_LOCKED(sc, cs, sessions);
}
@@ -265,8 +231,9 @@
if (!cr)
return (NULL);
- STAILQ_INIT(&cr->cr_tdesc);
STAILQ_INIT(&cr->cr_sdesc);
+ STAILQ_INIT(&cr->cr_packets);
+ cr->cr_processing = 0;
return (cr);
}
@@ -274,16 +241,14 @@
static void
cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
{
+ struct cesa_packet *cp, *tmp;
- /* Free TDMA descriptors assigned to this request */
- CESA_LOCK(sc, tdesc);
- STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
- CESA_UNLOCK(sc, tdesc);
+ CESA_LOCK(sc, sdesc);
+ STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
+ CESA_UNLOCK(sc, sdesc);
- /* Free SA descriptors assigned to this request */
- CESA_LOCK(sc, sdesc);
- STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
- CESA_UNLOCK(sc, sdesc);
+ STAILQ_FOREACH_SAFE(cp, &cr->cr_packets, cp_stq, tmp)
+ free(cp, M_CESA);
/* Unload DMA memory asociated with request */
if (cr->cr_dmap_loaded) {
@@ -303,60 +268,10 @@
CESA_UNLOCK(sc, requests);
}
-static struct cesa_tdma_desc *
-cesa_alloc_tdesc(struct cesa_softc *sc)
-{
- struct cesa_tdma_desc *ctd;
-
- CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
-
- if (!ctd)
- device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
- "Consider increasing CESA_TDMA_DESCRIPTORS.\n");
-
- return (ctd);
-}
-
-static struct cesa_sa_desc *
-cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
-{
- struct cesa_sa_desc *csd;
-
- CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
- if (!csd) {
- device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
- "Consider increasing CESA_SA_DESCRIPTORS.\n");
- return (NULL);
- }
-
- STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
-
- /* Fill-in SA descriptor with default values */
- csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
- csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
- csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
- csd->csd_cshd->cshd_enc_src = 0;
- csd->csd_cshd->cshd_enc_dst = 0;
- csd->csd_cshd->cshd_enc_dlen = 0;
- csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
- csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
- csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
- csd->csd_cshd->cshd_mac_src = 0;
- csd->csd_cshd->cshd_mac_dlen = 0;
-
- return (csd);
-}
-
-static struct cesa_tdma_desc *
-cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
+static void
+cesa_tdma_copy(struct cesa_tdma_desc *ctd, bus_addr_t dst, bus_addr_t src,
bus_size_t size)
{
- struct cesa_tdma_desc *ctd;
-
- ctd = cesa_alloc_tdesc(sc);
- if (!ctd)
- return (NULL);
-
ctd->ctd_cthd->cthd_dst = dst;
ctd->ctd_cthd->cthd_src = src;
ctd->ctd_cthd->cthd_byte_count = size;
@@ -366,83 +281,9 @@
ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
else
ctd->ctd_cthd->cthd_flags = 0;
-
- return (ctd);
}
-static struct cesa_tdma_desc *
-cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
-{
-
- return (cesa_tdma_copy(sc, sc->sc_sram_base +
- sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
- sizeof(struct cesa_sa_data)));
-}
-
-static struct cesa_tdma_desc *
-cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
-{
-
- return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base +
- sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
-}
-
-static struct cesa_tdma_desc *
-cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
-{
-
- return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr,
- sizeof(struct cesa_sa_hdesc)));
-}
-
-static void
-cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
-{
- struct cesa_tdma_desc *ctd_prev;
-
- if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
- ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
- ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
- }
-
- ctd->ctd_cthd->cthd_next = 0;
- STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
-}
-
static int
-cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
- struct cesa_packet *cp, struct cesa_sa_desc *csd)
-{
- struct cesa_tdma_desc *ctd, *tmp;
-
- /* Copy SA descriptor for this packet */
- ctd = cesa_tdma_copy_sdesc(sc, csd);
- if (!ctd)
- return (ENOMEM);
-
- cesa_append_tdesc(cr, ctd);
-
- /* Copy data to be processed */
- STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
- cesa_append_tdesc(cr, ctd);
- STAILQ_INIT(&cp->cp_copyin);
-
- /* Insert control descriptor */
- ctd = cesa_tdma_copy(sc, 0, 0, 0);
- if (!ctd)
- return (ENOMEM);
-
- cesa_append_tdesc(cr, ctd);
-
- /* Copy back results */
- STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
- cesa_append_tdesc(cr, ctd);
- STAILQ_INIT(&cp->cp_copyout);
-
- return (0);
-}
-
-static int
cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
{
uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
@@ -547,63 +388,19 @@
}
static void
-cesa_start_packet(struct cesa_packet *cp, unsigned int size)
-{
-
- cp->cp_size = size;
- cp->cp_offset = 0;
- STAILQ_INIT(&cp->cp_copyin);
- STAILQ_INIT(&cp->cp_copyout);
-}
-
-static int
-cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
- bus_dma_segment_t *seg)
-{
- struct cesa_tdma_desc *ctd;
- unsigned int bsize;
-
- /* Calculate size of block copy */
- bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
-
- if (bsize > 0) {
- ctd = cesa_tdma_copy(sc, sc->sc_sram_base +
- CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
- if (!ctd)
- return (-ENOMEM);
-
- STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
-
- ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base +
- CESA_DATA(cp->cp_offset), bsize);
- if (!ctd)
- return (-ENOMEM);
-
- STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
-
- seg->ds_len -= bsize;
- seg->ds_addr += bsize;
- cp->cp_offset += bsize;
- }
-
- return (bsize);
-}
-
-static void
cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
- unsigned int mpsize, fragmented;
- unsigned int mlen, mskip, tmlen;
struct cesa_chain_info *cci;
unsigned int elen, eskip;
- unsigned int skip, len;
- struct cesa_sa_desc *csd;
+ unsigned int mlen, mskip, tmlen;
+ unsigned int skip, len, fragmented;
+ unsigned int size;
+ bus_dma_segment_t seg;
+
struct cesa_request *cr;
struct cesa_softc *sc;
- struct cesa_packet cp;
- bus_dma_segment_t seg;
+ struct cesa_packet* cp;
uint32_t config;
- int size;
cci = arg;
sc = cci->cci_sc;
@@ -620,8 +417,8 @@
mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
if (elen && mlen &&
- ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
- (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
+ ((eskip > mskip && ((eskip - mskip) & cci->cci_emask)) ||
+ (mskip > eskip && ((mskip - eskip) & cci->cci_mmask)) ||
(eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
/*
* Data alignment in the request does not meet CESA requiremnts
@@ -661,23 +458,23 @@
tmlen = mlen;
fragmented = 0;
- mpsize = CESA_MAX_PACKET_SIZE;
- mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
if (elen && mlen) {
skip = MIN(eskip, mskip);
len = MAX(elen + eskip, mlen + mskip) - skip;
+
+ eskip -= skip;
+ mskip -= skip;
} else if (elen) {
skip = eskip;
len = elen;
+ eskip = 0;
} else {
skip = mskip;
len = mlen;
+ mskip = 0;
}
- /* Start first packet in chain */
- cesa_start_packet(&cp, MIN(mpsize, len));
-
while (nseg-- && len > 0) {
seg = *(segs++);
@@ -691,105 +488,71 @@
seg.ds_addr += size;
seg.ds_len -= size;
+ }
- if (eskip > 0)
- eskip -= size;
+ /*
+ * Create packet and fill with data from DMA segment
+ * until there is no more data in current DMA segment
+ * or an error occured.
+ */
+ while (seg.ds_len > 0) {
+ cp = malloc(sizeof(struct cesa_packet), M_CESA, M_WAITOK);
- if (mskip > 0)
- mskip -= size;
+ cp->cp_size = MIN(seg.ds_len, MIN(cci->cci_mpsize, len));
+ cp->cp_data = seg.ds_addr;
- if (seg.ds_len == 0)
- continue;
- }
+ seg.ds_len -= cp->cp_size;
+ seg.ds_addr += cp->cp_size;
+ len -= cp->cp_size;
- while (1) {
+ /* Create SA descriptor for this packet */
+ cp->cp_config = cci->cci_config;
+ cp->cp_mac_total_dlen = tmlen;
+
/*
- * Fill in current packet with data. Break if there is
- * no more data in current DMA segment or an error
- * occured.
+ * Enable fragmentation if request will not fit
+ * into one packet.
*/
- size = cesa_fill_packet(sc, &cp, &seg);
- if (size <= 0) {
- error = -size;
- break;
+ if (!fragmented && len > 0) {
+ fragmented = 1;
+ cp->cp_config |= CESA_CSHD_FRAG_FIRST;
+ } else if (len > 0) {
+ cp->cp_config |= CESA_CSHD_FRAG_MIDDLE;
+ } else {
+ cp->cp_config |= CESA_CSHD_FRAG_LAST;
}
- len -= size;
+ if (eskip < cp->cp_size && elen > 0) {
+ cp->cp_enc_src = CESA_DATA(eskip);
+ cp->cp_enc_dst = CESA_DATA(eskip);
+ cp->cp_enc_dlen = MIN(elen, cp->cp_size - eskip);
+ }
+ else
+ {
+ cp->cp_enc_src = 0;
+ cp->cp_enc_dst = 0;
+ cp->cp_enc_dlen = 0;
+ }
- /* If packet is full, append it to the chain */
- if (cp.cp_size == cp.cp_offset) {
- csd = cesa_alloc_sdesc(sc, cr);
- if (!csd) {
- error = ENOMEM;
- break;
- }
+ if (mskip < cp->cp_size && mlen > 0) {
+ cp->cp_mac_src = CESA_DATA(mskip);
+ cp->cp_mac_dlen = MIN(mlen, cp->cp_size - mskip);
+ }
+ else
+ {
+ cp->cp_mac_src = 0;
+ cp->cp_mac_dlen = 0;
+ }
- /* Create SA descriptor for this packet */
- csd->csd_cshd->cshd_config = cci->cci_config;
- csd->csd_cshd->cshd_mac_total_dlen = tmlen;
+ elen -= cp->cp_enc_dlen;
+ eskip -= MIN(eskip, cp->cp_size);
+ mlen -= cp->cp_mac_dlen;
+ mskip -= MIN(mskip, cp->cp_size);
- /*
- * Enable fragmentation if request will not fit
- * into one packet.
- */
- if (len > 0) {
- if (!fragmented) {
- fragmented = 1;
- csd->csd_cshd->cshd_config |=
- CESA_CSHD_FRAG_FIRST;
- } else
- csd->csd_cshd->cshd_config |=
- CESA_CSHD_FRAG_MIDDLE;
- } else if (fragmented)
- csd->csd_cshd->cshd_config |=
- CESA_CSHD_FRAG_LAST;
-
- if (eskip < cp.cp_size && elen > 0) {
- csd->csd_cshd->cshd_enc_src =
- CESA_DATA(eskip);
- csd->csd_cshd->cshd_enc_dst =
- CESA_DATA(eskip);
- csd->csd_cshd->cshd_enc_dlen =
- MIN(elen, cp.cp_size - eskip);
- }
-
- if (mskip < cp.cp_size && mlen > 0) {
- csd->csd_cshd->cshd_mac_src =
- CESA_DATA(mskip);
- csd->csd_cshd->cshd_mac_dlen =
- MIN(mlen, cp.cp_size - mskip);
- }
-
- elen -= csd->csd_cshd->cshd_enc_dlen;
- eskip -= MIN(eskip, cp.cp_size);
- mlen -= csd->csd_cshd->cshd_mac_dlen;
- mskip -= MIN(mskip, cp.cp_size);
-
- cesa_dump_cshd(sc, csd->csd_cshd);
-
- /* Append packet to the request */
- error = cesa_append_packet(sc, cr, &cp, csd);
- if (error)
- break;
-
- /* Start a new packet, as current is full */
- cesa_start_packet(&cp, MIN(mpsize, len));
- }
+ /* Append packet to the request */
+ STAILQ_INSERT_TAIL(&cr->cr_packets, cp, cp_stq);
}
-
- if (error)
- break;
}
-
- if (error) {
- /*
- * Move all allocated resources to the request. They will be
- * freed later.
- */
- STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
- STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
- cci->cci_error = error;
- }
}
static void
@@ -804,8 +567,6 @@
cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
{
struct cesa_chain_info cci;
- struct cesa_tdma_desc *ctd;
- uint32_t config;
int error;
error = 0;
@@ -829,31 +590,30 @@
CESA_MAX_HASH_LEN);
}
- ctd = cesa_tdma_copyin_sa_data(sc, cr);
- if (!ctd)
- return (ENOMEM);
-
- cesa_append_tdesc(cr, ctd);
-
/* Prepare SA configuration */
- config = cr->cr_cs->cs_config;
+ cci.cci_config = cr->cr_cs->cs_config;
if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
- config |= CESA_CSHD_DECRYPT;
+ cci.cci_config |= CESA_CSHD_DECRYPT;
if (cr->cr_enc && !cr->cr_mac)
- config |= CESA_CSHD_ENC;
+ cci.cci_config |= CESA_CSHD_ENC;
if (!cr->cr_enc && cr->cr_mac)
- config |= CESA_CSHD_MAC;
+ cci.cci_config |= CESA_CSHD_MAC;
if (cr->cr_enc && cr->cr_mac)
- config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
- CESA_CSHD_ENC_AND_MAC;
+ cci.cci_config |= (cci.cci_config & CESA_CSHD_DECRYPT)
+ ? CESA_CSHD_MAC_AND_ENC
+ : CESA_CSHD_ENC_AND_MAC;
- /* Create data packets */
+ /* Prepare chain configuration */
+ cci.cci_emask = cr->cr_cs->cs_ivlen - 1;
+ cci.cci_mmask = cr->cr_cs->cs_mblen - 1;
+ cci.cci_mpsize = CESA_MAX_PACKET_SIZE;
+ cci.cci_mpsize &= ~(cci.cci_emask | cci.cci_mmask);
+
cci.cci_sc = sc;
cci.cci_cr = cr;
cci.cci_enc = cr->cr_enc;
cci.cci_mac = cr->cr_mac;
- cci.cci_config = config;
cci.cci_error = 0;
if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
@@ -879,21 +639,20 @@
if (error)
return (error);
- /* Read back request metadata */
- ctd = cesa_tdma_copyout_sa_data(sc, cr);
- if (!ctd)
- return (ENOMEM);
-
- cesa_append_tdesc(cr, ctd);
-
return (0);
}
static void
cesa_execute(struct cesa_softc *sc)
{
- struct cesa_tdma_desc *prev_ctd, *ctd;
- struct cesa_request *prev_cr, *cr;
+ struct cesa_tdma_desc *ctd;
+ struct cesa_sa_desc *csd;
+ struct cesa_request *cr;
+ struct cesa_packet *cp;
+ unsigned int tdesc_curr;
+ bus_addr_t base_data;
+ bus_addr_t base_sa;
+ bus_addr_t base_sa_data;
CESA_LOCK(sc, requests);
@@ -902,8 +661,8 @@
* is not empty, the hardware is busy and we cannot start another
* execution.
*/
- if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
- !STAILQ_EMPTY(&sc->sc_queued_requests)) {
+ if ((STAILQ_EMPTY(&sc->sc_ready_requests)
+ && STAILQ_EMPTY(&sc->sc_queued_requests)) || sc->sc_busy) {
CESA_UNLOCK(sc, requests);
return;
}
@@ -912,37 +671,109 @@
STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
STAILQ_INIT(&sc->sc_ready_requests);
- /* Create one execution chain from all requests on the list */
- if (STAILQ_FIRST(&sc->sc_queued_requests) !=
- STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
- prev_cr = NULL;
- cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
- BUS_DMASYNC_POSTWRITE);
+ /* Create execution chain from all requests on the list */
+ tdesc_curr = 0;
- STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
- if (prev_cr) {
- ctd = STAILQ_FIRST(&cr->cr_tdesc);
- prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
- cesa_tdma_desc, ctd_stq);
+ base_sa = sc->sc_sram_base;
+ base_sa_data = base_sa + sizeof(struct cesa_sa_hdesc);
+ base_data = base_sa_data + sizeof(struct cesa_sa_data);
- prev_ctd->ctd_cthd->cthd_next =
- ctd->ctd_cthd_paddr;
+ cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ cesa_sync_dma_mem(&sc->sc_sdesc_cdm, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+ /* re-set next handle of tail of previous chain */
+ tdesc_curr = sc->sc_tdesc_curr;
+ if(tdesc_curr != CESA_TDMA_DESCRIPTORS - 1)
+ {
+ sc->sc_tdesc[tdesc_curr].ctd_cthd->cthd_next =
+ sc->sc_tdesc[tdesc_curr + 1].ctd_cthd_paddr;
+ }
+
+ #define CESA_NEXT_TDESC(obj) do { \
+ obj = &sc->sc_tdesc[tdesc_curr++]; \
+ } while(0)
+
+ STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
+ if(!cr->cr_processing)
+ {
+ /* check we can queue a write */
+ if(tdesc_curr < CESA_TDMA_DESCRIPTORS)
+ {
+ break;
}
- prev_cr = cr;
+ cr->cr_processing = 1;
+
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, base_sa_data, cr->cr_csd_paddr, sizeof(struct cesa_sa_data));
}
- cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
- BUS_DMASYNC_PREWRITE);
+ while(!STAILQ_EMPTY(&cr->cr_packets) && tdesc_curr < CESA_TDMA_DESCRIPTORS - 5) {
+ CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
+ if(!csd)
+ {
+ break;
+ }
+ STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
+
+ cp = STAILQ_FIRST(&cr->cr_packets);
+
+ csd->csd_cshd->cshd_config = cp->cp_config;
+ csd->csd_cshd->cshd_enc_src = cp->cp_enc_src;
+ csd->csd_cshd->cshd_enc_dst = cp->cp_enc_dst;
+ csd->csd_cshd->cshd_enc_dlen = cp->cp_enc_dlen;
+ csd->csd_cshd->cshd_mac_src = cp->cp_mac_src;
+ csd->csd_cshd->cshd_mac_total_dlen = cp->cp_mac_total_dlen;
+ csd->csd_cshd->cshd_mac_dlen = cp->cp_mac_dlen;
+
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, base_sa, csd->csd_cshd_paddr, sizeof(struct cesa_sa_hdesc));
+
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, base_data, cp->cp_data, cp->cp_size);
+
+ /* may be obsolete */
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, 0, 0, 0);
+
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, cp->cp_data, base_data, cp->cp_size);
+
+ STAILQ_REMOVE_HEAD(&cr->cr_packets, cp_stq);
+ free(cp, M_CESA);
+ }
+
+ /* check whether the complete request could be queued */
+ if(!STAILQ_EMPTY(&cr->cr_packets))
+ {
+ break;
+ }
+
+ CESA_NEXT_TDESC(ctd);
+ cesa_tdma_copy(ctd, cr->cr_csd_paddr, base_sa_data, sizeof(struct cesa_sa_data));
}
+ #undef CESA_NEXT_TDESC
+
+ /* make sure TDMA gets back to use once it processed all data */
+ --tdesc_curr;
+ if(tdesc_curr != CESA_TDMA_DESCRIPTORS - 1)
+ {
+ sc->sc_tdesc[tdesc_curr].ctd_cthd->cthd_next = 0;
+ }
+
+ cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+ cesa_sync_dma_mem(&sc->sc_sdesc_cdm, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+ sc->sc_tdesc_curr = tdesc_curr;
+
/* Start chain execution in hardware */
- cr = STAILQ_FIRST(&sc->sc_queued_requests);
- ctd = STAILQ_FIRST(&cr->cr_tdesc);
+ ctd = &sc->sc_tdesc[0];
CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
+ sc->sc_busy = 1;
CESA_UNLOCK(sc, requests);
}
@@ -992,6 +823,7 @@
sc = device_get_softc(dev);
sc->sc_blocked = 0;
+ sc->sc_busy = 0;
sc->sc_error = 0;
sc->sc_dev = dev;
@@ -1018,8 +850,6 @@
/* Initialize mutexes */
mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
"CESA Shared Data", MTX_DEF);
- mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
- "CESA TDMA Descriptors Pool", MTX_DEF);
mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
"CESA SA Descriptors Pool", MTX_DEF);
mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
@@ -1065,14 +895,16 @@
if (error)
goto err3;
- STAILQ_INIT(&sc->sc_free_tdesc);
+ sc->sc_tdesc_curr = 0;
for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
sc->sc_tdesc[i].ctd_cthd =
(struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
- (i * sizeof(struct cesa_tdma_hdesc));
- STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
- ctd_stq);
+ i * sizeof(struct cesa_tdma_hdesc);
+
+ if (i != CESA_TDMA_DESCRIPTORS - 1)
+ sc->sc_tdesc[i].ctd_cthd->cthd_next = sc->sc_tdesc_cdm.cdm_paddr +
+ (i + 1) * sizeof(struct cesa_tdma_hdesc);
}
/* Initialize data structures: SA Descriptors Pool */
@@ -1087,6 +919,15 @@
(struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
(i * sizeof(struct cesa_sa_hdesc));
+
+ /* fill constant descriptor values */
+ sc->sc_sdesc[i].csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
+ sc->sc_sdesc[i].csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
+ sc->sc_sdesc[i].csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
+ sc->sc_sdesc[i].csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
+ sc->sc_sdesc[i].csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
+ sc->sc_sdesc[i].csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
+
STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
csd_stq);
}
@@ -1195,7 +1036,6 @@
mtx_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_requests_lock);
mtx_destroy(&sc->sc_sdesc_lock);
- mtx_destroy(&sc->sc_tdesc_lock);
mtx_destroy(&sc->sc_sc_lock);
return (ENXIO);
}
@@ -1240,7 +1080,6 @@
mtx_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_requests_lock);
mtx_destroy(&sc->sc_sdesc_lock);
- mtx_destroy(&sc->sc_tdesc_lock);
mtx_destroy(&sc->sc_sc_lock);
return (0);
@@ -1295,10 +1134,30 @@
return;
/* Get all finished requests */
+ STAILQ_INIT(&requests);
CESA_LOCK(sc, requests);
- STAILQ_INIT(&requests);
- STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
- STAILQ_INIT(&sc->sc_queued_requests);
+ CESA_LOCK(sc, sdesc);
+ while(!STAILQ_EMPTY(&sc->sc_queued_requests))
+ {
+ cr = STAILQ_FIRST(&sc->sc_queued_requests);
+
+ /* free SA descriptors */
+ STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
+ STAILQ_INIT(&cr->cr_sdesc);
+
+ /* check for partial completion */
+ if(STAILQ_EMPTY(&cr->cr_packets))
+ {
+ STAILQ_REMOVE_HEAD(&sc->sc_queued_requests, cr_stq);
+ STAILQ_INSERT_TAIL(&requests, cr, cr_stq);
+ }
+ else
+ {
+ break;
+ }
+ }
+ sc->sc_busy = 0;
+ CESA_UNLOCK(sc, sdesc);
CESA_UNLOCK(sc, requests);
/* Execute all ready requests */
@@ -1545,8 +1404,9 @@
cr->cr_cs = cs;
CESA_LOCK(sc, sessions);
- cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
if (enc->crd_flags & CRD_F_IV_EXPLICIT)
memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
@@ -1586,7 +1446,8 @@
if (!error)
error = cesa_create_chain(sc, cr);
- cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+ cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
CESA_UNLOCK(sc, sessions);
if (error) {
--=_bx6zcmqegq--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20111217151858.182523ch5m4bclw2>