Date: Thu, 20 Nov 2008 10:48:01 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r185127 - in user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb: . sys ulp/iw_cxgb Message-ID: <200811201048.mAKAm1ra049339@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Thu Nov 20 10:48:01 2008 New Revision: 185127 URL: http://svn.freebsd.org/changeset/base/185127 Log: Fix cxgb to compile against new lock-less ring buffer Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_adapter.h user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_multiq.c user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_osdep.h user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_sge.c user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/sys/cxgb_support.c user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_adapter.h ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_adapter.h Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_adapter.h Thu Nov 20 10:48:01 2008 (r185127) @@ -41,6 +41,7 @@ $FreeBSD$ #include <sys/socket.h> #include <sys/sockio.h> #include <sys/condvar.h> +#include <sys/buf_ring.h> #include <net/ethernet.h> #include <net/if.h> @@ -258,7 +259,7 @@ struct sge_txq { * mbuf touches */ struct mbuf_head cleanq; - struct buf_ring txq_mr; + struct buf_ring *txq_mr; struct mbuf *immpkt; uint32_t txq_drops; uint32_t txq_skipped; Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_multiq.c ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_multiq.c Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_multiq.c Thu Nov 20 10:48:01 2008 (r185127) @@ -122,7 +122,7 @@ cxgb_pcpu_enqueue_packet_(struct sge_qse return (ENXIO); } txq = &qs->txq[TXQ_ETH]; - err = buf_ring_enqueue(&txq->txq_mr, m); + err = buf_ring_enqueue(txq->txq_mr, m); if (err) { txq->txq_drops++; m_freem(m); @@ -187,7 +187,7 @@ cxgb_dequeue_packet(struct sge_txq *txq, } sc = qs->port->adapter; - m = buf_ring_dequeue(&txq->txq_mr); + m = buf_ring_dequeue(txq->txq_mr); if (m == NULL) return (0); @@ -202,14 +202,14 @@ cxgb_dequeue_packet(struct sge_txq *txq, } size = m->m_pkthdr.len; - for (m = buf_ring_peek(&txq->txq_mr); m != NULL; - m = buf_ring_peek(&txq->txq_mr)) { + for (m = buf_ring_peek(txq->txq_mr); m != NULL; + m = buf_ring_peek(txq->txq_mr)) { if (m->m_pkthdr.tso_segsz > 0 || size + m->m_pkthdr.len > TX_WR_SIZE_MAX || m->m_next != NULL) break; - buf_ring_dequeue(&txq->txq_mr); + buf_ring_dequeue(txq->txq_mr); size += m->m_pkthdr.len; m_vec[count++] = m; @@ -232,7 +232,7 @@ cxgb_pcpu_free(struct sge_qset *qs) mtx_lock(&txq->lock); while ((m = mbufq_dequeue(&txq->sendq)) != NULL) m_freem(m); - while ((m = buf_ring_dequeue(&txq->txq_mr)) != NULL) + while ((m = buf_ring_dequeue(txq->txq_mr)) != NULL) m_freem(m); t3_free_tx_desc_all(txq); @@ -290,7 +290,7 @@ cxgb_pcpu_start_(struct sge_qset *qs, st initerr = ENXIO; else if (immpkt) { - if (!buf_ring_empty(&txq->txq_mr)) + if (!buf_ring_empty(txq->txq_mr)) initerr = cxgb_pcpu_enqueue_packet_(qs, immpkt); else txq->immpkt = immpkt; @@ -321,7 +321,7 @@ cxgb_pcpu_start_(struct sge_qset *qs, st } stopped = isset(&qs->txq_stopped, TXQ_ETH); - flush = (((!buf_ring_empty(&txq->txq_mr) || (!IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) && !stopped) || txq->immpkt); + flush = (((!buf_ring_empty(txq->txq_mr) || (!IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) && !stopped) || txq->immpkt); max_desc = tx_flush ? TX_ETH_Q_SIZE : TX_START_MAX_DESC; if (cxgb_debug) @@ -332,7 +332,7 @@ cxgb_pcpu_start_(struct sge_qset *qs, st if ((tx_flush && flush && err == 0) && - (!buf_ring_empty(&txq->txq_mr) || + (!buf_ring_empty(txq->txq_mr) || !IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) { struct thread *td = curthread; @@ -382,7 +382,7 @@ cxgb_pcpu_transmit(struct ifnet *ifp, st txq = &qs->txq[TXQ_ETH]; if (((sc->tunq_coalesce == 0) || - (buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) || + (buf_ring_count(txq->txq_mr) >= TX_WR_COUNT_MAX) || (coalesce_tx_enable == 0)) && mtx_trylock(&txq->lock)) { if (cxgb_debug) printf("doing immediate transmit\n"); @@ -390,12 +390,12 @@ cxgb_pcpu_transmit(struct ifnet *ifp, st txq->flags |= TXQ_TRANSMITTING; err = cxgb_pcpu_start_(qs, immpkt, FALSE); txq->flags &= ~TXQ_TRANSMITTING; - resid = (buf_ring_count(&txq->txq_mr) > 64) || (desc_reclaimable(txq) > 64); + resid = (buf_ring_count(txq->txq_mr) > 64) || (desc_reclaimable(txq) > 64); mtx_unlock(&txq->lock); } else if (immpkt) { if (cxgb_debug) printf("deferred coalesce=%jx ring_count=%d mtx_owned=%d\n", - sc->tunq_coalesce, buf_ring_count(&txq->txq_mr), mtx_owned(&txq->lock)); + sc->tunq_coalesce, buf_ring_count(txq->txq_mr), mtx_owned(&txq->lock)); err = cxgb_pcpu_enqueue_packet_(qs, immpkt); } @@ -444,7 +444,7 @@ cxgb_pcpu_start_proc(void *arg) if ((qs->port->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { idleticks = hz; - if (!buf_ring_empty(&txq->txq_mr) || + if (!buf_ring_empty(txq->txq_mr) || !mbufq_empty(&txq->sendq)) cxgb_pcpu_free(qs); goto done; @@ -469,11 +469,13 @@ cxgb_pcpu_start_proc(void *arg) mtx_unlock(&qs->rspq.lock); } #endif - if ((!buf_ring_empty(&txq->txq_mr)) && err == 0) { + if ((!buf_ring_empty(txq->txq_mr)) && err == 0) { +#if 0 if (cxgb_debug) printf("head=%p cons=%d prod=%d\n", txq->sendq.head, txq->txq_mr.br_cons, txq->txq_mr.br_prod); +#endif continue; } done: Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_osdep.h ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_osdep.h Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_osdep.h Thu Nov 20 10:48:01 2008 (r185127) @@ -156,9 +156,6 @@ struct t3_mbuf_hdr { #if defined(__i386__) || defined(__amd64__) -#define mb() __asm volatile("mfence":::"memory") -#define rmb() __asm volatile("lfence":::"memory") -#define wmb() __asm volatile("sfence" ::: "memory") #define smp_mb() mb() #define L1_CACHE_BYTES 128 @@ -187,164 +184,6 @@ extern void kdb_backtrace(void); #define L1_CACHE_BYTES 32 #endif -struct buf_ring { - caddr_t *br_ring; - volatile uint32_t br_cons; - volatile uint32_t br_prod; - int br_size; - struct mtx br_lock; -}; - -struct buf_ring *buf_ring_alloc(int count, int flags); -void buf_ring_free(struct buf_ring *); - -static __inline int -buf_ring_count(struct buf_ring *mr) -{ - int size = mr->br_size; - uint32_t mask = size - 1; - - return ((size + mr->br_prod - mr->br_cons) & mask); -} - -static __inline int -buf_ring_empty(struct buf_ring *mr) -{ - return (mr->br_cons == mr->br_prod); -} - -static __inline int -buf_ring_full(struct buf_ring *mr) -{ - uint32_t mask; - - mask = mr->br_size - 1; - return (mr->br_cons == ((mr->br_prod + 1) & mask)); -} - -/* - * The producer and consumer are independently locked - * this relies on the consumer providing his own serialization - * - */ -static __inline void * -__buf_ring_dequeue(struct buf_ring *mr) -{ - uint32_t prod, cons, mask; - caddr_t *ring, m; - - ring = (caddr_t *)mr->br_ring; - mask = mr->br_size - 1; - cons = mr->br_cons; - prod = mr->br_prod; - m = NULL; - if (cons != prod) { - m = ring[cons]; - ring[cons] = NULL; - mr->br_cons = (cons + 1) & mask; - } - return (m); -} - -static __inline void * -buf_ring_dequeue(struct buf_ring *mr) -{ - void *buf; - - mtx_lock(&mr->br_lock); - buf = __buf_ring_dequeue(mr); - mtx_unlock(&mr->br_lock); - return (buf); -} - -#ifdef INVARIANT_SUPPORT -static __inline void -__buf_ring_scan(struct buf_ring *mr, void *m, char *file, int line) -{ - int i; - - for (i = 0; i < mr->br_size; i++) - if (m == mr->br_ring[i]) - panic("%s:%d m=%p present prod=%d cons=%d idx=%d", file, - line, m, mr->br_prod, mr->br_cons, i); -} - -static __inline void -buf_ring_scan(struct buf_ring *mr, void *m, char *file, int line) -{ - mtx_lock(&mr->br_lock); - __buf_ring_scan(mr, m, file, line); - mtx_unlock(&mr->br_lock); -} - -#else -static __inline void -__buf_ring_scan(struct buf_ring *mr, void *m, char *file, int line) -{ -} - -static __inline void -buf_ring_scan(struct buf_ring *mr, void *m, char *file, int line) -{ -} -#endif - -static __inline int -__buf_ring_enqueue(struct buf_ring *mr, void *m, char *file, int line) -{ - - uint32_t prod, cons, mask; - int err; - - mask = mr->br_size - 1; - prod = mr->br_prod; - mb(); - cons = mr->br_cons; - __buf_ring_scan(mr, m, file, line); - if (((prod + 1) & mask) != cons) { - KASSERT(mr->br_ring[prod] == NULL, ("overwriting entry")); - mr->br_ring[prod] = m; - mb(); - mr->br_prod = (prod + 1) & mask; - err = 0; - } else - err = ENOBUFS; - - return (err); -} - -static __inline int -buf_ring_enqueue_(struct buf_ring *mr, void *m, char *file, int line) -{ - int err; - - mtx_lock(&mr->br_lock); - err = __buf_ring_enqueue(mr, m, file, line); - mtx_unlock(&mr->br_lock); - - return (err); -} - -#define buf_ring_enqueue(mr, m) buf_ring_enqueue_((mr), (m), __FILE__, __LINE__) - - -static __inline void * -buf_ring_peek(struct buf_ring *mr) -{ - int prod, cons, mask; - caddr_t *ring, m; - - ring = (caddr_t *)mr->br_ring; - mask = mr->br_size - 1; - cons = mr->br_cons; - prod = mr->br_prod; - m = NULL; - if (cons != prod) - m = ring[cons]; - - return (m); -} - #define DBG_RX (1 << 0) static const int debug_flags = DBG_RX; Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_sge.c ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_sge.c Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/cxgb_sge.c Thu Nov 20 10:48:01 2008 (r185127) @@ -1715,10 +1715,8 @@ t3_free_qset(adapter_t *sc, struct sge_q t3_free_tx_desc_all(&q->txq[TXQ_ETH]); for (i = 0; i < SGE_TXQ_PER_SET; i++) - if (q->txq[i].txq_mr.br_ring != NULL) { - free(q->txq[i].txq_mr.br_ring, M_DEVBUF); - mtx_destroy(&q->txq[i].txq_mr.br_lock); - } + if (q->txq[i].txq_mr != NULL) + buf_ring_free(q->txq[i].txq_mr, M_DEVBUF); for (i = 0; i < SGE_RXQ_PER_SET; ++i) { if (q->fl[i].desc) { mtx_lock_spin(&sc->sge.reg_lock); @@ -2277,14 +2275,12 @@ t3_sge_alloc_qset(adapter_t *sc, u_int i int i, header_size, ret = 0; for (i = 0; i < SGE_TXQ_PER_SET; i++) { - if ((q->txq[i].txq_mr.br_ring = malloc(cxgb_txq_buf_ring_size*sizeof(struct mbuf *), - M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { + + if ((q->txq[i].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, + M_DEVBUF, M_WAITOK)) == NULL) { device_printf(sc->dev, "failed to allocate mbuf ring\n"); goto err; } - q->txq[i].txq_mr.br_prod = q->txq[i].txq_mr.br_cons = 0; - q->txq[i].txq_mr.br_size = cxgb_txq_buf_ring_size; - mtx_init(&q->txq[i].txq_mr.br_lock, "txq mbuf ring", NULL, MTX_DEF); } init_qset_cntxt(q, id); @@ -3509,12 +3505,14 @@ t3_add_configured_sysctls(adapter_t *sc) SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "sendqlen", CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.qlen, 0, "#tunneled packets waiting to be sent"); +#if 0 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx", CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, 0, "#tunneled packets queue producer index"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx", CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, 0, "#tunneled packets queue consumer index"); +#endif SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "processed", CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, 0, "#tunneled packets processed by the card"); Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/sys/cxgb_support.c ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/sys/cxgb_support.c Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/sys/cxgb_support.c Thu Nov 20 10:48:01 2008 (r185127) @@ -303,33 +303,3 @@ free: uma_zfree(zone, vec[i]); } -struct buf_ring * -buf_ring_alloc(int count, int flags) -{ - struct buf_ring *br; - - KASSERT(powerof2(count), ("buf ring must be size power of 2")); - - br = malloc(sizeof(struct buf_ring), M_DEVBUF, flags|M_ZERO); - if (br == NULL) - return (NULL); - - br->br_ring = malloc(sizeof(caddr_t)*count, M_DEVBUF, flags|M_ZERO); - if (br->br_ring == NULL) { - free(br, M_DEVBUF); - return (NULL); - } - - mtx_init(&br->br_lock, "buf ring", NULL, MTX_DUPOK|MTX_DEF); - br->br_size = count; - br->br_prod = br->br_cons = 0; - - return (br); -} - -void -buf_ring_free(struct buf_ring *br) -{ - free(br->br_ring, M_DEVBUF); - free(br, M_DEVBUF); -} Modified: user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c Thu Nov 20 10:47:21 2008 (r185126) +++ user/kmacy/HEAD_fast_multi_xmit/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c Thu Nov 20 10:48:01 2008 (r185127) @@ -90,7 +90,7 @@ static int __cxio_init_resource_fifo(str u32 rarray[16]; mtx_init(fifo_lock, "cxio fifo", NULL, MTX_DEF|MTX_DUPOK); - *fifo = buf_ring_alloc(nr, M_NOWAIT); + *fifo = buf_ring_alloc(nr, M_DEVBUF, M_NOWAIT); if (*fifo == NULL) return (-ENOMEM); #if 0 @@ -149,7 +149,7 @@ static int cxio_init_qpid_fifo(struct cx mtx_init(&rdev_p->rscp->qpid_fifo_lock, "qpid fifo", NULL, MTX_DEF); - rdev_p->rscp->qpid_fifo = buf_ring_alloc(T3_MAX_NUM_QP, M_NOWAIT); + rdev_p->rscp->qpid_fifo = buf_ring_alloc(T3_MAX_NUM_QP, M_DEVBUF, M_NOWAIT); if (rdev_p->rscp->qpid_fifo == NULL) return (-ENOMEM); @@ -168,7 +168,7 @@ int cxio_hal_init_rhdl_resource(u32 nr_r void cxio_hal_destroy_rhdl_resource(void) { - buf_ring_free(rhdl_fifo); + buf_ring_free(rhdl_fifo, M_DEVBUF); } #endif @@ -202,11 +202,11 @@ int cxio_hal_init_resource(struct cxio_r goto pdid_err; return 0; pdid_err: - buf_ring_free(rscp->cqid_fifo); + buf_ring_free(rscp->cqid_fifo, M_DEVBUF); cqid_err: - buf_ring_free(rscp->qpid_fifo); + buf_ring_free(rscp->qpid_fifo, M_DEVBUF); qpid_err: - buf_ring_free(rscp->tpt_fifo); + buf_ring_free(rscp->tpt_fifo, M_DEVBUF); tpt_err: return (-ENOMEM); } @@ -276,10 +276,10 @@ void cxio_hal_put_pdid(struct cxio_hal_r void cxio_hal_destroy_resource(struct cxio_hal_resource *rscp) { - buf_ring_free(rscp->tpt_fifo); - buf_ring_free(rscp->cqid_fifo); - buf_ring_free(rscp->qpid_fifo); - buf_ring_free(rscp->pdid_fifo); + buf_ring_free(rscp->tpt_fifo, M_DEVBUF); + buf_ring_free(rscp->cqid_fifo, M_DEVBUF); + buf_ring_free(rscp->qpid_fifo, M_DEVBUF); + buf_ring_free(rscp->pdid_fifo, M_DEVBUF); free(rscp, M_DEVBUF); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200811201048.mAKAm1ra049339>