Date: Sat, 5 Mar 2011 03:42:04 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r219290 - head/sys/dev/cxgbe Message-ID: <201103050342.p253g42W001608@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Sat Mar 5 03:42:03 2011 New Revision: 219290 URL: http://svn.freebsd.org/changeset/base/219290 Log: Tweaks for rx: - everything related to LRO should be in #ifdef INET blocks - reorder sge_iq's fields so that the most frequently used are all together - pull all rx code into t4_intr_data directly - let go of the ingress queue lock when passing up data - refill the freelist only if it is short of at least 32 buffers Modified: head/sys/dev/cxgbe/adapter.h head/sys/dev/cxgbe/t4_sge.c Modified: head/sys/dev/cxgbe/adapter.h ============================================================================== --- head/sys/dev/cxgbe/adapter.h Sat Mar 5 03:27:14 2011 (r219289) +++ head/sys/dev/cxgbe/adapter.h Sat Mar 5 03:42:03 2011 (r219290) @@ -224,24 +224,25 @@ enum { struct sge_iq { bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; - struct mtx iq_lock; + bus_addr_t ba; /* bus address of descriptor ring */ char lockname[16]; - unsigned int flags; - struct adapter *adapter; + uint32_t flags; + uint16_t abs_id; /* absolute SGE id for the iq */ + int8_t intr_pktc_idx; /* packet count threshold index */ + int8_t pad0; + iq_intr_handler_t *handler; + __be64 *desc; /* KVA of descriptor ring */ - __be64 *desc; /* KVA of descriptor ring */ - bus_addr_t ba; /* bus address of descriptor ring */ + struct mtx iq_lock; + struct adapter *adapter; const __be64 *cdesc; /* current descriptor */ uint8_t gen; /* generation bit */ uint8_t intr_params; /* interrupt holdoff parameters */ - int8_t intr_pktc_idx; /* packet count threshold index */ uint8_t intr_next; /* holdoff for next interrupt */ uint8_t esize; /* size (bytes) of each entry in the queue */ uint16_t qsize; /* size (# of entries) of the queue */ uint16_t cidx; /* consumer index */ uint16_t cntxt_id; /* SGE context id for the iq */ - uint16_t abs_id; /* absolute SGE id for the iq */ - iq_intr_handler_t *handler; }; enum { @@ -342,9 +343,11 @@ struct sge_rxq { struct sge_iq iq; /* MUST be first */ struct sge_fl fl; - unsigned int flags; struct ifnet *ifp; /* the interface this rxq belongs to */ + unsigned int flags; +#ifdef INET struct lro_ctrl lro; /* LRO state */ +#endif /* stats for common events first */ Modified: head/sys/dev/cxgbe/t4_sge.c ============================================================================== --- head/sys/dev/cxgbe/t4_sge.c Sat Mar 5 03:27:14 2011 (r219289) +++ head/sys/dev/cxgbe/t4_sge.c Sat Mar 5 03:42:03 2011 (r219290) @@ -121,7 +121,6 @@ static int alloc_fl_sdesc(struct sge_fl static void free_fl_sdesc(struct sge_fl *); static int alloc_eq_maps(struct sge_eq *); static void free_eq_maps(struct sge_eq *); -static struct mbuf *get_fl_sdesc_data(struct sge_fl *, int, int); static void set_fl_tag_idx(struct sge_fl *, int); static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int); @@ -533,18 +532,22 @@ t4_intr_data(void *arg) struct sge_iq *iq = arg; struct adapter *sc = iq->adapter; struct rsp_ctrl *ctrl; - struct sge_fl *fl = &rxq->fl; struct ifnet *ifp = rxq->ifp; + struct sge_fl *fl = &rxq->fl; + struct fl_sdesc *sd = &fl->sdesc[fl->cidx], *sd_next; const struct rss_header *rss; const struct cpl_rx_pkt *cpl; - int ndescs = 0, rsp_type; uint32_t len; + int ndescs = 0, i; struct mbuf *m0, *m; #ifdef INET struct lro_ctrl *lro = &rxq->lro; struct lro_entry *l; #endif + prefetch(sd->m); + prefetch(sd->cl); + IQ_LOCK(iq); iq->intr_next = iq->intr_params; while (is_new_response(iq, &ctrl)) { @@ -552,30 +555,49 @@ t4_intr_data(void *arg) rmb(); rss = (const void *)iq->cdesc; - cpl = (const void *)(rss + 1); + i = G_RSPD_TYPE(ctrl->u.type_gen); - rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); + if (__predict_false(i == X_RSPD_TYPE_CPL)) { - if (__predict_false(rsp_type == X_RSPD_TYPE_CPL)) { /* Can't be anything except an egress update */ - handle_sge_egr_update(sc, (const void *)cpl); + KASSERT(rss->opcode == CPL_SGE_EGR_UPDATE, + ("%s: unexpected CPL %x", __func__, rss->opcode)); + + handle_sge_egr_update(sc, (const void *)(rss + 1)); goto nextdesc; } + KASSERT(i == X_RSPD_TYPE_FLBUF && rss->opcode == CPL_RX_PKT, + ("%s: unexpected CPL %x rsp %d", __func__, rss->opcode, i)); - KASSERT(G_RSPD_TYPE(ctrl->u.type_gen) == X_RSPD_TYPE_FLBUF, - ("unexpected event on data ingress queue: %x", - G_RSPD_TYPE(ctrl->u.type_gen))); + sd_next = sd + 1; + if (__predict_false(fl->cidx + 1 == fl->cap)) + sd_next = fl->sdesc; + prefetch(sd_next->m); + prefetch(sd_next->cl); - len = be32toh(ctrl->pldbuflen_qid); + cpl = (const void *)(rss + 1); - KASSERT(len & F_RSPD_NEWBUF, - ("%s: T4 misconfigured to pack buffers.", __func__)); + m0 = sd->m; + sd->m = NULL; /* consumed */ + len = be32toh(ctrl->pldbuflen_qid); + if (__predict_false((len & F_RSPD_NEWBUF) == 0)) + panic("%s: cannot handle packed frames", __func__); len = G_RSPD_LEN(len); - m0 = get_fl_sdesc_data(fl, len, M_PKTHDR); - if (m0 == NULL) { - iq->intr_next = V_QINTR_TIMER_IDX(SGE_NTIMERS - 1); - break; + + bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, + BUS_DMASYNC_POSTREAD); + + m_init(m0, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, M_PKTHDR); + if (len < MINCLSIZE) { + /* copy data to mbuf, buffer will be recycled */ + bcopy(sd->cl, mtod(m0, caddr_t), len); + m0->m_len = len; + } else { + bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map); + m_cljset(m0, sd->cl, FL_BUF_TYPE(sd->tag_idx)); + sd->cl = NULL; /* consumed */ + m0->m_len = min(len, FL_BUF_SIZE(sd->tag_idx)); } len -= FL_PKTSHIFT; @@ -604,16 +626,50 @@ t4_intr_data(void *arg) rxq->vlan_extraction++; } + i = 1; /* # of fl sdesc used */ + sd = sd_next; + if (__predict_false(++fl->cidx == fl->cap)) + fl->cidx = 0; + len -= m0->m_len; m = m0; while (len) { - m->m_next = get_fl_sdesc_data(fl, len, 0); - if (m->m_next == NULL) - CXGBE_UNIMPLEMENTED("mbuf recovery"); + i++; + + sd_next = sd + 1; + if (__predict_false(fl->cidx + 1 == fl->cap)) + sd_next = fl->sdesc; + prefetch(sd_next->m); + prefetch(sd_next->cl); + m->m_next = sd->m; + sd->m = NULL; /* consumed */ m = m->m_next; + + bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, + BUS_DMASYNC_POSTREAD); + + m_init(m, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, 0); + if (len <= MLEN) { + bcopy(sd->cl, mtod(m, caddr_t), len); + m->m_len = len; + } else { + bus_dmamap_unload(fl->tag[sd->tag_idx], + sd->map); + m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx)); + sd->cl = NULL; /* consumed */ + m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx)); + } + + i++; + sd = sd_next; + if (__predict_false(++fl->cidx == fl->cap)) + fl->cidx = 0; + len -= m->m_len; } + + IQ_UNLOCK(iq); #ifdef INET if (cpl->l2info & htobe32(F_RXF_LRO) && rxq->flags & RXQ_LRO_ENABLED && @@ -621,14 +677,15 @@ t4_intr_data(void *arg) /* queued for LRO */ } else #endif - (*ifp->if_input)(ifp, m0); + ifp->if_input(ifp, m0); + IQ_LOCK(iq); FL_LOCK(fl); - if (fl->needed >= 32) { + fl->needed += i; + if (fl->needed >= 32) refill_fl(fl, 64); - if (fl->pending >= 32) - ring_fl_db(sc, fl); - } + if (fl->pending >= 32) + ring_fl_db(sc, fl); FL_UNLOCK(fl); nextdesc: ndescs++; @@ -642,6 +699,7 @@ nextdesc: ndescs++; ndescs = 0; } } + IQ_UNLOCK(iq); #ifdef INET while (!SLIST_EMPTY(&lro->lro_active)) { @@ -654,14 +712,11 @@ nextdesc: ndescs++; t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); - IQ_UNLOCK(iq); - FL_LOCK(fl); - if (fl->needed) { - refill_fl(fl, -1); - if (fl->pending >= 8) - ring_fl_db(sc, fl); - } + if (fl->needed >= 32) + refill_fl(fl, 128); + if (fl->pending >= 8) + ring_fl_db(sc, fl); FL_UNLOCK(fl); } @@ -1201,10 +1256,12 @@ alloc_rxq(struct port_info *pi, struct s NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); +#ifdef INET SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, &rxq->lro.lro_queued, 0, NULL); SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, &rxq->lro.lro_flushed, 0, NULL); +#endif SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, &rxq->rxcsum, "# of times hardware assisted with checksum"); SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_extraction", @@ -1492,9 +1549,8 @@ refill_fl(struct sge_fl *fl, int nbufs) if (cl == NULL) break; - rc = bus_dmamap_load(tag, sd->map, cl, - FL_BUF_SIZE(sd->tag_idx), oneseg_dma_callback, - &pa, 0); + rc = bus_dmamap_load(tag, sd->map, cl, FL_BUF_SIZE(sd->tag_idx), + oneseg_dma_callback, &pa, 0); if (rc != 0 || pa == 0) { fl->dmamap_failed++; uma_zfree(FL_BUF_ZONE(sd->tag_idx), cl); @@ -1508,7 +1564,15 @@ refill_fl(struct sge_fl *fl, int nbufs) sd->ba_tag = htobe64(pa | sd->tag_idx); #endif -recycled: fl->pending++; +recycled: + /* sd->m is never recycled, should always be NULL */ + KASSERT(sd->m == NULL, ("%s: stray mbuf", __func__)); + + sd->m = m_gethdr(M_NOWAIT, MT_NOINIT); + if (sd->m == NULL) + break; + + fl->pending++; fl->needed--; sd++; if (++fl->pidx == fl->cap) { @@ -1516,10 +1580,6 @@ recycled: fl->pending++; sd = fl->sdesc; d = fl->desc; } - - /* No harm if gethdr fails, we'll retry after rx */ - if (sd->m == NULL) - sd->m = m_gethdr(M_NOWAIT, MT_NOINIT); } } @@ -2335,42 +2395,6 @@ get_flit(bus_dma_segment_t *sgl, int nse return (0); } -static struct mbuf * -get_fl_sdesc_data(struct sge_fl *fl, int len, int flags) -{ - struct fl_sdesc *sd; - struct mbuf *m; - - sd = &fl->sdesc[fl->cidx]; - FL_LOCK(fl); - if (++fl->cidx == fl->cap) - fl->cidx = 0; - fl->needed++; - FL_UNLOCK(fl); - - m = sd->m; - if (m == NULL) { - m = m_gethdr(M_NOWAIT, MT_NOINIT); - if (m == NULL) - return (NULL); - } - sd->m = NULL; /* consumed */ - - bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD); - m_init(m, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, flags); - if ((flags && len < MINCLSIZE) || (!flags && len <= MLEN)) - bcopy(sd->cl, mtod(m, caddr_t), len); - else { - bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map); - m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx)); - sd->cl = NULL; /* consumed */ - } - - m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx)); - - return (m); -} - static void set_fl_tag_idx(struct sge_fl *fl, int mtu) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201103050342.p253g42W001608>