Date: Mon, 13 Mar 2017 22:53:06 +0000 (UTC) From: Sean Bruno <sbruno@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r315217 - in head/sys: dev/bnxt dev/e1000 net Message-ID: <201703132253.v2DMr7Qq094175@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: sbruno Date: Mon Mar 13 22:53:06 2017 New Revision: 315217 URL: https://svnweb.freebsd.org/changeset/base/315217 Log: IFLIB updates - unconditionally enable BUS_DMA on non-x86 architectures - speed up rxd zeroing via customized function - support out of order updates to rxd's - add prefetching to hardware descriptor rings - only prefetch on 10G or faster hardware - add seperate tx queue intr function - preliminary rework of NETMAP interfaces, WIP Submitted by: Matt Macy <mmacy@nextbsd.org> Sponsored by: Limelight Networks Modified: head/sys/dev/bnxt/bnxt_txrx.c head/sys/dev/bnxt/if_bnxt.c head/sys/dev/e1000/em_txrx.c head/sys/dev/e1000/if_em.c head/sys/dev/e1000/if_em.h head/sys/dev/e1000/igb_txrx.c head/sys/net/ifdi_if.m head/sys/net/iflib.c head/sys/net/iflib.h Modified: head/sys/dev/bnxt/bnxt_txrx.c ============================================================================== --- head/sys/dev/bnxt/bnxt_txrx.c Mon Mar 13 22:45:51 2017 (r315216) +++ head/sys/dev/bnxt/bnxt_txrx.c Mon Mar 13 22:53:06 2017 (r315217) @@ -48,17 +48,19 @@ __FBSDID("$FreeBSD$"); */ static int bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi); -static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx); -static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t cidx, - bool clear); +static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx); +static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear); -static void bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid, +static void bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru); + +/* uint16_t rxqid, uint8_t flid, uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs, uint16_t count, uint16_t buf_size); +*/ static void bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid, - uint32_t pidx); -static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx, - int budget); + qidx_t pidx); +static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx, + qidx_t budget); static int bnxt_isc_rxd_pkt_get(void *sc, if_rxd_info_t ri); static int bnxt_intr(void *sc); @@ -172,7 +174,7 @@ bnxt_isc_txd_encap(void *sc, if_pkt_info } static void -bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx) +bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx) { struct bnxt_softc *softc = (struct bnxt_softc *)sc; struct bnxt_ring *tx_ring = &softc->tx_rings[txqid]; @@ -185,7 +187,7 @@ bnxt_isc_txd_flush(void *sc, uint16_t tx } static int -bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t idx, bool clear) +bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear) { struct bnxt_softc *softc = (struct bnxt_softc *)sc; struct bnxt_cp_ring *cpr = &softc->tx_cp_rings[txqid]; @@ -249,15 +251,27 @@ done: } static void -bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid, - uint32_t pidx, uint64_t *paddrs, - caddr_t *vaddrs, uint16_t count, uint16_t len) +bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru) { struct bnxt_softc *softc = (struct bnxt_softc *)sc; struct bnxt_ring *rx_ring; struct rx_prod_pkt_bd *rxbd; uint16_t type; uint16_t i; + uint16_t rxqid; + uint16_t count, len; + uint32_t pidx; + uint8_t flid; + uint64_t *paddrs; + caddr_t *vaddrs; + + rxqid = iru->iru_qsidx; + count = iru->iru_count; + len = iru->iru_buf_size; + pidx = iru->iru_pidx; + flid = iru->iru_flidx; + vaddrs = iru->iru_vaddrs; + paddrs = iru->iru_paddrs; if (flid == 0) { rx_ring = &softc->rx_rings[rxqid]; @@ -284,7 +298,7 @@ bnxt_isc_rxd_refill(void *sc, uint16_t r static void bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid, - uint32_t pidx) + qidx_t pidx) { struct bnxt_softc *softc = (struct bnxt_softc *)sc; struct bnxt_ring *rx_ring; @@ -310,7 +324,7 @@ bnxt_isc_rxd_flush(void *sc, uint16_t rx } static int -bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx, int budget) +bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx, qidx_t budget) { struct bnxt_softc *softc = (struct bnxt_softc *)sc; struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[rxqid]; @@ -412,37 +426,6 @@ cmpl_invalid: return avail; } -static void -bnxt_set_rsstype(if_rxd_info_t ri, uint8_t rss_hash_type) -{ - uint8_t rss_profile_id; - - rss_profile_id = BNXT_GET_RSS_PROFILE_ID(rss_hash_type); - switch (rss_profile_id) { - case BNXT_RSS_HASH_TYPE_TCPV4: - ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4; - break; - case BNXT_RSS_HASH_TYPE_UDPV4: - ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV4; - break; - case BNXT_RSS_HASH_TYPE_IPV4: - ri->iri_rsstype = M_HASHTYPE_RSS_IPV4; - break; - case BNXT_RSS_HASH_TYPE_TCPV6: - ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6; - break; - case BNXT_RSS_HASH_TYPE_UDPV6: - ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV6; - break; - case BNXT_RSS_HASH_TYPE_IPV6: - ri->iri_rsstype = M_HASHTYPE_RSS_IPV6; - break; - default: - ri->iri_rsstype = M_HASHTYPE_OPAQUE; - break; - } -} - static int bnxt_pkt_get_l2(struct bnxt_softc *softc, if_rxd_info_t ri, struct bnxt_cp_ring *cpr, uint16_t flags_type) @@ -460,7 +443,13 @@ bnxt_pkt_get_l2(struct bnxt_softc *softc /* Extract from the first 16-byte BD */ if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { ri->iri_flowid = le32toh(rcp->rss_hash); - bnxt_set_rsstype(ri, rcp->rss_hash_type); + /* + * TODO: Extract something useful from rcp->rss_hash_type + * (undocumented) + * May be documented in the "LSI ES" + * also check the firmware code. + */ + ri->iri_rsstype = M_HASHTYPE_OPAQUE; } else { ri->iri_rsstype = M_HASHTYPE_NONE; @@ -540,7 +529,13 @@ bnxt_pkt_get_tpa(struct bnxt_softc *soft /* Extract from the first 16-byte BD */ if (le16toh(tpas->low.flags_type) & RX_TPA_START_CMPL_FLAGS_RSS_VALID) { ri->iri_flowid = le32toh(tpas->low.rss_hash); - bnxt_set_rsstype(ri, tpas->low.rss_hash_type); + /* + * TODO: Extract something useful from tpas->low.rss_hash_type + * (undocumented) + * May be documented in the "LSI ES" + * also check the firmware code. + */ + ri->iri_rsstype = M_HASHTYPE_OPAQUE; } else { ri->iri_rsstype = M_HASHTYPE_NONE; Modified: head/sys/dev/bnxt/if_bnxt.c ============================================================================== --- head/sys/dev/bnxt/if_bnxt.c Mon Mar 13 22:45:51 2017 (r315216) +++ head/sys/dev/bnxt/if_bnxt.c Mon Mar 13 22:53:06 2017 (r315217) @@ -253,7 +253,8 @@ static device_method_t bnxt_iflib_method DEVMETHOD(ifdi_update_admin_status, bnxt_update_admin_status), DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable), - DEVMETHOD(ifdi_queue_intr_enable, bnxt_queue_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_queue_intr_enable), + DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_queue_intr_enable), DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr), DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign), @@ -711,6 +712,8 @@ bnxt_attach_pre(if_ctx_t ctx) scctx->isc_tx_tso_size_max = BNXT_TSO_SIZE; scctx->isc_tx_tso_segsize_max = BNXT_TSO_SIZE; scctx->isc_vectors = softc->func.max_cp_rings; + scctx->isc_txrx = &bnxt_txrx; + if (scctx->isc_nrxd[0] < ((scctx->isc_nrxd[1] * 4) + scctx->isc_nrxd[2])) device_printf(softc->dev, @@ -1479,7 +1482,7 @@ bnxt_msix_intr_assign(if_ctx_t ctx, int for (i=0; i<softc->scctx->isc_nrxqsets; i++) { rc = iflib_irq_alloc_generic(ctx, &softc->rx_cp_rings[i].irq, - softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RX, + softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RXTX, bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, "rx_cp"); if (rc) { device_printf(iflib_get_dev(ctx), Modified: head/sys/dev/e1000/em_txrx.c ============================================================================== --- head/sys/dev/e1000/em_txrx.c Mon Mar 13 22:45:51 2017 (r315216) +++ head/sys/dev/e1000/em_txrx.c Mon Mar 13 22:53:06 2017 (r315217) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> + * Copyright (c) 2016-2017 Matt Macy <mmacy@nextbsd.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,7 +27,7 @@ /* $FreeBSD$ */ #include "if_em.h" -#ifdef RSS +#ifdef RSS #include <net/rss_config.h> #include <netinet/in_rss.h> #endif @@ -41,23 +41,24 @@ /********************************************************************* * Local Function prototypes *********************************************************************/ -static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); -static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); +static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, + u32 *txd_lower); +static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, + u32 *txd_upper, u32 *txd_lower); static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); -static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); -static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear); -static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, - uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); -static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); -static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, - int budget); +static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); +static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); +static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); +static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, + qidx_t pidx); +static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, + qidx_t budget); static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); -static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, - uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); +static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); -static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, - int budget); +static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, + qidx_t budget); static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); @@ -65,7 +66,7 @@ static void em_receive_checksum(uint32_t static int em_determine_rsstype(u32 pkt_info); extern int em_intr(void *arg); -struct if_txrx em_txrx = { +struct if_txrx em_txrx = { em_isc_txd_encap, em_isc_txd_flush, em_isc_txd_credits_update, @@ -76,7 +77,7 @@ struct if_txrx em_txrx = { em_intr }; -struct if_txrx lem_txrx = { +struct if_txrx lem_txrx = { em_isc_txd_encap, em_isc_txd_flush, em_isc_txd_credits_update, @@ -87,7 +88,42 @@ struct if_txrx lem_txrx = { em_intr }; -extern if_shared_ctx_t em_sctx; +extern if_shared_ctx_t em_sctx; + +void +em_dump_rs(struct adapter *adapter) +{ + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que; + struct tx_ring *txr; + qidx_t i, ntxd, qid, cur; + int16_t rs_cidx; + uint8_t status; + + printf("\n"); + ntxd = scctx->isc_ntxd[0]; + for (qid = 0; qid < adapter->tx_num_queues; qid++) { + que = &adapter->tx_queues[qid]; + txr = &que->txr; + rs_cidx = txr->tx_rs_cidx; + if (rs_cidx != txr->tx_rs_pidx) { + cur = txr->tx_rsq[rs_cidx]; + status = txr->tx_base[cur].upper.fields.status; + if (!(status & E1000_TXD_STAT_DD)) + printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); + } else { + rs_cidx = (rs_cidx-1)&(ntxd-1); + cur = txr->tx_rsq[rs_cidx]; + printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); + } + printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx); + for (i = 0; i < ntxd; i++) { + if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) + printf("%d set ", i); + } + printf("\n"); + } +} /********************************************************************** * @@ -99,14 +135,13 @@ static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) { if_softc_ctx_t scctx = adapter->shared; - struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; - struct tx_ring *txr = &que->txr; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; struct e1000_context_desc *TXD; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; + int cur, hdr_len; hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; - *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ + *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ E1000_TXD_DTYP_D | /* Data descr type */ E1000_TXD_CMD_TSE); /* Do TSE on this packet */ @@ -114,10 +149,9 @@ em_tso_setup(struct adapter *adapter, if *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; cur = pi->ipi_pidx; - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - tx_buffer = &txr->tx_buffers[cur]; - - /* + TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; + + /* * Start offset for header checksum calculation. * End offset for header checksum calculation. * Offset of place put the checksum. @@ -127,7 +161,7 @@ em_tso_setup(struct adapter *adapter, if htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); - /* + /* * Start offset for payload checksum calculation. * End offset for payload checksum calculation. * Offset of place to put the checksum. @@ -136,8 +170,8 @@ em_tso_setup(struct adapter *adapter, if TXD->upper_setup.tcp_fields.tucse = 0; TXD->upper_setup.tcp_fields.tucso = pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); - - /* + + /* * Payload size per packet w/o any headers. * Length of all headers up to payload. */ @@ -150,7 +184,6 @@ em_tso_setup(struct adapter *adapter, if E1000_TXD_CMD_IP | /* Do IP csum */ E1000_TXD_CMD_TCP | /* Do TCP checksum */ (pi->ipi_len - hdr_len)); /* Total len */ - tx_buffer->eop = -1; txr->tx_tso = TRUE; if (++cur == scctx->isc_ntxd[0]) { @@ -178,21 +211,20 @@ em_tso_setup(struct adapter *adapter, if * until the previous request completes. This means setting up * a new context effectively disables pipelined Tx data DMA which * in turn greatly slow down performance to send small sized - * frames. + * frames. **********************************************************************/ static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) { - struct e1000_context_desc *TXD = NULL; - if_softc_ctx_t scctx = adapter->shared; - struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; - struct tx_ring *txr = &que->txr; - struct em_txbuffer *tx_buffer; - int csum_flags = pi->ipi_csum_flags; - int cur, hdr_len; - u32 cmd; - + struct e1000_context_desc *TXD = NULL; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + int csum_flags = pi->ipi_csum_flags; + int cur, hdr_len; + u32 cmd; + cur = pi->ipi_pidx; hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; cmd = adapter->txd_cmd; @@ -220,7 +252,7 @@ em_transmit_checksum_setup(struct adapte TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; if (csum_flags & CSUM_IP) { - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; + *txd_upper |= E1000_TXD_POPTS_IXSM << 8; /* * Start offset for header checksum calculation. * End offset for header checksum calculation. @@ -235,7 +267,7 @@ em_transmit_checksum_setup(struct adapte if (csum_flags & (CSUM_TCP|CSUM_UDP)) { uint8_t tucso; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; + *txd_upper |= E1000_TXD_POPTS_TXSM << 8; *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; if (csum_flags & CSUM_TCP) { @@ -243,9 +275,9 @@ em_transmit_checksum_setup(struct adapte cmd |= E1000_TXD_CMD_TCP; } else tucso = hdr_len + offsetof(struct udphdr, uh_sum); - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; + TXD->upper_setup.tcp_fields.tucss = hdr_len; + TXD->upper_setup.tcp_fields.tucse = htole16(0); + TXD->upper_setup.tcp_fields.tucso = tucso; } txr->csum_lhlen = pi->ipi_ehdrlen; @@ -258,9 +290,6 @@ em_transmit_checksum_setup(struct adapte TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &txr->tx_buffers[cur]; - tx_buffer->eop = -1; - if (++cur == scctx->isc_ntxd[0]) { cur = 0; } @@ -272,24 +301,26 @@ em_transmit_checksum_setup(struct adapte static int em_isc_txd_encap(void *arg, if_pkt_info_t pi) { - struct adapter *sc = arg; - if_softc_ctx_t scctx = sc->shared; - struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; - struct tx_ring *txr = &que->txr; - bus_dma_segment_t *segs = pi->ipi_segs; - int nsegs = pi->ipi_nsegs; - int csum_flags = pi->ipi_csum_flags; - int i, j, first, pidx_last; - u32 txd_upper = 0, txd_lower = 0; - - struct em_txbuffer *tx_buffer; + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + bus_dma_segment_t *segs = pi->ipi_segs; + int nsegs = pi->ipi_nsegs; + int csum_flags = pi->ipi_csum_flags; + int i, j, first, pidx_last; + u32 txd_flags, txd_upper = 0, txd_lower = 0; + struct e1000_tx_desc *ctxd = NULL; - bool do_tso, tso_desc; - - i = first = pi->ipi_pidx; + bool do_tso, tso_desc; + qidx_t ntxd; + + txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; + i = first = pi->ipi_pidx; do_tso = (csum_flags & CSUM_TSO); tso_desc = FALSE; - /* + ntxd = scctx->isc_ntxd[0]; + /* * TSO Hardware workaround, if this packet is not * TSO, and is only a single descriptor long, and * it follows a TSO burst, then we need to add a @@ -310,10 +341,10 @@ em_isc_txd_encap(void *arg, if_pkt_info_ } if (pi->ipi_mflags & M_VLANTAG) { - /* Set the vlan id. */ + /* Set the vlan id. */ txd_upper |= htole16(pi->ipi_vtag) << 16; - /* Tell hardware to add tag */ - txd_lower |= htole32(E1000_TXD_CMD_VLE); + /* Tell hardware to add tag */ + txd_lower |= htole32(E1000_TXD_CMD_VLE); } DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); @@ -326,28 +357,26 @@ em_isc_txd_encap(void *arg, if_pkt_info_ uint32_t cmd; ctxd = &txr->tx_base[i]; - tx_buffer = &txr->tx_buffers[i]; seg_addr = segs[j].ds_addr; seg_len = segs[j].ds_len; cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; /* - ** TSO Workaround: - ** If this is the last descriptor, we want to - ** split it so we have a small final sentinel - */ + * TSO Workaround: + * If this is the last descriptor, we want to + * split it so we have a small final sentinel + */ if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { seg_len -= TSO_WORKAROUND; ctxd->buffer_addr = htole64(seg_addr); ctxd->lower.data = htole32(cmd | txd_lower | seg_len); ctxd->upper.data = htole32(txd_upper); - if (++i == scctx->isc_ntxd[0]) + if (++i == scctx->isc_ntxd[0]) i = 0; /* Now make the sentinel */ ctxd = &txr->tx_base[i]; - tx_buffer = &txr->tx_buffers[i]; ctxd->buffer_addr = htole64(seg_addr + seg_len); ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); ctxd->upper.data = htole32(txd_upper); @@ -364,27 +393,28 @@ em_isc_txd_encap(void *arg, if_pkt_info_ i = 0; DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); } - tx_buffer->eop = -1; } /* - * Last Descriptor of Packet + * Last Descriptor of Packet * needs End Of Packet (EOP) * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - - tx_buffer = &txr->tx_buffers[first]; - tx_buffer->eop = pidx_last; + */ + if (txd_flags) { + txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; + DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first); + txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); + MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); + } + ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); pi->ipi_new_pidx = i; - return (0); + return (0); } static void -em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) +em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) { struct adapter *adapter = arg; struct em_tx_queue *que = &adapter->tx_queues[txqid]; @@ -394,86 +424,72 @@ em_isc_txd_flush(void *arg, uint16_t txq } static int -em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear) +em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) { struct adapter *adapter = arg; if_softc_ctx_t scctx = adapter->shared; struct em_tx_queue *que = &adapter->tx_queues[txqid]; struct tx_ring *txr = &que->txr; - u32 cidx, processed = 0; - int last, done; - struct em_txbuffer *buf; - struct e1000_tx_desc *tx_desc, *eop_desc; - - cidx = cidx_init; - buf = &txr->tx_buffers[cidx]; - tx_desc = &txr->tx_base[cidx]; - last = buf->eop; - if (last == -1) - return (processed); - eop_desc = &txr->tx_base[last]; - - DPRINTF(iflib_get_dev(adapter->ctx), - "credits_update: cidx_init=%d clear=%d last=%d\n", - cidx_init, clear, last); - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == scctx->isc_ntxd[0]) - last = 0; - done = last; - - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (cidx != done) { - if (clear) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - buf->eop = -1; - } - tx_desc++; - buf++; - processed++; - - /* wrap the ring ? */ - if (++cidx == scctx->isc_ntxd[0]) { - cidx = 0; - } - buf = &txr->tx_buffers[cidx]; - tx_desc = &txr->tx_base[cidx]; - } - /* See if we can continue to the next packet */ - last = buf->eop; - if (last == -1) + qidx_t processed = 0; + int updated; + qidx_t cur, prev, ntxd, rs_cidx; + int32_t delta; + uint8_t status; + + rs_cidx = txr->tx_rs_cidx; + if (rs_cidx == txr->tx_rs_pidx) + return (0); + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + status = txr->tx_base[cur].upper.fields.status; + updated = !!(status & E1000_TXD_STAT_DD); + + if (clear == false || updated == 0) + return (updated); + + prev = txr->tx_cidx_processed; + ntxd = scctx->isc_ntxd[0]; + do { + delta = (int32_t)cur - (int32_t)prev; + MPASS(prev == 0 || delta != 0); + if (delta < 0) + delta += ntxd; + DPRINTF(iflib_get_dev(adapter->ctx), + "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", + __FUNCTION__, prev, cur, clear, delta); + + processed += delta; + prev = cur; + rs_cidx = (rs_cidx + 1) & (ntxd-1); + if (rs_cidx == txr->tx_rs_pidx) break; - eop_desc = &txr->tx_base[last]; - /* Get new done point */ - if (++last == scctx->isc_ntxd[0]) - last = 0; - done = last; - } + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + status = txr->tx_base[cur].upper.fields.status; + } while ((status & E1000_TXD_STAT_DD)); - DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed); + txr->tx_rs_cidx = rs_cidx; + txr->tx_cidx_processed = prev; return(processed); } static void -lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, - uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) +lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) { struct adapter *sc = arg; if_softc_ctx_t scctx = sc->shared; - struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; struct rx_ring *rxr = &que->rxr; struct e1000_rx_desc *rxd; + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; int i; - uint32_t next_pidx; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; for (i = 0, next_pidx = pidx; i < count; i++) { rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; @@ -487,48 +503,60 @@ lem_isc_rxd_refill(void *arg, uint16_t r } static void -em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, - uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) +em_isc_rxd_refill(void *arg, if_rxd_update_t iru) { struct adapter *sc = arg; if_softc_ctx_t scctx = sc->shared; + uint16_t rxqid = iru->iru_qsidx; struct em_rx_queue *que = &sc->rx_queues[rxqid]; struct rx_ring *rxr = &que->rxr; union e1000_rx_desc_extended *rxd; + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; int i; - uint32_t next_pidx; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; for (i = 0, next_pidx = pidx; i < count; i++) { rxd = &rxr->rx_base[next_pidx]; rxd->read.buffer_addr = htole64(paddrs[i]); /* DD bits must be cleared */ - rxd->wb.upper.status_error = 0; - + rxd->wb.upper.status_error = 0; + if (++next_pidx == scctx->isc_nrxd[0]) next_pidx = 0; } } static void -em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) +em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) { - struct adapter *sc = arg; - struct em_rx_queue *que = &sc->rx_queues[rxqid]; - struct rx_ring *rxr = &que->rxr; + struct adapter *sc = arg; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; - E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); + E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); } static int -lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) +lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) { - struct adapter *sc = arg; + struct adapter *sc = arg; if_softc_ctx_t scctx = sc->shared; - struct em_rx_queue *que = &sc->rx_queues[rxqid]; - struct rx_ring *rxr = &que->rxr; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; struct e1000_rx_desc *rxd; - u32 staterr = 0; - int cnt, i; + u32 staterr = 0; + int cnt, i; + + if (budget == 1) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[idx]; + staterr = rxd->status; + return (staterr & E1000_RXD_STAT_DD); + } for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; @@ -547,15 +575,21 @@ lem_isc_rxd_available(void *arg, uint16_ } static int -em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) +em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) { - struct adapter *sc = arg; + struct adapter *sc = arg; if_softc_ctx_t scctx = sc->shared; - struct em_rx_queue *que = &sc->rx_queues[rxqid]; - struct rx_ring *rxr = &que->rxr; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; union e1000_rx_desc_extended *rxd; - u32 staterr = 0; - int cnt, i; + u32 staterr = 0; + int cnt, i; + + if (budget == 1) { + rxd = &rxr->rx_base[idx]; + staterr = le32toh(rxd->wb.upper.status_error); + return (staterr & E1000_RXD_STAT_DD); + } for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { rxd = &rxr->rx_base[i]; @@ -578,15 +612,15 @@ em_isc_rxd_available(void *arg, uint16_t static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) { - struct adapter *adapter = arg; - if_softc_ctx_t scctx = adapter->shared; - struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; - struct rx_ring *rxr = &que->rxr; + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; struct e1000_rx_desc *rxd; - u16 len; - u32 status, errors; - bool eop; - int i, cidx; + u16 len; + u32 status, errors; + bool eop; + int i, cidx; status = errors = i = 0; cidx = ri->iri_cidx; @@ -639,31 +673,31 @@ lem_isc_rxd_pkt_get(void *arg, if_rxd_in static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) { - struct adapter *adapter = arg; - if_softc_ctx_t scctx = adapter->shared; - struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; - struct rx_ring *rxr = &que->rxr; + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; union e1000_rx_desc_extended *rxd; - u16 len; - u32 pkt_info; - u32 staterr = 0; - bool eop; - int i, cidx, vtag; + u16 len; + u32 pkt_info; + u32 staterr = 0; + bool eop; + int i, cidx, vtag; i = vtag = 0; cidx = ri->iri_cidx; do { rxd = &rxr->rx_base[cidx]; - staterr = le32toh(rxd->wb.upper.status_error); + staterr = le32toh(rxd->wb.upper.status_error); pkt_info = le32toh(rxd->wb.lower.mrq); - + /* Error Checking then decrement count */ MPASS ((staterr & E1000_RXD_STAT_DD) != 0); len = le16toh(rxd->wb.upper.length); - ri->iri_len += len; + ri->iri_len += len; eop = (staterr & E1000_RXD_STAT_EOP) != 0; @@ -690,15 +724,14 @@ em_isc_rxd_pkt_get(void *arg, if_rxd_inf if (staterr & E1000_RXD_STAT_VP) { vtag = le16toh(rxd->wb.upper.vlan); - } - + } + ri->iri_vtag = vtag; if (vtag) ri->iri_flags |= M_VLANTAG; - - ri->iri_flowid = - le32toh(rxd->wb.lower.hi_dword.rss); - ri->iri_rsstype = em_determine_rsstype(pkt_info); + + ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); + ri->iri_rsstype = em_determine_rsstype(pkt_info); ri->iri_nfrags = i; return (0); @@ -736,23 +769,24 @@ lem_receive_checksum(int status, int err static int em_determine_rsstype(u32 pkt_info) { - switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { - case E1000_RXDADV_RSSTYPE_IPV4_TCP: - return M_HASHTYPE_RSS_TCP_IPV4; - case E1000_RXDADV_RSSTYPE_IPV4: - return M_HASHTYPE_RSS_IPV4; - case E1000_RXDADV_RSSTYPE_IPV6_TCP: - return M_HASHTYPE_RSS_TCP_IPV6; - case E1000_RXDADV_RSSTYPE_IPV6_EX: - return M_HASHTYPE_RSS_IPV6_EX; - case E1000_RXDADV_RSSTYPE_IPV6: - return M_HASHTYPE_RSS_IPV6; - case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: - return M_HASHTYPE_RSS_TCP_IPV6_EX; - default: - return M_HASHTYPE_OPAQUE; - } + switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { + case E1000_RXDADV_RSSTYPE_IPV4_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV4: + return M_HASHTYPE_RSS_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_TCP: + return M_HASHTYPE_RSS_TCP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_EX: + return M_HASHTYPE_RSS_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV6: + return M_HASHTYPE_RSS_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: + return M_HASHTYPE_RSS_TCP_IPV6_EX; + default: + return M_HASHTYPE_OPAQUE; + } } + static void em_receive_checksum(uint32_t status, if_rxd_info_t ri) { @@ -764,7 +798,7 @@ em_receive_checksum(uint32_t status, if_ /* If the IP checksum exists and there is no IP Checksum error */ if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == - E1000_RXD_STAT_IPCS) { + E1000_RXD_STAT_IPCS) { ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); } Modified: head/sys/dev/e1000/if_em.c ============================================================================== --- head/sys/dev/e1000/if_em.c Mon Mar 13 22:45:51 2017 (r315216) +++ head/sys/dev/e1000/if_em.c Mon Mar 13 22:53:06 2017 (r315217) @@ -50,113 +50,113 @@ char em_driver_version[] = "7.6.1-k"; static pci_vendor_info_t em_vendor_info_array[] = { /* Intel(R) PRO/1000 Network Connection - Legacy em*/ - PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), - PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"), - - PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"), *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201703132253.v2DMr7Qq094175>