From owner-svn-src-all@freebsd.org Tue Dec 1 14:55:26 2015 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2CA77A3ED24; Tue, 1 Dec 2015 14:55:26 +0000 (UTC) (envelope-from arybchik@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id DDCC41E0B; Tue, 1 Dec 2015 14:55:25 +0000 (UTC) (envelope-from arybchik@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id tB1EtOSI079340; Tue, 1 Dec 2015 14:55:24 GMT (envelope-from arybchik@FreeBSD.org) Received: (from arybchik@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id tB1EtOjs079338; Tue, 1 Dec 2015 14:55:24 GMT (envelope-from arybchik@FreeBSD.org) Message-Id: <201512011455.tB1EtOjs079338@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: arybchik set sender to arybchik@FreeBSD.org using -f From: Andrew Rybchenko Date: Tue, 1 Dec 2015 14:55:24 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r291584 - head/sys/dev/sfxge X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 01 Dec 2015 14:55:26 -0000 Author: arybchik Date: Tue Dec 1 14:55:24 2015 New Revision: 291584 URL: https://svnweb.freebsd.org/changeset/base/291584 Log: sfxge: parse packets for TSO early in if_transmit Submitted by: Artem V. Andreev Sponsored by: Solarflare Communications, Inc. MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D4309 Modified: head/sys/dev/sfxge/sfxge_tx.c head/sys/dev/sfxge/sfxge_tx.h Modified: head/sys/dev/sfxge/sfxge_tx.c ============================================================================== --- head/sys/dev/sfxge/sfxge_tx.c Tue Dec 1 14:02:14 2015 (r291583) +++ head/sys/dev/sfxge/sfxge_tx.c Tue Dec 1 14:55:24 2015 (r291584) @@ -710,6 +710,84 @@ sfxge_if_qflush(struct ifnet *ifp) sfxge_tx_qdpl_flush(sc->txq[i]); } +#if SFXGE_TX_PARSE_EARLY + +/* There is little space for user data in mbuf pkthdr, so we + * use l*hlen fields which are not used by the driver otherwise + * to store header offsets. + * The fields are 8-bit, but it's ok, no header may be longer than 255 bytes. + */ + + +#define TSO_MBUF_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[0]) +/* We abuse l5hlen here because PH_loc can hold only 64 bits of data */ +#define TSO_MBUF_FLAGS(_mbuf) ((_mbuf)->m_pkthdr.l5hlen) +#define TSO_MBUF_PACKETID(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[1]) +#define TSO_MBUF_SEQNUM(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.thirtytwo[1]) + +static void sfxge_parse_tx_packet(struct mbuf *mbuf) +{ + struct ether_header *eh = mtod(mbuf, struct ether_header *); + const struct tcphdr *th; + struct tcphdr th_copy; + + /* Find network protocol and header */ + TSO_MBUF_PROTO(mbuf) = eh->ether_type; + if (TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_VLAN)) { + struct ether_vlan_header *veh = + mtod(mbuf, struct ether_vlan_header *); + TSO_MBUF_PROTO(mbuf) = veh->evl_proto; + mbuf->m_pkthdr.l2hlen = sizeof(*veh); + } else { + mbuf->m_pkthdr.l2hlen = sizeof(*eh); + } + + /* Find TCP header */ + if (TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_IP)) { + const struct ip *iph = (const struct ip *)mtodo(mbuf, mbuf->m_pkthdr.l2hlen); + + KASSERT(iph->ip_p == IPPROTO_TCP, + ("TSO required on non-TCP packet")); + mbuf->m_pkthdr.l3hlen = mbuf->m_pkthdr.l2hlen + 4 * iph->ip_hl; + TSO_MBUF_PACKETID(mbuf) = iph->ip_id; + } else { + KASSERT(TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_IPV6), + ("TSO required on non-IP packet")); + KASSERT(((const struct ip6_hdr *)mtodo(mbuf, mbuf->m_pkthdr.l2hlen))->ip6_nxt == + IPPROTO_TCP, + ("TSO required on non-TCP packet")); + mbuf->m_pkthdr.l3hlen = mbuf->m_pkthdr.l2hlen + sizeof(struct ip6_hdr); + TSO_MBUF_PACKETID(mbuf) = 0; + } + + KASSERT(mbuf->m_len >= mbuf->m_pkthdr.l3hlen, + ("network header is fragmented in mbuf")); + + /* We need TCP header including flags (window is the next) */ + if (mbuf->m_len < mbuf->m_pkthdr.l3hlen + offsetof(struct tcphdr, th_win)) { + m_copydata(mbuf, mbuf->m_pkthdr.l3hlen, sizeof(th_copy), + (caddr_t)&th_copy); + th = &th_copy; + } else { + th = (const struct tcphdr *)mtodo(mbuf, mbuf->m_pkthdr.l3hlen); + } + + mbuf->m_pkthdr.l4hlen = mbuf->m_pkthdr.l3hlen + 4 * th->th_off; + TSO_MBUF_SEQNUM(mbuf) = ntohl(th->th_seq); + + /* These flags must not be duplicated */ + /* + * RST should not be duplicated as well, but FreeBSD kernel + * generates TSO packets with RST flag. So, do not assert + * its absence. + */ + KASSERT(!(th->th_flags & (TH_URG | TH_SYN)), + ("incompatible TCP flag 0x%x on TSO packet", + th->th_flags & (TH_URG | TH_SYN))); + TSO_MBUF_FLAGS(mbuf) = th->th_flags; +} +#endif + /* * TX start -- called by the stack. */ @@ -744,6 +822,10 @@ sfxge_if_transmit(struct ifnet *ifp, str index = sc->rx_indir_table[hash % SFXGE_RX_SCALE_MAX]; } +#if SFXGE_TX_PARSE_EARLY + if (m->m_pkthdr.csum_flags & CSUM_TSO) + sfxge_parse_tx_packet(m); +#endif txq = sc->txq[SFXGE_TXQ_IP_TCP_UDP_CKSUM + index]; } else if (m->m_pkthdr.csum_flags & CSUM_DELAY_IP) { txq = sc->txq[SFXGE_TXQ_IP_CKSUM]; @@ -781,26 +863,32 @@ struct sfxge_tso_state { unsigned seg_size; /* TCP segment size */ int fw_assisted; /* Use FW-assisted TSO */ u_short packet_id; /* IPv4 packet ID from the original packet */ + uint8_t tcp_flags; /* TCP flags */ efx_desc_t header_desc; /* Precomputed header descriptor for * FW-assisted TSO */ }; +#if !SFXGE_TX_PARSE_EARLY static const struct ip *tso_iph(const struct sfxge_tso_state *tso) { KASSERT(tso->protocol == htons(ETHERTYPE_IP), ("tso_iph() in non-IPv4 state")); return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); } + static __unused const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) { KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), ("tso_ip6h() in non-IPv6 state")); return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); } + static const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) { return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); } +#endif + /* Size of preallocated TSO header buffers. Larger blocks must be * allocated from the heap. @@ -857,15 +945,18 @@ static void tso_start(struct sfxge_txq * const bus_dma_segment_t *hdr_dma_seg, struct mbuf *mbuf) { - struct ether_header *eh = mtod(mbuf, struct ether_header *); const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->sc->enp); +#if !SFXGE_TX_PARSE_EARLY + struct ether_header *eh = mtod(mbuf, struct ether_header *); const struct tcphdr *th; struct tcphdr th_copy; +#endif tso->fw_assisted = txq->sc->tso_fw_assisted; tso->mbuf = mbuf; /* Find network protocol and header */ +#if !SFXGE_TX_PARSE_EARLY tso->protocol = eh->ether_type; if (tso->protocol == htons(ETHERTYPE_VLAN)) { struct ether_vlan_header *veh = @@ -875,7 +966,14 @@ static void tso_start(struct sfxge_txq * } else { tso->nh_off = sizeof(*eh); } +#else + tso->protocol = TSO_MBUF_PROTO(mbuf); + tso->nh_off = mbuf->m_pkthdr.l2hlen; + tso->tcph_off = mbuf->m_pkthdr.l3hlen; + tso->packet_id = TSO_MBUF_PACKETID(mbuf); +#endif +#if !SFXGE_TX_PARSE_EARLY /* Find TCP header */ if (tso->protocol == htons(ETHERTYPE_IP)) { KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP, @@ -890,12 +988,17 @@ static void tso_start(struct sfxge_txq * tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr); tso->packet_id = 0; } +#endif + + if (tso->fw_assisted && __predict_false(tso->tcph_off > encp->enc_tx_tso_tcp_header_offset_limit)) { tso->fw_assisted = 0; } + +#if !SFXGE_TX_PARSE_EARLY KASSERT(mbuf->m_len >= tso->tcph_off, ("network header is fragmented in mbuf")); /* We need TCP header including flags (window is the next) */ @@ -906,10 +1009,13 @@ static void tso_start(struct sfxge_txq * } else { th = tso_tcph(tso); } - tso->header_len = tso->tcph_off + 4 * th->th_off; +#else + tso->header_len = mbuf->m_pkthdr.l4hlen; +#endif tso->seg_size = mbuf->m_pkthdr.tso_segsz; +#if !SFXGE_TX_PARSE_EARLY tso->seqnum = ntohl(th->th_seq); /* These flags must not be duplicated */ @@ -921,6 +1027,11 @@ static void tso_start(struct sfxge_txq * KASSERT(!(th->th_flags & (TH_URG | TH_SYN)), ("incompatible TCP flag 0x%x on TSO packet", th->th_flags & (TH_URG | TH_SYN))); + tso->tcp_flags = th->th_flags; +#else + tso->seqnum = TSO_MBUF_SEQNUM(mbuf); + tso->tcp_flags = TSO_MBUF_FLAGS(mbuf); +#endif tso->out_len = mbuf->m_pkthdr.len - tso->header_len; @@ -1001,7 +1112,7 @@ static int tso_start_new_packet(struct s int rc; if (tso->fw_assisted) { - uint8_t tcp_flags = tso_tcph(tso)->th_flags; + uint8_t tcp_flags = tso->tcp_flags; if (tso->out_len > tso->seg_size) tcp_flags &= ~(TH_FIN | TH_PUSH); Modified: head/sys/dev/sfxge/sfxge_tx.h ============================================================================== --- head/sys/dev/sfxge/sfxge_tx.h Tue Dec 1 14:02:14 2015 (r291583) +++ head/sys/dev/sfxge/sfxge_tx.h Tue Dec 1 14:55:24 2015 (r291584) @@ -40,6 +40,11 @@ #include #include +/* If defined, parse TX packets directly in if_transmit + * for better cache locality and reduced time under TX lock + */ +#define SFXGE_TX_PARSE_EARLY 1 + /* Maximum size of TSO packet */ #define SFXGE_TSO_MAX_SIZE (65535)