From owner-svn-src-head@FreeBSD.ORG Tue Sep 30 20:57:26 2014 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 2824560B; Tue, 30 Sep 2014 20:57:26 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 143B8CAB; Tue, 30 Sep 2014 20:57:26 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id s8UKvPLR013019; Tue, 30 Sep 2014 20:57:25 GMT (envelope-from gnn@FreeBSD.org) Received: (from gnn@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id s8UKvP4W013017; Tue, 30 Sep 2014 20:57:25 GMT (envelope-from gnn@FreeBSD.org) Message-Id: <201409302057.s8UKvP4W013017@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: gnn set sender to gnn@FreeBSD.org using -f From: "George V. Neville-Neil" Date: Tue, 30 Sep 2014 20:57:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r272331 - head/sys/dev/sfxge X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 30 Sep 2014 20:57:26 -0000 Author: gnn Date: Tue Sep 30 20:57:25 2014 New Revision: 272331 URL: http://svnweb.freebsd.org/changeset/base/272331 Log: Support tunable to control Tx deferred packet list limits Also increase default for Tx queue get-list limit. Too small limit results in TCP packets drops especiall when many streams are running simultaneously. Put list may be kept small enough since it is just a temporary location if transmit function can't get Tx queue lock. Submitted by: Andrew Rybchenko Sponsored by: Solarflare Communications, Inc. Modified: head/sys/dev/sfxge/sfxge_tx.c head/sys/dev/sfxge/sfxge_tx.h Modified: head/sys/dev/sfxge/sfxge_tx.c ============================================================================== --- head/sys/dev/sfxge/sfxge_tx.c Tue Sep 30 20:43:21 2014 (r272330) +++ head/sys/dev/sfxge/sfxge_tx.c Tue Sep 30 20:57:25 2014 (r272331) @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -77,6 +78,25 @@ __FBSDID("$FreeBSD$"); #define SFXGE_TSO_MAX_DESC ((65535 / 512) * 2 + SFXGE_TX_MAPPING_MAX_SEG - 1) #define SFXGE_TXQ_BLOCK_LEVEL(_entries) ((_entries) - SFXGE_TSO_MAX_DESC) +#ifdef SFXGE_HAVE_MQ + +#define SFXGE_PARAM_TX_DPL_GET_MAX SFXGE_PARAM(tx_dpl_get_max) +static int sfxge_tx_dpl_get_max = SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT; +TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_MAX, &sfxge_tx_dpl_get_max); +SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_max, CTLFLAG_RDTUN, + &sfxge_tx_dpl_get_max, 0, + "Maximum number of packets in deferred packet get-list"); + +#define SFXGE_PARAM_TX_DPL_PUT_MAX SFXGE_PARAM(tx_dpl_put_max) +static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT; +TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max); +SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN, + &sfxge_tx_dpl_put_max, 0, + "Maximum number of packets in deferred packet put-list"); + +#endif + + /* Forward declarations. */ static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq); static void sfxge_tx_qlist_post(struct sfxge_txq *txq); @@ -476,7 +496,7 @@ sfxge_tx_qdpl_put(struct sfxge_txq *txq, sfxge_tx_qdpl_swizzle(txq); - if (stdp->std_get_count >= SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT) + if (stdp->std_get_count >= stdp->std_get_max) return (ENOBUFS); *(stdp->std_getp) = mbuf; @@ -498,7 +518,7 @@ sfxge_tx_qdpl_put(struct sfxge_txq *txq, old_len = mp->m_pkthdr.csum_data; } else old_len = 0; - if (old_len >= SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT) + if (old_len >= stdp->std_put_max) return (ENOBUFS); mbuf->m_pkthdr.csum_data = old_len + 1; mbuf->m_nextpkt = (void *)old; @@ -1384,8 +1404,23 @@ sfxge_tx_qinit(struct sfxge_softc *sc, u goto fail3; #ifdef SFXGE_HAVE_MQ + if (sfxge_tx_dpl_get_max <= 0) { + log(LOG_ERR, "%s=%d must be greater than 0", + SFXGE_PARAM_TX_DPL_GET_MAX, sfxge_tx_dpl_get_max); + rc = EINVAL; + goto fail_tx_dpl_get_max; + } + if (sfxge_tx_dpl_put_max < 0) { + log(LOG_ERR, "%s=%d must be greater or equal to 0", + SFXGE_PARAM_TX_DPL_PUT_MAX, sfxge_tx_dpl_put_max); + rc = EINVAL; + goto fail_tx_dpl_put_max; + } + /* Initialize the deferred packet list. */ stdp = &txq->dpl; + stdp->std_put_max = sfxge_tx_dpl_put_max; + stdp->std_get_max = sfxge_tx_dpl_get_max; stdp->std_getp = &stdp->std_get; mtx_init(&txq->lock, "txq", NULL, MTX_DEF); @@ -1403,6 +1438,8 @@ sfxge_tx_qinit(struct sfxge_softc *sc, u return (0); +fail_tx_dpl_put_max: +fail_tx_dpl_get_max: fail3: fail_txq_node: free(txq->pend_desc, M_SFXGE); Modified: head/sys/dev/sfxge/sfxge_tx.h ============================================================================== --- head/sys/dev/sfxge/sfxge_tx.h Tue Sep 30 20:43:21 2014 (r272330) +++ head/sys/dev/sfxge/sfxge_tx.h Tue Sep 30 20:57:25 2014 (r272331) @@ -75,13 +75,17 @@ struct sfxge_tx_mapping { enum sfxge_tx_buf_flags flags; }; -#define SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT 64 +#define SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT 1024 #define SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT 64 /* * Deferred packet list. */ struct sfxge_tx_dpl { + unsigned int std_get_max; /* Maximum number of packets + * in get list */ + unsigned int std_put_max; /* Maximum number of packets + * in put list */ uintptr_t std_put; /* Head of put list. */ struct mbuf *std_get; /* Head of get list. */ struct mbuf **std_getp; /* Tail of get list. */