Date: Wed, 26 Nov 2014 20:19:36 +0000 (UTC) From: Alfred Perlstein <alfred@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r275136 - in head/sys: dev/e1000 dev/ixgbe kern sys Message-ID: <201411262019.sAQKJaw4043557@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alfred Date: Wed Nov 26 20:19:36 2014 New Revision: 275136 URL: https://svnweb.freebsd.org/changeset/base/275136 Log: Make igb and ixgbe check tunables at probe time. This allows one to make a kernel module to tune the number of queues before the driver loads. This is needed so that a module at SI_SUB_CPU can set tunables for these drivers to take. Otherwise getenv is called too early by the TUNABLE macros. Reviewed by: smh Phabric: https://reviews.freebsd.org/D1149 Modified: head/sys/dev/e1000/if_igb.c head/sys/dev/ixgbe/ixgbe.c head/sys/kern/subr_bus.c head/sys/sys/bus.h Modified: head/sys/dev/e1000/if_igb.c ============================================================================== --- head/sys/dev/e1000/if_igb.c Wed Nov 26 18:03:25 2014 (r275135) +++ head/sys/dev/e1000/if_igb.c Wed Nov 26 20:19:36 2014 (r275136) @@ -188,6 +188,7 @@ static char *igb_strings[] = { /********************************************************************* * Function prototypes *********************************************************************/ +static int igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS); static int igb_probe(device_t); static int igb_attach(device_t); static int igb_detach(device_t); @@ -493,6 +494,11 @@ igb_attach(device_t dev) OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, igb_sysctl_nvm_info, "I", "NVM Information"); + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "num_queues", CTLTYPE_INT | CTLFLAG_RD, + adapter, 0, igb_per_unit_num_queues, "I", "Number of Queues"); + igb_set_sysctl_value(adapter, "enable_aim", "Interrupt Moderation", &adapter->enable_aim, igb_enable_aim); @@ -2831,6 +2837,7 @@ igb_setup_msix(struct adapter *adapter) { device_t dev = adapter->dev; int bar, want, queues, msgs, maxqueues; + int n_queues; /* tuneable override */ if (igb_enable_msix == 0) @@ -2858,8 +2865,18 @@ igb_setup_msix(struct adapter *adapter) goto msi; } - /* Figure out a reasonable auto config value */ - queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; + n_queues = 0; + /* try more specific tunable, then global, then finally default to boot time tunable if set. */ + if (device_getenv_int(dev, "num_queues", &n_queues) != 0) { + device_printf(dev, "using specific tunable num_queues=%d", n_queues); + } else if (TUNABLE_INT_FETCH("hw.igb.num_queues", &n_queues) != 0) { + if (igb_num_queues != n_queues) { + device_printf(dev, "using global tunable hw.igb.num_queues=%d", n_queues); + igb_num_queues = n_queues; + } + } else { + n_queues = igb_num_queues; + } #ifdef RSS /* If we're doing RSS, clamp at the number of RSS buckets */ @@ -2867,10 +2884,12 @@ igb_setup_msix(struct adapter *adapter) queues = rss_getnumbuckets(); #endif - - /* Manual override */ - if (igb_num_queues != 0) - queues = igb_num_queues; + if (n_queues != 0) { + queues = n_queues; + } else { + /* Figure out a reasonable auto config value */ + queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; + } /* Sanity check based on HW */ switch (adapter->hw.mac.type) { @@ -2893,12 +2912,17 @@ igb_setup_msix(struct adapter *adapter) maxqueues = 1; break; } - if (queues > maxqueues) + if (queues > maxqueues) { + device_printf(adapter->dev, "requested %d queues, but max for this adapter is %d\n", + queues, maxqueues); queues = maxqueues; - - /* Manual override */ - if (igb_num_queues != 0) - queues = igb_num_queues; + } else if (queues == 0) { + queues = 1; + } else if (queues < 0) { + device_printf(adapter->dev, "requested %d queues, but min for this adapter is %d\n", + queues, 1); + queues = 1; + } /* ** One vector (RX/TX pair) per queue @@ -6384,3 +6408,14 @@ igb_sysctl_eee(SYSCTL_HANDLER_ARGS) IGB_CORE_UNLOCK(adapter); return (0); } + +static int +igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS) +{ + struct adapter *adapter; + + adapter = (struct adapter *) arg1; + + return sysctl_handle_int(oidp, &adapter->num_queues, 0, req); +} + Modified: head/sys/dev/ixgbe/ixgbe.c ============================================================================== --- head/sys/dev/ixgbe/ixgbe.c Wed Nov 26 18:03:25 2014 (r275135) +++ head/sys/dev/ixgbe/ixgbe.c Wed Nov 26 20:19:36 2014 (r275136) @@ -103,6 +103,7 @@ static char *ixgbe_strings[] = { /********************************************************************* * Function prototypes *********************************************************************/ +static int ixgbe_per_unit_num_queues(SYSCTL_HANDLER_ARGS); static int ixgbe_probe(device_t); static int ixgbe_attach(device_t); static int ixgbe_detach(device_t); @@ -475,6 +476,11 @@ ixgbe_attach(device_t dev) SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "num_queues", CTLTYPE_INT | CTLFLAG_RD, + adapter, 0, ixgbe_per_unit_num_queues, "I", "Number of Queues"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_set_thermal_test, "I", "Thermal Test"); @@ -2516,6 +2522,7 @@ ixgbe_setup_msix(struct adapter *adapter { device_t dev = adapter->dev; int rid, want, queues, msgs; + int n_queues; /* Override by tuneable */ if (ixgbe_enable_msix == 0) @@ -2548,15 +2555,29 @@ ixgbe_setup_msix(struct adapter *adapter queues = rss_getnumbuckets(); #endif - if (ixgbe_num_queues != 0) - queues = ixgbe_num_queues; + /* try more specific tunable, then global, then finally default to boot time tunable if set. */ + if (device_getenv_int(dev, "num_queues", &n_queues) != 0) { + device_printf(dev, "using specific tunable numqueues=%d", n_queues); + } else if (TUNABLE_INT_FETCH("hw.ix.num_queues", &n_queues) != 0) { + if (ixgbe_num_queues != n_queues) { + device_printf(dev, "using global tunable num_queues=%d", n_queues); + ixgbe_num_queues = n_queues; + } + } else { + n_queues = ixgbe_num_queues; + } + + if (n_queues < 0) { + device_printf(dev, "tunable < 0, resetting to default"); + n_queues = 0; + } + + if (n_queues != 0) + queues = n_queues; /* Set max queues to 8 when autoconfiguring */ else if ((ixgbe_num_queues == 0) && (queues > 8)) queues = 8; - /* reflect correct sysctl value */ - ixgbe_num_queues = queues; - /* ** Want one vector (RX/TX pair) per queue ** plus an additional for Link. @@ -5943,6 +5964,16 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) return error; } +static int +ixgbe_per_unit_num_queues(SYSCTL_HANDLER_ARGS) +{ + struct adapter *adapter; + + adapter = (struct adapter *) arg1; + + return sysctl_handle_int(oidp, &adapter->num_queues, 0, req); +} + /* ** Control link advertise speed: ** 1 - advertise only 1G Modified: head/sys/kern/subr_bus.c ============================================================================== --- head/sys/kern/subr_bus.c Wed Nov 26 18:03:25 2014 (r275135) +++ head/sys/kern/subr_bus.c Wed Nov 26 20:19:36 2014 (r275136) @@ -56,6 +56,8 @@ __FBSDID("$FreeBSD$"); #include <sys/interrupt.h> #include <sys/cpuset.h> +#include <sys/syslog.h> + #include <net/vnet.h> #include <machine/cpu.h> @@ -5031,3 +5033,18 @@ bus_free_resource(device_t dev, int type return (0); return (bus_release_resource(dev, type, rman_get_rid(r), r)); } + +int +device_getenv_int(device_t dev, const char *knob, int *iptr) +{ + char env[128]; + int sz; + + sz = snprintf(env, sizeof(env), "hw.%s.%d.%s", device_get_name(dev), device_get_unit(dev), knob); + if (sz >= sizeof(env)) { + /* XXX: log? return error? bump sysctl error? */ + log(LOG_ERR, "device_getenv_int: knob too long: '%s'", knob); + return 0; + } + return (getenv_int(env, iptr)); +} Modified: head/sys/sys/bus.h ============================================================================== --- head/sys/sys/bus.h Wed Nov 26 18:03:25 2014 (r275135) +++ head/sys/sys/bus.h Wed Nov 26 20:19:36 2014 (r275136) @@ -449,6 +449,7 @@ const char *device_get_nameunit(device_t void *device_get_softc(device_t dev); device_state_t device_get_state(device_t dev); int device_get_unit(device_t dev); +int device_getenv_int(device_t dev, const char *knob, int *iptr); struct sysctl_ctx_list *device_get_sysctl_ctx(device_t dev); struct sysctl_oid *device_get_sysctl_tree(device_t dev); int device_is_alive(device_t dev); /* did probe succeed? */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201411262019.sAQKJaw4043557>