Date: Thu, 5 Jan 2017 05:51:00 +0000 (UTC) From: Sepherosa Ziehau <sephe@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r311368 - in stable/11/sys/dev/hyperv: netvsc vmbus Message-ID: <201701050551.v055p00S008287@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: sephe Date: Thu Jan 5 05:51:00 2017 New Revision: 311368 URL: https://svnweb.freebsd.org/changeset/base/311368 Log: MFC 309310,309311,309316,309318 309310 hyperv/hn: Nuke the unused TX taskqueue CPU binding tunable. It was an experimental tunable, and is now deemed to be road blocker for further changes. Time to retire it. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8654 309311 hyperv/hn: Allow multiple TX taskqueues. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8655 309316 hyperv/vmbus: Add DEVMETHOD to map cpu to event taskq. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8658 309318 hyperv/hn: Allow TX to share event taskqueues. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8659 Modified: stable/11/sys/dev/hyperv/netvsc/if_hn.c stable/11/sys/dev/hyperv/netvsc/if_hnvar.h stable/11/sys/dev/hyperv/vmbus/vmbus.c stable/11/sys/dev/hyperv/vmbus/vmbus_if.m Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/hyperv/netvsc/if_hn.c ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/if_hn.c Thu Jan 5 05:46:06 2017 (r311367) +++ stable/11/sys/dev/hyperv/netvsc/if_hn.c Thu Jan 5 05:51:00 2017 (r311368) @@ -169,6 +169,8 @@ do { \ #define HN_PKTSIZE(m, align) \ roundup2((m)->m_pkthdr.len + HN_RNDIS_PKT_LEN, (align)) +#define HN_RING_IDX2CPU(sc, idx) (((sc)->hn_cpu + (idx)) % mp_ncpus) + struct hn_txdesc { #ifndef HN_USE_TXDESC_BUFRING SLIST_ENTRY(hn_txdesc) link; @@ -411,10 +413,18 @@ SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_c #endif #endif -/* Use shared TX taskqueue */ -static int hn_share_tx_taskq = 0; -SYSCTL_INT(_hw_hn, OID_AUTO, share_tx_taskq, CTLFLAG_RDTUN, - &hn_share_tx_taskq, 0, "Enable shared TX taskqueue"); +static int hn_tx_taskq_cnt = 1; +SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_cnt, CTLFLAG_RDTUN, + &hn_tx_taskq_cnt, 0, "# of TX taskqueues"); + +#define HN_TX_TASKQ_M_INDEP 0 +#define HN_TX_TASKQ_M_GLOBAL 1 +#define HN_TX_TASKQ_M_EVTTQ 2 + +static int hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; +SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_mode, CTLFLAG_RDTUN, + &hn_tx_taskq_mode, 0, "TX taskqueue modes: " + "0 - independent, 1 - share global tx taskqs, 2 - share event taskqs"); #ifndef HN_USE_TXDESC_BUFRING static int hn_use_txdesc_bufring = 0; @@ -424,11 +434,6 @@ static int hn_use_txdesc_bufring = 1; SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD, &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors"); -/* Bind TX taskqueue to the target CPU */ -static int hn_bind_tx_taskq = -1; -SYSCTL_INT(_hw_hn, OID_AUTO, bind_tx_taskq, CTLFLAG_RDTUN, - &hn_bind_tx_taskq, 0, "Bind TX taskqueue to the specified cpu"); - #ifdef HN_IFSTART_SUPPORT /* Use ifnet.if_start instead of ifnet.if_transmit */ static int hn_use_if_start = 0; @@ -470,7 +475,7 @@ SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_pkts &hn_tx_agg_pkts, 0, "Packet transmission aggregation packet limit"); static u_int hn_cpu_index; /* next CPU for channel */ -static struct taskqueue *hn_tx_taskq; /* shared TX taskqueue */ +static struct taskqueue **hn_tx_taskque;/* shared TX taskqueues */ static const uint8_t hn_rss_key_default[NDIS_HASH_KEYSIZE_TOEPLITZ] = { @@ -903,25 +908,21 @@ hn_attach(device_t dev) /* * Setup taskqueue for transmission. */ - if (hn_tx_taskq == NULL) { - sc->hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK, - taskqueue_thread_enqueue, &sc->hn_tx_taskq); - if (hn_bind_tx_taskq >= 0) { - int cpu = hn_bind_tx_taskq; - cpuset_t cpu_set; - - if (cpu > mp_ncpus - 1) - cpu = mp_ncpus - 1; - CPU_SETOF(cpu, &cpu_set); - taskqueue_start_threads_cpuset(&sc->hn_tx_taskq, 1, - PI_NET, &cpu_set, "%s tx", - device_get_nameunit(dev)); - } else { - taskqueue_start_threads(&sc->hn_tx_taskq, 1, PI_NET, - "%s tx", device_get_nameunit(dev)); + if (hn_tx_taskq_mode == HN_TX_TASKQ_M_INDEP) { + int i; + + sc->hn_tx_taskqs = + malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), + M_DEVBUF, M_WAITOK); + for (i = 0; i < hn_tx_taskq_cnt; ++i) { + sc->hn_tx_taskqs[i] = taskqueue_create("hn_tx", + M_WAITOK, taskqueue_thread_enqueue, + &sc->hn_tx_taskqs[i]); + taskqueue_start_threads(&sc->hn_tx_taskqs[i], 1, PI_NET, + "%s tx%d", device_get_nameunit(dev), i); } - } else { - sc->hn_tx_taskq = hn_tx_taskq; + } else if (hn_tx_taskq_mode == HN_TX_TASKQ_M_GLOBAL) { + sc->hn_tx_taskqs = hn_tx_taskque; } /* @@ -1221,8 +1222,13 @@ hn_detach(device_t dev) hn_destroy_rx_data(sc); hn_destroy_tx_data(sc); - if (sc->hn_tx_taskq != hn_tx_taskq) - taskqueue_free(sc->hn_tx_taskq); + if (sc->hn_tx_taskqs != NULL && sc->hn_tx_taskqs != hn_tx_taskque) { + int i; + + for (i = 0; i < hn_tx_taskq_cnt; ++i) + taskqueue_free(sc->hn_tx_taskqs[i]); + free(sc->hn_tx_taskqs, M_DEVBUF); + } taskqueue_free(sc->hn_mgmt_taskq0); if (sc->hn_xact != NULL) { @@ -3312,7 +3318,12 @@ hn_tx_ring_create(struct hn_softc *sc, i M_WAITOK, &txr->hn_tx_lock); #endif - txr->hn_tx_taskq = sc->hn_tx_taskq; + if (hn_tx_taskq_mode == HN_TX_TASKQ_M_EVTTQ) { + txr->hn_tx_taskq = VMBUS_GET_EVENT_TASKQ( + device_get_parent(dev), dev, HN_RING_IDX2CPU(sc, id)); + } else { + txr->hn_tx_taskq = sc->hn_tx_taskqs[id % hn_tx_taskq_cnt]; + } #ifdef HN_IFSTART_SUPPORT if (hn_use_if_start) { @@ -4205,7 +4216,7 @@ hn_chan_attach(struct hn_softc *sc, stru } /* Bind this channel to a proper CPU. */ - vmbus_chan_cpu_set(chan, (sc->hn_cpu + idx) % mp_ncpus); + vmbus_chan_cpu_set(chan, HN_RING_IDX2CPU(sc, idx)); /* * Open this channel @@ -5351,26 +5362,42 @@ hn_chan_callback(struct vmbus_channel *c static void hn_tx_taskq_create(void *arg __unused) { + int i; + + /* + * Fix the # of TX taskqueues. + */ + if (hn_tx_taskq_cnt <= 0) + hn_tx_taskq_cnt = 1; + else if (hn_tx_taskq_cnt > mp_ncpus) + hn_tx_taskq_cnt = mp_ncpus; + + /* + * Fix the TX taskqueue mode. + */ + switch (hn_tx_taskq_mode) { + case HN_TX_TASKQ_M_INDEP: + case HN_TX_TASKQ_M_GLOBAL: + case HN_TX_TASKQ_M_EVTTQ: + break; + default: + hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; + break; + } if (vm_guest != VM_GUEST_HV) return; - if (!hn_share_tx_taskq) + if (hn_tx_taskq_mode != HN_TX_TASKQ_M_GLOBAL) return; - hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK, - taskqueue_thread_enqueue, &hn_tx_taskq); - if (hn_bind_tx_taskq >= 0) { - int cpu = hn_bind_tx_taskq; - cpuset_t cpu_set; - - if (cpu > mp_ncpus - 1) - cpu = mp_ncpus - 1; - CPU_SETOF(cpu, &cpu_set); - taskqueue_start_threads_cpuset(&hn_tx_taskq, 1, PI_NET, - &cpu_set, "hn tx"); - } else { - taskqueue_start_threads(&hn_tx_taskq, 1, PI_NET, "hn tx"); + hn_tx_taskque = malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), + M_DEVBUF, M_WAITOK); + for (i = 0; i < hn_tx_taskq_cnt; ++i) { + hn_tx_taskque[i] = taskqueue_create("hn_tx", M_WAITOK, + taskqueue_thread_enqueue, &hn_tx_taskque[i]); + taskqueue_start_threads(&hn_tx_taskque[i], 1, PI_NET, + "hn tx%d", i); } } SYSINIT(hn_txtq_create, SI_SUB_DRIVERS, SI_ORDER_SECOND, @@ -5380,8 +5407,13 @@ static void hn_tx_taskq_destroy(void *arg __unused) { - if (hn_tx_taskq != NULL) - taskqueue_free(hn_tx_taskq); + if (hn_tx_taskque != NULL) { + int i; + + for (i = 0; i < hn_tx_taskq_cnt; ++i) + taskqueue_free(hn_tx_taskque[i]); + free(hn_tx_taskque, M_DEVBUF); + } } SYSUNINIT(hn_txtq_destroy, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_tx_taskq_destroy, NULL); Modified: stable/11/sys/dev/hyperv/netvsc/if_hnvar.h ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/if_hnvar.h Thu Jan 5 05:46:06 2017 (r311367) +++ stable/11/sys/dev/hyperv/netvsc/if_hnvar.h Thu Jan 5 05:51:00 2017 (r311368) @@ -192,7 +192,7 @@ struct hn_softc { int hn_chim_szmax; int hn_cpu; - struct taskqueue *hn_tx_taskq; + struct taskqueue **hn_tx_taskqs; struct sysctl_oid *hn_tx_sysctl_tree; struct sysctl_oid *hn_rx_sysctl_tree; struct vmbus_xact_ctx *hn_xact; Modified: stable/11/sys/dev/hyperv/vmbus/vmbus.c ============================================================================== --- stable/11/sys/dev/hyperv/vmbus/vmbus.c Thu Jan 5 05:46:06 2017 (r311367) +++ stable/11/sys/dev/hyperv/vmbus/vmbus.c Thu Jan 5 05:51:00 2017 (r311368) @@ -97,6 +97,8 @@ static int vmbus_probe_guid_method(dev const struct hyperv_guid *); static uint32_t vmbus_get_vcpu_id_method(device_t bus, device_t dev, int cpu); +static struct taskqueue *vmbus_get_eventtq_method(device_t, device_t, + int); #ifdef EARLY_AP_STARTUP static void vmbus_intrhook(void *); #endif @@ -175,6 +177,7 @@ static device_method_t vmbus_methods[] = DEVMETHOD(vmbus_get_version, vmbus_get_version_method), DEVMETHOD(vmbus_probe_guid, vmbus_probe_guid_method), DEVMETHOD(vmbus_get_vcpu_id, vmbus_get_vcpu_id_method), + DEVMETHOD(vmbus_get_event_taskq, vmbus_get_eventtq_method), DEVMETHOD_END }; @@ -1129,6 +1132,15 @@ vmbus_get_vcpu_id_method(device_t bus, d return (VMBUS_PCPU_GET(sc, vcpuid, cpu)); } +static struct taskqueue * +vmbus_get_eventtq_method(device_t bus, device_t dev __unused, int cpu) +{ + const struct vmbus_softc *sc = device_get_softc(bus); + + KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu%d", cpu)); + return (VMBUS_PCPU_GET(sc, event_tq, cpu)); +} + #ifdef NEW_PCIB #define VTPM_BASE_ADDR 0xfed40000 #define FOUR_GB (1ULL << 32) Modified: stable/11/sys/dev/hyperv/vmbus/vmbus_if.m ============================================================================== --- stable/11/sys/dev/hyperv/vmbus/vmbus_if.m Thu Jan 5 05:46:06 2017 (r311367) +++ stable/11/sys/dev/hyperv/vmbus/vmbus_if.m Thu Jan 5 05:51:00 2017 (r311368) @@ -33,6 +33,7 @@ INTERFACE vmbus; HEADER { struct hyperv_guid; + struct taskqueue; }; METHOD uint32_t get_version { @@ -51,3 +52,9 @@ METHOD uint32_t get_vcpu_id { device_t dev; int cpu; }; + +METHOD struct taskqueue * get_event_taskq { + device_t bus; + device_t dev; + int cpu; +};
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201701050551.v055p00S008287>