Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 29 Dec 2016 09:02:49 +0000 (UTC)
From:      Sepherosa Ziehau <sephe@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org
Subject:   svn commit: r310768 - in stable/10/sys/dev/hyperv: netvsc vmbus
Message-ID:  <201612290902.uBT92nTM004737@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: sephe
Date: Thu Dec 29 09:02:49 2016
New Revision: 310768
URL: https://svnweb.freebsd.org/changeset/base/310768

Log:
  MFC 309310,309311,309316,309318
  
  309310
      hyperv/hn: Nuke the unused TX taskqueue CPU binding tunable.
  
      It was an experimental tunable, and is now deemed to be road blocker
      for further changes.  Time to retire it.
  
      Sponsored by:   Microsoft
      Differential Revision:  https://reviews.freebsd.org/D8654
  
  309311
      hyperv/hn: Allow multiple TX taskqueues.
  
      Sponsored by:   Microsoft
      Differential Revision:  https://reviews.freebsd.org/D8655
  
  309316
      hyperv/vmbus: Add DEVMETHOD to map cpu to event taskq.
  
      Sponsored by:   Microsoft
      Differential Revision:  https://reviews.freebsd.org/D8658
  
  309318
      hyperv/hn: Allow TX to share event taskqueues.
  
      Sponsored by:   Microsoft
      Differential Revision:  https://reviews.freebsd.org/D8659

Modified:
  stable/10/sys/dev/hyperv/netvsc/if_hn.c
  stable/10/sys/dev/hyperv/netvsc/if_hnvar.h
  stable/10/sys/dev/hyperv/vmbus/vmbus.c
  stable/10/sys/dev/hyperv/vmbus/vmbus_if.m
Directory Properties:
  stable/10/   (props changed)

Modified: stable/10/sys/dev/hyperv/netvsc/if_hn.c
==============================================================================
--- stable/10/sys/dev/hyperv/netvsc/if_hn.c	Thu Dec 29 08:41:25 2016	(r310767)
+++ stable/10/sys/dev/hyperv/netvsc/if_hn.c	Thu Dec 29 09:02:49 2016	(r310768)
@@ -172,6 +172,8 @@ do {							\
 #define HN_PKTSIZE(m, align)		\
 	roundup2((m)->m_pkthdr.len + HN_RNDIS_PKT_LEN, (align))
 
+#define HN_RING_IDX2CPU(sc, idx)	(((sc)->hn_cpu + (idx)) % mp_ncpus)
+
 struct hn_txdesc {
 #ifndef HN_USE_TXDESC_BUFRING
 	SLIST_ENTRY(hn_txdesc)		link;
@@ -414,10 +416,18 @@ SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_c
 #endif
 #endif
 
-/* Use shared TX taskqueue */
-static int			hn_share_tx_taskq = 0;
-SYSCTL_INT(_hw_hn, OID_AUTO, share_tx_taskq, CTLFLAG_RDTUN,
-    &hn_share_tx_taskq, 0, "Enable shared TX taskqueue");
+static int			hn_tx_taskq_cnt = 1;
+SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_cnt, CTLFLAG_RDTUN,
+    &hn_tx_taskq_cnt, 0, "# of TX taskqueues");
+
+#define HN_TX_TASKQ_M_INDEP	0
+#define HN_TX_TASKQ_M_GLOBAL	1
+#define HN_TX_TASKQ_M_EVTTQ	2
+
+static int			hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP;
+SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_mode, CTLFLAG_RDTUN,
+    &hn_tx_taskq_mode, 0, "TX taskqueue modes: "
+    "0 - independent, 1 - share global tx taskqs, 2 - share event taskqs");
 
 #ifndef HN_USE_TXDESC_BUFRING
 static int			hn_use_txdesc_bufring = 0;
@@ -427,11 +437,6 @@ static int			hn_use_txdesc_bufring = 1;
 SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD,
     &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors");
 
-/* Bind TX taskqueue to the target CPU */
-static int			hn_bind_tx_taskq = -1;
-SYSCTL_INT(_hw_hn, OID_AUTO, bind_tx_taskq, CTLFLAG_RDTUN,
-    &hn_bind_tx_taskq, 0, "Bind TX taskqueue to the specified cpu");
-
 #ifdef HN_IFSTART_SUPPORT
 /* Use ifnet.if_start instead of ifnet.if_transmit */
 static int			hn_use_if_start = 0;
@@ -473,7 +478,7 @@ SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_pkts
     &hn_tx_agg_pkts, 0, "Packet transmission aggregation packet limit");
 
 static u_int			hn_cpu_index;	/* next CPU for channel */
-static struct taskqueue		*hn_tx_taskq;	/* shared TX taskqueue */
+static struct taskqueue		**hn_tx_taskque;/* shared TX taskqueues */
 
 static const uint8_t
 hn_rss_key_default[NDIS_HASH_KEYSIZE_TOEPLITZ] = {
@@ -883,19 +888,6 @@ hn_probe(device_t dev)
 	return ENXIO;
 }
 
-static void
-hn_cpuset_setthread_task(void *xmask, int pending __unused)
-{
-	cpuset_t *mask = xmask;
-	int error;
-
-	error = cpuset_setthread(curthread->td_tid, mask);
-	if (error) {
-		panic("curthread=%ju: can't pin; error=%d",
-		    (uintmax_t)curthread->td_tid, error);
-	}
-}
-
 static int
 hn_attach(device_t dev)
 {
@@ -919,26 +911,21 @@ hn_attach(device_t dev)
 	/*
 	 * Setup taskqueue for transmission.
 	 */
-	if (hn_tx_taskq == NULL) {
-		sc->hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK,
-		    taskqueue_thread_enqueue, &sc->hn_tx_taskq);
-		taskqueue_start_threads(&sc->hn_tx_taskq, 1, PI_NET, "%s tx",
-		    device_get_nameunit(dev));
-		if (hn_bind_tx_taskq >= 0) {
-			int cpu = hn_bind_tx_taskq;
-			struct task cpuset_task;
-			cpuset_t cpu_set;
-
-			if (cpu > mp_ncpus - 1)
-				cpu = mp_ncpus - 1;
-			CPU_SETOF(cpu, &cpu_set);
-			TASK_INIT(&cpuset_task, 0, hn_cpuset_setthread_task,
-			    &cpu_set);
-			taskqueue_enqueue(sc->hn_tx_taskq, &cpuset_task);
-			taskqueue_drain(sc->hn_tx_taskq, &cpuset_task);
+	if (hn_tx_taskq_mode == HN_TX_TASKQ_M_INDEP) {
+		int i;
+
+		sc->hn_tx_taskqs =
+		    malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *),
+		    M_DEVBUF, M_WAITOK);
+		for (i = 0; i < hn_tx_taskq_cnt; ++i) {
+			sc->hn_tx_taskqs[i] = taskqueue_create("hn_tx",
+			    M_WAITOK, taskqueue_thread_enqueue,
+			    &sc->hn_tx_taskqs[i]);
+			taskqueue_start_threads(&sc->hn_tx_taskqs[i], 1, PI_NET,
+			    "%s tx%d", device_get_nameunit(dev), i);
 		}
-	} else {
-		sc->hn_tx_taskq = hn_tx_taskq;
+	} else if (hn_tx_taskq_mode == HN_TX_TASKQ_M_GLOBAL) {
+		sc->hn_tx_taskqs = hn_tx_taskque;
 	}
 
 	/*
@@ -1243,8 +1230,13 @@ hn_detach(device_t dev)
 	hn_destroy_rx_data(sc);
 	hn_destroy_tx_data(sc);
 
-	if (sc->hn_tx_taskq != hn_tx_taskq)
-		taskqueue_free(sc->hn_tx_taskq);
+	if (sc->hn_tx_taskqs != NULL && sc->hn_tx_taskqs != hn_tx_taskque) {
+		int i;
+
+		for (i = 0; i < hn_tx_taskq_cnt; ++i)
+			taskqueue_free(sc->hn_tx_taskqs[i]);
+		free(sc->hn_tx_taskqs, M_DEVBUF);
+	}
 	taskqueue_free(sc->hn_mgmt_taskq0);
 
 	if (sc->hn_xact != NULL) {
@@ -3338,7 +3330,12 @@ hn_tx_ring_create(struct hn_softc *sc, i
 	    M_WAITOK, &txr->hn_tx_lock);
 #endif
 
-	txr->hn_tx_taskq = sc->hn_tx_taskq;
+	if (hn_tx_taskq_mode == HN_TX_TASKQ_M_EVTTQ) {
+		txr->hn_tx_taskq = VMBUS_GET_EVENT_TASKQ(
+		    device_get_parent(dev), dev, HN_RING_IDX2CPU(sc, id));
+	} else {
+		txr->hn_tx_taskq = sc->hn_tx_taskqs[id % hn_tx_taskq_cnt];
+	}
 
 #ifdef HN_IFSTART_SUPPORT
 	if (hn_use_if_start) {
@@ -4231,7 +4228,7 @@ hn_chan_attach(struct hn_softc *sc, stru
 	}
 
 	/* Bind this channel to a proper CPU. */
-	vmbus_chan_cpu_set(chan, (sc->hn_cpu + idx) % mp_ncpus);
+	vmbus_chan_cpu_set(chan, HN_RING_IDX2CPU(sc, idx));
 
 	/*
 	 * Open this channel
@@ -5377,27 +5374,42 @@ hn_chan_callback(struct vmbus_channel *c
 static void
 hn_tx_taskq_create(void *arg __unused)
 {
+	int i;
+
+	/*
+	 * Fix the # of TX taskqueues.
+	 */
+	if (hn_tx_taskq_cnt <= 0)
+		hn_tx_taskq_cnt = 1;
+	else if (hn_tx_taskq_cnt > mp_ncpus)
+		hn_tx_taskq_cnt = mp_ncpus;
+
+	/*
+	 * Fix the TX taskqueue mode.
+	 */
+	switch (hn_tx_taskq_mode) {
+	case HN_TX_TASKQ_M_INDEP:
+	case HN_TX_TASKQ_M_GLOBAL:
+	case HN_TX_TASKQ_M_EVTTQ:
+		break;
+	default:
+		hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP;
+		break;
+	}
 
 	if (vm_guest != VM_GUEST_HV)
 		return;
 
-	if (!hn_share_tx_taskq)
+	if (hn_tx_taskq_mode != HN_TX_TASKQ_M_GLOBAL)
 		return;
 
-	hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK,
-	    taskqueue_thread_enqueue, &hn_tx_taskq);
-	taskqueue_start_threads(&hn_tx_taskq, 1, PI_NET, "hn tx");
-	if (hn_bind_tx_taskq >= 0) {
-		int cpu = hn_bind_tx_taskq;
-		struct task cpuset_task;
-		cpuset_t cpu_set;
-
-		if (cpu > mp_ncpus - 1)
-			cpu = mp_ncpus - 1;
-		CPU_SETOF(cpu, &cpu_set);
-		TASK_INIT(&cpuset_task, 0, hn_cpuset_setthread_task, &cpu_set);
-		taskqueue_enqueue(hn_tx_taskq, &cpuset_task);
-		taskqueue_drain(hn_tx_taskq, &cpuset_task);
+	hn_tx_taskque = malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *),
+	    M_DEVBUF, M_WAITOK);
+	for (i = 0; i < hn_tx_taskq_cnt; ++i) {
+		hn_tx_taskque[i] = taskqueue_create("hn_tx", M_WAITOK,
+		    taskqueue_thread_enqueue, &hn_tx_taskque[i]);
+		taskqueue_start_threads(&hn_tx_taskque[i], 1, PI_NET,
+		    "hn tx%d", i);
 	}
 }
 SYSINIT(hn_txtq_create, SI_SUB_DRIVERS, SI_ORDER_SECOND,
@@ -5407,8 +5419,13 @@ static void
 hn_tx_taskq_destroy(void *arg __unused)
 {
 
-	if (hn_tx_taskq != NULL)
-		taskqueue_free(hn_tx_taskq);
+	if (hn_tx_taskque != NULL) {
+		int i;
+
+		for (i = 0; i < hn_tx_taskq_cnt; ++i)
+			taskqueue_free(hn_tx_taskque[i]);
+		free(hn_tx_taskque, M_DEVBUF);
+	}
 }
 SYSUNINIT(hn_txtq_destroy, SI_SUB_DRIVERS, SI_ORDER_SECOND,
     hn_tx_taskq_destroy, NULL);

Modified: stable/10/sys/dev/hyperv/netvsc/if_hnvar.h
==============================================================================
--- stable/10/sys/dev/hyperv/netvsc/if_hnvar.h	Thu Dec 29 08:41:25 2016	(r310767)
+++ stable/10/sys/dev/hyperv/netvsc/if_hnvar.h	Thu Dec 29 09:02:49 2016	(r310768)
@@ -193,7 +193,7 @@ struct hn_softc {
 	int		hn_chim_szmax;
 
 	int		hn_cpu;
-	struct taskqueue *hn_tx_taskq;
+	struct taskqueue **hn_tx_taskqs;
 	struct sysctl_oid *hn_tx_sysctl_tree;
 	struct sysctl_oid *hn_rx_sysctl_tree;
 	struct vmbus_xact_ctx *hn_xact;

Modified: stable/10/sys/dev/hyperv/vmbus/vmbus.c
==============================================================================
--- stable/10/sys/dev/hyperv/vmbus/vmbus.c	Thu Dec 29 08:41:25 2016	(r310767)
+++ stable/10/sys/dev/hyperv/vmbus/vmbus.c	Thu Dec 29 09:02:49 2016	(r310768)
@@ -99,6 +99,8 @@ static int			vmbus_probe_guid_method(dev
 				    const struct hyperv_guid *);
 static uint32_t			vmbus_get_vcpu_id_method(device_t bus,
 				    device_t dev, int cpu);
+static struct taskqueue		*vmbus_get_eventtq_method(device_t, device_t,
+				    int);
 
 static int			vmbus_init(struct vmbus_softc *);
 static int			vmbus_connect(struct vmbus_softc *, uint32_t);
@@ -174,6 +176,7 @@ static device_method_t vmbus_methods[] =
 	DEVMETHOD(vmbus_get_version,		vmbus_get_version_method),
 	DEVMETHOD(vmbus_probe_guid,		vmbus_probe_guid_method),
 	DEVMETHOD(vmbus_get_vcpu_id,		vmbus_get_vcpu_id_method),
+	DEVMETHOD(vmbus_get_event_taskq,	vmbus_get_eventtq_method),
 
 	DEVMETHOD_END
 };
@@ -1208,6 +1211,15 @@ vmbus_get_vcpu_id_method(device_t bus, d
 	return (VMBUS_PCPU_GET(sc, vcpuid, cpu));
 }
 
+static struct taskqueue *
+vmbus_get_eventtq_method(device_t bus, device_t dev __unused, int cpu)
+{
+	const struct vmbus_softc *sc = device_get_softc(bus);
+
+	KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu%d", cpu));
+	return (VMBUS_PCPU_GET(sc, event_tq, cpu));
+}
+
 #ifdef NEW_PCIB
 #define VTPM_BASE_ADDR 0xfed40000
 #define FOUR_GB (1ULL << 32)

Modified: stable/10/sys/dev/hyperv/vmbus/vmbus_if.m
==============================================================================
--- stable/10/sys/dev/hyperv/vmbus/vmbus_if.m	Thu Dec 29 08:41:25 2016	(r310767)
+++ stable/10/sys/dev/hyperv/vmbus/vmbus_if.m	Thu Dec 29 09:02:49 2016	(r310768)
@@ -33,6 +33,7 @@ INTERFACE vmbus;
 
 HEADER {
 	struct hyperv_guid;
+	struct taskqueue;
 };
 
 METHOD uint32_t get_version {
@@ -51,3 +52,9 @@ METHOD uint32_t get_vcpu_id {
 	device_t dev;
 	int cpu;
 };
+
+METHOD struct taskqueue * get_event_taskq {
+	device_t bus;
+	device_t dev;
+	int cpu;
+};



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201612290902.uBT92nTM004737>