Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 17 Mar 2014 05:45:29 +0000 (UTC)
From:      Bryan Venteicher <bryanv@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r263259 - in head: share/man/man4 sys/dev/vmware/vmxnet3 sys/modules/vmware/vmxnet3
Message-ID:  <201403170545.s2H5jTZT076017@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: bryanv
Date: Mon Mar 17 05:45:29 2014
New Revision: 263259
URL: http://svnweb.freebsd.org/changeset/base/263259

Log:
  Add Tx/Rx multiqueue support to vmx(4)
  
  As a prerequisite for multiple queues, the guest must have MSIX enabled.
  Unfortunately, to work around device passthrough bugs, FreeBSD disables
  MSIX when running as a VMWare guest due to the hw.pci.honor_msi_blacklist
  tunable; this tunable must be disabled for multiple queues.
  
  Also included is various minor changes from the projects/vmxnet branch.
  
  MFC after:	1 month

Modified:
  head/share/man/man4/vmx.4
  head/sys/dev/vmware/vmxnet3/if_vmx.c
  head/sys/dev/vmware/vmxnet3/if_vmxreg.h
  head/sys/dev/vmware/vmxnet3/if_vmxvar.h
  head/sys/modules/vmware/vmxnet3/Makefile

Modified: head/share/man/man4/vmx.4
==============================================================================
--- head/share/man/man4/vmx.4	Mon Mar 17 05:03:53 2014	(r263258)
+++ head/share/man/man4/vmx.4	Mon Mar 17 05:45:29 2014	(r263259)
@@ -81,12 +81,35 @@ VMware Fusion 2.0 and newer
 .Pp
 For more information on configuring this device, see
 .Xr ifconfig 8 .
+.Sh MULTIPLE QUEUES
+The
+.Nm
+driver supports multiple transmit and receive queues.
+Multiple queues are only supported by certain VMware products, such as ESXi.
+The number of queues allocated depends on the presence of MSI-X,
+the number of configured CPUs,
+and the tunables listed below.
+.Fx
+does not enable MSI-X support on VMware by default.
+The
+.Va hw.pci.honor_msi_blacklist
+tunable must be disabled to enable MSI-X support.
 .Sh LOADER TUNABLES
 Tunables can be set at the
 .Xr loader 8
 prompt before booting the kernel or stored in
 .Xr loader.conf 5 .
 .Bl -tag -width indent
+.It Va hw.vmx.txnqueue
+.It Va hw.vmx. Ns Ar X Ns Va .txnqueue
+Maximum number of transmit queues allocated by default by the driver.
+The default value is 8.
+The maximum supported by the VMXNET3 virtual NIC is 8.
+.It Va hw.vmx.rxnqueue
+.It Va hw.vmx. Ns Ar X Ns Va .rxnqueue
+Maximum number of receive queues allocated by default by the driver.
+The default value is 8.
+The maximum supported by the VMXNET3 virtual NIC is 16.
 .It Va hw.vmx.txndesc
 .It Va hw.vmx. Ns Ar X Ns Va .txndesc
 .Pp

Modified: head/sys/dev/vmware/vmxnet3/if_vmx.c
==============================================================================
--- head/sys/dev/vmware/vmxnet3/if_vmx.c	Mon Mar 17 05:03:53 2014	(r263258)
+++ head/sys/dev/vmware/vmxnet3/if_vmx.c	Mon Mar 17 05:45:29 2014	(r263259)
@@ -33,6 +33,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
+#include <sys/smp.h>
+#include <sys/taskqueue.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
@@ -69,9 +71,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
-/* Always enable for now - useful for queue hangs. */
-#define VMXNET3_DEBUG_SYSCTL
-
 #ifdef VMXNET3_FAILPOINTS
 #include <sys/fail.h>
 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
@@ -88,6 +87,7 @@ static int	vmxnet3_alloc_resources(struc
 static void	vmxnet3_free_resources(struct vmxnet3_softc *);
 static int	vmxnet3_check_version(struct vmxnet3_softc *);
 static void	vmxnet3_initial_config(struct vmxnet3_softc *);
+static void	vmxnet3_check_multiqueue(struct vmxnet3_softc *);
 
 static int	vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
 static int	vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
@@ -104,6 +104,13 @@ static void	vmxnet3_free_interrupt(struc
 		    struct vmxnet3_interrupt *);
 static void	vmxnet3_free_interrupts(struct vmxnet3_softc *);
 
+#ifndef VMXNET3_LEGACY_TX
+static int	vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
+static void	vmxnet3_start_taskqueue(struct vmxnet3_softc *);
+static void	vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
+static void	vmxnet3_free_taskqueue(struct vmxnet3_softc *);
+#endif
+
 static int	vmxnet3_init_rxq(struct vmxnet3_softc *, int);
 static int	vmxnet3_init_txq(struct vmxnet3_softc *, int);
 static int	vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
@@ -122,6 +129,7 @@ static void	vmxnet3_free_queue_data(stru
 static int	vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
 static void	vmxnet3_init_shared_data(struct vmxnet3_softc *);
 static void	vmxnet3_reinit_interface(struct vmxnet3_softc *);
+static void	vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
 static void	vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
 static int	vmxnet3_alloc_data(struct vmxnet3_softc *);
 static void	vmxnet3_free_data(struct vmxnet3_softc *);
@@ -152,13 +160,24 @@ static int	vmxnet3_reinit(struct vmxnet3
 static void	vmxnet3_init_locked(struct vmxnet3_softc *);
 static void	vmxnet3_init(void *);
 
-static int	vmxnet3_txq_offload_ctx(struct mbuf *, int *, int *, int *);
+static int	vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
+		    int *, int *, int *);
 static int	vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
 		    bus_dmamap_t, bus_dma_segment_t [], int *);
 static void	vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
 static int	vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
+static void	vmxnet3_txq_update_pending(struct vmxnet3_txqueue *);
+#ifdef VMXNET3_LEGACY_TX
 static void	vmxnet3_start_locked(struct ifnet *);
 static void	vmxnet3_start(struct ifnet *);
+#else
+static int	vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
+		    struct mbuf *);
+static int	vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
+static void	vmxnet3_txq_tq_deferred(void *, int);
+#endif
+static void	vmxnet3_txq_start(struct vmxnet3_txqueue *);
+static void	vmxnet3_tx_start_all(struct vmxnet3_softc *);
 
 static void	vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
 		    uint16_t);
@@ -168,7 +187,16 @@ static void	vmxnet3_set_rxfilter(struct 
 static int	vmxnet3_change_mtu(struct vmxnet3_softc *, int);
 static int	vmxnet3_ioctl(struct ifnet *, u_long, caddr_t);
 
+#ifndef VMXNET3_LEGACY_TX
+static void	vmxnet3_qflush(struct ifnet *);
+#endif
+
 static int	vmxnet3_watchdog(struct vmxnet3_txqueue *);
+static void	vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
+static void	vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *,
+		    struct vmxnet3_txq_stats *);
+static void	vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *,
+		    struct vmxnet3_rxq_stats *);
 static void	vmxnet3_tick(void *);
 static void	vmxnet3_link_status(struct vmxnet3_softc *);
 static void	vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
@@ -213,6 +241,12 @@ typedef enum {
 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
 
 /* Tunables. */
+static int vmxnet3_mq_disable = 0;
+TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
+static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
+TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
+static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
+TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
@@ -282,10 +316,18 @@ vmxnet3_attach(device_t dev)
 	if (error)
 		goto fail;
 
+#ifndef VMXNET3_LEGACY_TX
+	error = vmxnet3_alloc_taskqueue(sc);
+	if (error)
+		goto fail;
+#endif
+
 	error = vmxnet3_alloc_interrupts(sc);
 	if (error)
 		goto fail;
 
+	vmxnet3_check_multiqueue(sc);
+
 	error = vmxnet3_alloc_data(sc);
 	if (error)
 		goto fail;
@@ -302,7 +344,9 @@ vmxnet3_attach(device_t dev)
 	}
 
 	vmxnet3_setup_sysctl(sc);
-	vmxnet3_link_status(sc);
+#ifndef VMXNET3_LEGACY_TX
+	vmxnet3_start_taskqueue(sc);
+#endif
 
 fail:
 	if (error)
@@ -321,11 +365,16 @@ vmxnet3_detach(device_t dev)
 	ifp = sc->vmx_ifp;
 
 	if (device_is_attached(dev)) {
-		ether_ifdetach(ifp);
 		VMXNET3_CORE_LOCK(sc);
 		vmxnet3_stop(sc);
 		VMXNET3_CORE_UNLOCK(sc);
+
 		callout_drain(&sc->vmx_tick);
+#ifndef VMXNET3_LEGACY_TX
+		vmxnet3_drain_taskqueue(sc);
+#endif
+
+		ether_ifdetach(ifp);
 	}
 
 	if (sc->vmx_vlan_attach != NULL) {
@@ -337,6 +386,9 @@ vmxnet3_detach(device_t dev)
 		sc->vmx_vlan_detach = NULL;
 	}
 
+#ifndef VMXNET3_LEGACY_TX
+	vmxnet3_free_taskqueue(sc);
+#endif
 	vmxnet3_free_interrupts(sc);
 
 	if (ifp != NULL) {
@@ -463,14 +515,26 @@ vmxnet3_check_version(struct vmxnet3_sof
 static void
 vmxnet3_initial_config(struct vmxnet3_softc *sc)
 {
-	int ndesc;
+	int nqueue, ndesc;
 
-	/*
-	 * BMV Much of the work is already done, but this driver does
-	 * not support multiqueue yet.
-	 */
-	sc->vmx_ntxqueues = VMXNET3_TX_QUEUES;
-	sc->vmx_nrxqueues = VMXNET3_RX_QUEUES;
+	nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
+	if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
+		nqueue = VMXNET3_DEF_TX_QUEUES;
+	if (nqueue > mp_ncpus)
+		nqueue = mp_ncpus;
+	sc->vmx_max_ntxqueues = nqueue;
+
+	nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
+	if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
+		nqueue = VMXNET3_DEF_RX_QUEUES;
+	if (nqueue > mp_ncpus)
+		nqueue = mp_ncpus;
+	sc->vmx_max_nrxqueues = nqueue;
+
+	if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
+		sc->vmx_max_nrxqueues = 1;
+		sc->vmx_max_ntxqueues = 1;
+	}
 
 	ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
 	if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
@@ -488,6 +552,27 @@ vmxnet3_initial_config(struct vmxnet3_so
 	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
 }
 
+static void
+vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
+{
+
+	if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
+		goto out;
+
+	/* BMV: Just use the maximum configured for now. */
+	sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
+	sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
+
+	if (sc->vmx_nrxqueues > 1)
+		sc->vmx_flags |= VMXNET3_FLAG_RSS;
+
+	return;
+
+out:
+	sc->vmx_ntxqueues = 1;
+	sc->vmx_nrxqueues = 1;
+}
+
 static int
 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
 {
@@ -500,7 +585,7 @@ vmxnet3_alloc_msix_interrupts(struct vmx
 		return (1);
 
 	/* Allocate an additional vector for the events interrupt. */
-	required = sc->vmx_nrxqueues + sc->vmx_ntxqueues + 1;
+	required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
 
 	nmsix = pci_msix_count(dev);
 	if (nmsix < required)
@@ -513,6 +598,8 @@ vmxnet3_alloc_msix_interrupts(struct vmx
 	} else
 		pci_release_msi(dev);
 
+	/* BMV TODO Fallback to sharing MSIX vectors if possible. */
+
 	return (1);
 }
 
@@ -586,10 +673,6 @@ vmxnet3_alloc_intr_resources(struct vmxn
 	return (0);
 }
 
-/*
- * NOTE: We only support the simple case of each Rx and Tx queue on its
- * own MSIX vector. This is good enough until we support mulitqueue.
- */
 static int
 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
 {
@@ -651,10 +734,6 @@ vmxnet3_setup_legacy_interrupt(struct vm
 	return (error);
 }
 
-/*
- * XXX BMV Should probably reorganize the attach and just do
- * this in vmxnet3_init_shared_data().
- */
 static void
 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
 {
@@ -784,6 +863,71 @@ vmxnet3_free_interrupts(struct vmxnet3_s
 		pci_release_msi(sc->vmx_dev);
 }
 
+#ifndef VMXNET3_LEGACY_TX
+static int
+vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
+{
+	device_t dev;
+
+	dev = sc->vmx_dev;
+
+	sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
+	    taskqueue_thread_enqueue, &sc->vmx_tq);
+	if (sc->vmx_tq == NULL)
+		return (ENOMEM);
+
+	return (0);
+}
+
+static void
+vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
+{
+	device_t dev;
+	int nthreads, error;
+
+	dev = sc->vmx_dev;
+
+	/*
+	 * The taskqueue is typically not frequently used, so a dedicated
+	 * thread for each queue is unnecessary.
+	 */
+	nthreads = MAX(1, sc->vmx_ntxqueues / 2);
+
+	/*
+	 * Most drivers just ignore the return value - it only fails
+	 * with ENOMEM so an error is not likely. It is hard for us
+	 * to recover from an error here.
+	 */
+	error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
+	    "%s taskq", device_get_nameunit(dev));
+	if (error)
+		device_printf(dev, "failed to start taskqueue: %d", error);
+}
+
+static void
+vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
+{
+	struct vmxnet3_txqueue *txq;
+	int i;
+
+	if (sc->vmx_tq != NULL) {
+		for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
+			txq = &sc->vmx_txq[i];
+			taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
+		}
+	}
+}
+
+static void
+vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
+{
+	if (sc->vmx_tq != NULL) {
+		taskqueue_free(sc->vmx_tq);
+		sc->vmx_tq = NULL;
+	}
+}
+#endif
+
 static int
 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
 {
@@ -839,6 +983,15 @@ vmxnet3_init_txq(struct vmxnet3_softc *s
 
 	txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
 
+#ifndef VMXNET3_LEGACY_TX
+	TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
+
+	txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
+	    M_NOWAIT, &txq->vxtxq_mtx);
+	if (txq->vxtxq_br == NULL)
+		return (ENOMEM);
+#endif
+
 	return (0);
 }
 
@@ -847,20 +1000,32 @@ vmxnet3_alloc_rxtx_queues(struct vmxnet3
 {
 	int i, error;
 
+	/*
+	 * Only attempt to create multiple queues if MSIX is available. MSIX is
+	 * disabled by default because its apparently broken for devices passed
+	 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
+	 * must be set to zero for MSIX. This check prevents us from allocating
+	 * queue structures that we will not use.
+	 */
+	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
+		sc->vmx_max_nrxqueues = 1;
+		sc->vmx_max_ntxqueues = 1;
+	}
+
 	sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
-	    sc->vmx_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
+	    sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
 	sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
-	    sc->vmx_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
+	    sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
 		return (ENOMEM);
 
-	for (i = 0; i < sc->vmx_nrxqueues; i++) {
+	for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
 		error = vmxnet3_init_rxq(sc, i);
 		if (error)
 			return (error);
 	}
 
-	for (i = 0; i < sc->vmx_ntxqueues; i++) {
+	for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
 		error = vmxnet3_init_txq(sc, i);
 		if (error)
 			return (error);
@@ -901,6 +1066,13 @@ vmxnet3_destroy_txq(struct vmxnet3_txque
 	txq->vxtxq_sc = NULL;
 	txq->vxtxq_id = -1;
 
+#ifndef VMXNET3_LEGACY_TX
+	if (txq->vxtxq_br != NULL) {
+		buf_ring_free(txq->vxtxq_br, M_DEVBUF);
+		txq->vxtxq_br = NULL;
+	}
+#endif
+
 	if (txr->vxtxr_txbuf != NULL) {
 		free(txr->vxtxr_txbuf, M_DEVBUF);
 		txr->vxtxr_txbuf = NULL;
@@ -916,14 +1088,14 @@ vmxnet3_free_rxtx_queues(struct vmxnet3_
 	int i;
 
 	if (sc->vmx_rxq != NULL) {
-		for (i = 0; i < sc->vmx_nrxqueues; i++)
+		for (i = 0; i < sc->vmx_max_nrxqueues; i++)
 			vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
 		free(sc->vmx_rxq, M_DEVBUF);
 		sc->vmx_rxq = NULL;
 	}
 
 	if (sc->vmx_txq != NULL) {
-		for (i = 0; i < sc->vmx_ntxqueues; i++)
+		for (i = 0; i < sc->vmx_max_ntxqueues; i++)
 			vmxnet3_destroy_txq(&sc->vmx_txq[i]);
 		free(sc->vmx_txq, M_DEVBUF);
 		sc->vmx_txq = NULL;
@@ -967,6 +1139,17 @@ vmxnet3_alloc_shared_data(struct vmxnet3
 		kva += sizeof(struct vmxnet3_rxq_shared);
 	}
 
+	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+		size = sizeof(struct vmxnet3_rss_shared);
+		error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
+		if (error) {
+			device_printf(dev, "cannot alloc rss shared memory\n");
+			return (error);
+		}
+		sc->vmx_rss =
+		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
+	}
+
 	return (0);
 }
 
@@ -974,6 +1157,11 @@ static void
 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
 {
 
+	if (sc->vmx_rss != NULL) {
+		vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
+		sc->vmx_rss = NULL;
+	}
+
 	if (sc->vmx_qs != NULL) {
 		vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
 		sc->vmx_qs = NULL;
@@ -1010,7 +1198,7 @@ vmxnet3_alloc_txq_data(struct vmxnet3_so
 		    BUS_SPACE_MAXADDR,		/* lowaddr */
 		    BUS_SPACE_MAXADDR,		/* highaddr */
 		    NULL, NULL,			/* filter, filterarg */
-		    VMXNET3_TSO_MAXSIZE,	/* maxsize */
+		    VMXNET3_TX_MAXSIZE,		/* maxsize */
 		    VMXNET3_TX_MAXSEGS,		/* nsegments */
 		    VMXNET3_TX_MAXSEGSIZE,	/* maxsegsize */
 		    0,				/* flags */
@@ -1335,6 +1523,13 @@ vmxnet3_init_shared_data(struct vmxnet3_
 	ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
 	ds->nrxsg_max = sc->vmx_max_rxsegs;
 
+	/* RSS conf */
+	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+		ds->rss.version = 1;
+		ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
+		ds->rss.len = sc->vmx_rss_dma.dma_size;
+	}
+
 	/* Interrupt control. */
 	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
 	ds->nintr = sc->vmx_nintrs;
@@ -1394,9 +1589,43 @@ vmxnet3_reinit_interface(struct vmxnet3_
 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 		ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
 	if (ifp->if_capenable & IFCAP_TSO4)
-		ifp->if_hwassist |= CSUM_TSO;
+		ifp->if_hwassist |= CSUM_IP_TSO;
 	if (ifp->if_capenable & IFCAP_TSO6)
-		ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */
+		ifp->if_hwassist |= CSUM_IP6_TSO;
+}
+
+static void
+vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
+{
+	/*
+	 * Use the same key as the Linux driver until FreeBSD can do
+	 * RSS (presumably Toeplitz) in software.
+	 */
+	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
+	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
+	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
+	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
+	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
+	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
+	};
+
+	struct vmxnet3_driver_shared *ds;
+	struct vmxnet3_rss_shared *rss;
+	int i;
+
+	ds = sc->vmx_ds;
+	rss = sc->vmx_rss;
+
+	rss->hash_type =
+	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
+	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
+	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
+	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
+	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
+	memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
+
+	for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
+		rss->ind_table[i] = i % sc->vmx_nrxqueues;
 }
 
 static void
@@ -1408,6 +1637,10 @@ vmxnet3_reinit_shared_data(struct vmxnet
 	ifp = sc->vmx_ifp;
 	ds = sc->vmx_ds;
 
+	ds->mtu = ifp->if_mtu;
+	ds->ntxqueue = sc->vmx_ntxqueues;
+	ds->nrxqueue = sc->vmx_nrxqueues;
+
 	ds->upt_features = 0;
 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
 		ds->upt_features |= UPT1_F_CSUM;
@@ -1416,9 +1649,10 @@ vmxnet3_reinit_shared_data(struct vmxnet
 	if (ifp->if_capenable & IFCAP_LRO)
 		ds->upt_features |= UPT1_F_LRO;
 
-	ds->mtu = ifp->if_mtu;
-	ds->ntxqueue = sc->vmx_ntxqueues;
-	ds->nrxqueue = sc->vmx_nrxqueues;
+	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+		ds->upt_features |= UPT1_F_RSS;
+		vmxnet3_reinit_rss_shared_data(sc);
+	}
 
 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
@@ -1471,15 +1705,28 @@ vmxnet3_setup_interface(struct vmxnet3_s
 	}
 
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+#if __FreeBSD_version < 1000025
+	ifp->if_baudrate = 1000000000;
+#elif __FreeBSD_version < 1100011
+	if_initbaudrate(ifp, IF_Gbps(10));
+#else
 	ifp->if_baudrate = IF_Gbps(10);
+#endif
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_init = vmxnet3_init;
 	ifp->if_ioctl = vmxnet3_ioctl;
+	ifp->if_hw_tsomax = VMXNET3_TSO_MAXSIZE;
+
+#ifdef VMXNET3_LEGACY_TX
 	ifp->if_start = vmxnet3_start;
 	ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1;
 	IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1);
 	IFQ_SET_READY(&ifp->if_snd);
+#else
+	ifp->if_transmit = vmxnet3_txq_mq_start;
+	ifp->if_qflush = vmxnet3_qflush;
+#endif
 
 	vmxnet3_get_lladdr(sc);
 	ether_ifattach(ifp, sc->vmx_lladdr);
@@ -1527,8 +1774,11 @@ vmxnet3_evintr(struct vmxnet3_softc *sc)
 	event = sc->vmx_ds->event;
 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
 
-	if (event & VMXNET3_EVENT_LINK)
+	if (event & VMXNET3_EVENT_LINK) {
 		vmxnet3_link_status(sc);
+		if (sc->vmx_link_active != 0)
+			vmxnet3_tx_start_all(sc);
+	}
 
 	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
 		reset = 1;
@@ -1564,6 +1814,7 @@ vmxnet3_txq_eof(struct vmxnet3_txqueue *
 	struct vmxnet3_comp_ring *txc;
 	struct vmxnet3_txcompdesc *txcd;
 	struct vmxnet3_txbuf *txb;
+	struct mbuf *m;
 	u_int sop;
 
 	sc = txq->vxtxq_sc;
@@ -1587,15 +1838,18 @@ vmxnet3_txq_eof(struct vmxnet3_txqueue *
 		sop = txr->vxtxr_next;
 		txb = &txr->vxtxr_txbuf[sop];
 
-		if (txb->vtxb_m != NULL) {
+		if ((m = txb->vtxb_m) != NULL) {
 			bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
 			    BUS_DMASYNC_POSTWRITE);
 			bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
 
-			m_freem(txb->vtxb_m);
-			txb->vtxb_m = NULL;
+			txq->vxtxq_stats.vmtxs_opackets++;
+			txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
+			if (m->m_flags & M_MCAST)
+				txq->vxtxq_stats.vmtxs_omcasts++;
 
-			ifp->if_opackets++;
+			m_freem(m);
+			txb->vtxb_m = NULL;
 		}
 
 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
@@ -1769,11 +2023,39 @@ vmxnet3_rxq_input(struct vmxnet3_rxqueue
 	ifp = sc->vmx_ifp;
 
 	if (rxcd->error) {
-		ifp->if_ierrors++;
+		rxq->vxrxq_stats.vmrxs_ierrors++;
 		m_freem(m);
 		return;
 	}
 
+#ifdef notyet
+	switch (rxcd->rss_type) {
+	case VMXNET3_RCD_RSS_TYPE_IPV4:
+		m->m_pkthdr.flowid = rxcd->rss_hash;
+		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
+		break;
+	case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
+		m->m_pkthdr.flowid = rxcd->rss_hash;
+		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
+		break;
+	case VMXNET3_RCD_RSS_TYPE_IPV6:
+		m->m_pkthdr.flowid = rxcd->rss_hash;
+		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
+		break;
+	case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
+		m->m_pkthdr.flowid = rxcd->rss_hash;
+		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
+		break;
+	default: /* VMXNET3_RCD_RSS_TYPE_NONE */
+		m->m_pkthdr.flowid = rxq->vxrxq_id;
+		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
+		break;
+	}
+#else
+	m->m_pkthdr.flowid = rxq->vxrxq_id;
+	m->m_flags |= M_FLOWID;
+#endif
+
 	if (!rxcd->no_csum)
 		vmxnet3_rx_csum(rxcd, m);
 	if (rxcd->vlan) {
@@ -1781,7 +2063,9 @@ vmxnet3_rxq_input(struct vmxnet3_rxqueue
 		m->m_pkthdr.ether_vtag = rxcd->vtag;
 	}
 
-	ifp->if_ipackets++;
+	rxq->vxrxq_stats.vmrxs_ipackets++;
+	rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
+
 	VMXNET3_RXQ_UNLOCK(rxq);
 	(*ifp->if_input)(ifp, m);
 	VMXNET3_RXQ_LOCK(rxq);
@@ -1863,7 +2147,7 @@ vmxnet3_rxq_eof(struct vmxnet3_rxqueue *
 			}
 
 			if (vmxnet3_newbuf(sc, rxr) != 0) {
-				ifp->if_iqdrops++;
+				rxq->vxrxq_stats.vmrxs_iqdrops++;
 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
 				if (!rxcd->eop)
 					vmxnet3_rxq_discard_chain(rxq);
@@ -1882,7 +2166,7 @@ vmxnet3_rxq_eof(struct vmxnet3_rxqueue *
 			    ("%s: frame not started?", __func__));
 
 			if (vmxnet3_newbuf(sc, rxr) != 0) {
-				ifp->if_iqdrops++;
+				rxq->vxrxq_stats.vmrxs_iqdrops++;
 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
 				if (!rxcd->eop)
 					vmxnet3_rxq_discard_chain(rxq);
@@ -1951,8 +2235,7 @@ vmxnet3_legacy_intr(void *xsc)
 
 	VMXNET3_TXQ_LOCK(txq);
 	vmxnet3_txq_eof(txq);
-	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
-		vmxnet3_start_locked(ifp);
+	vmxnet3_txq_start(txq);
 	VMXNET3_TXQ_UNLOCK(txq);
 
 	vmxnet3_enable_all_intrs(sc);
@@ -1974,8 +2257,7 @@ vmxnet3_txq_intr(void *xtxq)
 
 	VMXNET3_TXQ_LOCK(txq);
 	vmxnet3_txq_eof(txq);
-	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
-		vmxnet3_start_locked(ifp);
+	vmxnet3_txq_start(txq);
 	VMXNET3_TXQ_UNLOCK(txq);
 
 	vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
@@ -2054,6 +2336,7 @@ vmxnet3_rxstop(struct vmxnet3_softc *sc,
 
 			if (rxb->vrxb_m == NULL)
 				continue;
+
 			bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
 			    BUS_DMASYNC_POSTREAD);
 			bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
@@ -2316,7 +2599,8 @@ vmxnet3_init(void *xsc)
  * the mbuf packet header. Bug andre@.
  */
 static int
-vmxnet3_txq_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
+vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
+    int *etype, int *proto, int *start)
 {
 	struct ether_vlan_header *evh;
 	int offset;
@@ -2340,7 +2624,7 @@ vmxnet3_txq_offload_ctx(struct mbuf *m, 
 			    (caddr_t) &iphdr);
 			ip = &iphdr;
 		} else
-			ip = (struct ip *)(m->m_data + offset);
+			ip = mtodo(m, offset);
 		*proto = ip->ip_p;
 		*start = offset + (ip->ip_hl << 2);
 		break;
@@ -2368,19 +2652,21 @@ vmxnet3_txq_offload_ctx(struct mbuf *m, 
 			return (EINVAL);
 		}
 
-		if (m->m_len < *start + sizeof(struct tcphdr)) {
-			m_copydata(m, offset, sizeof(struct tcphdr),
-			    (caddr_t) &tcphdr);
-			tcp = &tcphdr;
-		} else
-			tcp = (struct tcphdr *)(m->m_data + *start);
+		txq->vxtxq_stats.vmtxs_tso++;
 
 		/*
 		 * For TSO, the size of the protocol header is also
 		 * included in the descriptor header size.
 		 */
+		if (m->m_len < *start + sizeof(struct tcphdr)) {
+			m_copydata(m, offset, sizeof(struct tcphdr),
+			    (caddr_t) &tcphdr);
+			tcp = &tcphdr;
+		} else
+			tcp = mtodo(m, *start);
 		*start += (tcp->th_off << 2);
-	}
+	} else
+		txq->vxtxq_stats.vmtxs_csum++;
 
 	return (0);
 }
@@ -2392,18 +2678,17 @@ vmxnet3_txq_load_mbuf(struct vmxnet3_txq
 	struct vmxnet3_txring *txr;
 	struct mbuf *m;
 	bus_dma_tag_t tag;
-	int maxsegs, error;
+	int error;
 
 	txr = &txq->vxtxq_cmd_ring;
 	m = *m0;
 	tag = txr->vxtxr_txtag;
-	maxsegs = VMXNET3_TX_MAXSEGS;
 
 	error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
 	if (error == 0 || error != EFBIG)
 		return (error);
 
-	m = m_collapse(m, M_NOWAIT, maxsegs);
+	m = m_defrag(m, M_NOWAIT);
 	if (m != NULL) {
 		*m0 = m;
 		error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
@@ -2413,8 +2698,9 @@ vmxnet3_txq_load_mbuf(struct vmxnet3_txq
 	if (error) {
 		m_freem(*m0);
 		*m0 = NULL;
+		txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
 	} else
-		txq->vxtxq_sc->vmx_stats.vmst_collapsed++;
+		txq->vxtxq_sc->vmx_stats.vmst_defragged++;
 
 	return (error);
 }
@@ -2457,13 +2743,13 @@ vmxnet3_txq_encap(struct vmxnet3_txqueue
 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
 
 	if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
-		txq->vxtxq_stats.vtxrs_full++;
+		txq->vxtxq_stats.vmtxs_full++;
 		vmxnet3_txq_unload_mbuf(txq, dmap);
 		return (ENOSPC);
 	} else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
-		error = vmxnet3_txq_offload_ctx(m, &etype, &proto, &start);
+		error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
 		if (error) {
-			txq->vxtxq_stats.vtxrs_offload_failed++;
+			txq->vxtxq_stats.vmtxs_offload_failed++;
 			vmxnet3_txq_unload_mbuf(txq, dmap);
 			m_freem(m);
 			*m0 = NULL;
@@ -2529,6 +2815,22 @@ vmxnet3_txq_encap(struct vmxnet3_txqueue
 }
 
 static void
+vmxnet3_txq_update_pending(struct vmxnet3_txqueue *txq)
+{
+	struct vmxnet3_txring *txr;
+
+	txr = &txq->vxtxq_cmd_ring;
+
+	if (txq->vxtxq_ts->npending > 0) {
+		txq->vxtxq_ts->npending = 0;
+		vmxnet3_write_bar0(txq->vxtxq_sc,
+		    VMXNET3_BAR0_TXH(txq->vxtxq_id), txr->vxtxr_head);
+	}
+}
+
+#ifdef VMXNET3_LEGACY_TX
+
+static void
 vmxnet3_start_locked(struct ifnet *ifp)
 {
 	struct vmxnet3_softc *sc;
@@ -2573,11 +2875,7 @@ vmxnet3_start_locked(struct ifnet *ifp)
 	}
 
 	if (tx > 0) {
-		if (txq->vxtxq_ts->npending > 0) {
-			txq->vxtxq_ts->npending = 0;
-			vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
-			    txr->vxtxr_head);
-		}
+		vmxnet3_txq_update_pending(txq);
 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
 	}
 }
@@ -2596,6 +2894,153 @@ vmxnet3_start(struct ifnet *ifp)
 	VMXNET3_TXQ_UNLOCK(txq);
 }
 
+#else /* !VMXNET3_LEGACY_TX */
+
+static int
+vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
+{
+	struct vmxnet3_softc *sc;
+	struct vmxnet3_txring *txr;
+	struct buf_ring *br;
+	struct ifnet *ifp;
+	int tx, avail, error;
+
+	sc = txq->vxtxq_sc;
+	br = txq->vxtxq_br;
+	ifp = sc->vmx_ifp;
+	txr = &txq->vxtxq_cmd_ring;
+	tx = 0;
+	error = 0;
+
+	VMXNET3_TXQ_LOCK_ASSERT(txq);
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
+	    sc->vmx_link_active == 0) {
+		if (m != NULL)
+			error = drbr_enqueue(ifp, br, m);
+		return (error);
+	}
+
+	if (m != NULL) {
+		error = drbr_enqueue(ifp, br, m);
+		if (error)
+			return (error);
+	}
+
+	while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
+		m = drbr_peek(ifp, br);
+		if (m == NULL)
+			break;
+
+		/* Assume worse case if this mbuf is the head of a chain. */
+		if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
+			drbr_putback(ifp, br, m);
+			error = ENOBUFS;
+			break;
+		}
+
+		error = vmxnet3_txq_encap(txq, &m);
+		if (error) {
+			if (m != NULL)
+				drbr_putback(ifp, br, m);
+			else
+				drbr_advance(ifp, br);
+			break;
+		}
+		drbr_advance(ifp, br);
+
+		tx++;
+		ETHER_BPF_MTAP(ifp, m);
+	}
+
+	if (tx > 0) {
+		vmxnet3_txq_update_pending(txq);
+		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
+	}
+
+	return (error);
+}
+
+static int
+vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
+{
+	struct vmxnet3_softc *sc;
+	struct vmxnet3_txqueue *txq;
+	int i, ntxq, error;
+
+	sc = ifp->if_softc;
+	ntxq = sc->vmx_ntxqueues;
+
+	if (m->m_flags & M_FLOWID)
+		i = m->m_pkthdr.flowid % ntxq;
+	else
+		i = curcpu % ntxq;
+
+	txq = &sc->vmx_txq[i];
+
+	if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
+		error = vmxnet3_txq_mq_start_locked(txq, m);
+		VMXNET3_TXQ_UNLOCK(txq);
+	} else {
+		error = drbr_enqueue(ifp, txq->vxtxq_br, m);
+		taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
+	}
+
+	return (error);
+}
+
+static void
+vmxnet3_txq_tq_deferred(void *xtxq, int pending)
+{
+	struct vmxnet3_softc *sc;
+	struct vmxnet3_txqueue *txq;
+
+	txq = xtxq;
+	sc = txq->vxtxq_sc;
+
+	VMXNET3_TXQ_LOCK(txq);
+	if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
+		vmxnet3_txq_mq_start_locked(txq, NULL);

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403170545.s2H5jTZT076017>