Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 8 Feb 2015 09:28:56 +0000 (UTC)
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r278374 - in head/sys/dev/cxgbe: . iw_cxgbe tom
Message-ID:  <201502080928.t189Subt053419@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: np
Date: Sun Feb  8 09:28:55 2015
New Revision: 278374
URL: https://svnweb.freebsd.org/changeset/base/278374

Log:
  cxgbe(4): tidy up some of the interaction between the Upper Layer
  Drivers (ULDs) and the base if_cxgbe driver.
  
  Track the per-adapter activation of ULDs in a new "active_ulds" field.
  This was done pretty arbitrarily before this change -- via TOM_INIT_DONE
  in adapter->flags for TOM, and the (1 << MAX_NPORTS) bit in
  adapter->offload_map for iWARP.
  
  iWARP and hw-accelerated iSCSI rely on the TOE (supported by the TOM
  ULD).  The rules are:
  a) If the iWARP and/or iSCSI ULDs are available when TOE is enabled then
     iWARP and/or iSCSI are enabled too.
  b) When the iWARP and iSCSI modules are loaded they go looking for
     adapters with TOE enabled and enable themselves on that adapter.
  c) You cannot deactivate or unload the TOM module from underneath iWARP
     or iSCSI.  Any such attempt will fail with EBUSY.
  
  MFC after:	2 weeks

Modified:
  head/sys/dev/cxgbe/adapter.h
  head/sys/dev/cxgbe/iw_cxgbe/device.c
  head/sys/dev/cxgbe/offload.h
  head/sys/dev/cxgbe/t4_main.c
  head/sys/dev/cxgbe/tom/t4_listen.c
  head/sys/dev/cxgbe/tom/t4_tom.c

Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/adapter.h	Sun Feb  8 09:28:55 2015	(r278374)
@@ -192,7 +192,7 @@ enum {
 	/* INTR_DIRECT	= (1 << 2),	No longer used. */
 	MASTER_PF	= (1 << 3),
 	ADAP_SYSCTL_CTX	= (1 << 4),
-	TOM_INIT_DONE	= (1 << 5),
+	/* TOM_INIT_DONE= (1 << 5),	No longer used */
 	BUF_PACKING_OK	= (1 << 6),
 
 	CXGBE_BUSY	= (1 << 9),
@@ -758,7 +758,8 @@ struct adapter {
 	uint16_t doorbells;
 	int open_device_map;
 #ifdef TCP_OFFLOAD
-	int offload_map;
+	int offload_map;	/* ports with IFCAP_TOE enabled */
+	int active_ulds;	/* ULDs activated on this adapter */
 #endif
 	int flags;
 

Modified: head/sys/dev/cxgbe/iw_cxgbe/device.c
==============================================================================
--- head/sys/dev/cxgbe/iw_cxgbe/device.c	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/iw_cxgbe/device.c	Sun Feb  8 09:28:55 2015	(r278374)
@@ -213,7 +213,7 @@ c4iw_activate(struct adapter *sc)
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
-	if (isset(&sc->offload_map, MAX_NPORTS)) {
+	if (uld_active(sc, ULD_IWARP)) {
 		KASSERT(0, ("%s: RDMA already eanbled on sc %p", __func__, sc));
 		return (0);
 	}
@@ -265,9 +265,9 @@ c4iw_activate_all(struct adapter *sc, vo
 	if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwact") != 0)
 		return;
 
-	if (!isset(&sc->offload_map, MAX_NPORTS) &&
-	    t4_activate_uld(sc, ULD_IWARP) == 0)
-		setbit(&sc->offload_map, MAX_NPORTS);
+	/* Activate iWARP if any port on this adapter has IFCAP_TOE enabled. */
+	if (sc->offload_map && !uld_active(sc, ULD_IWARP))
+		(void) t4_activate_uld(sc, ULD_IWARP);
 
 	end_synchronized_op(sc, 0);
 }
@@ -279,9 +279,8 @@ c4iw_deactivate_all(struct adapter *sc, 
 	if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwdea") != 0)
 		return;
 
-	if (isset(&sc->offload_map, MAX_NPORTS) &&
-	    t4_deactivate_uld(sc, ULD_IWARP) == 0)
-		clrbit(&sc->offload_map, MAX_NPORTS);
+	if (uld_active(sc, ULD_IWARP))
+	    (void) t4_deactivate_uld(sc, ULD_IWARP);
 
 	end_synchronized_op(sc, 0);
 }

Modified: head/sys/dev/cxgbe/offload.h
==============================================================================
--- head/sys/dev/cxgbe/offload.h	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/offload.h	Sun Feb  8 09:28:55 2015	(r278374)
@@ -127,9 +127,10 @@ struct t4_virt_res {                    
 
 #ifdef TCP_OFFLOAD
 enum {
-	ULD_TOM = 1,
-	ULD_IWARP = 2,
-	ULD_ISCSI = 3,
+	ULD_TOM = 0,
+	ULD_IWARP,
+	ULD_ISCSI,
+	ULD_MAX = ULD_ISCSI
 };
 
 struct adapter;
@@ -156,5 +157,6 @@ int t4_unregister_uld(struct uld_info *)
 int t4_activate_uld(struct adapter *, int);
 int t4_deactivate_uld(struct adapter *, int);
 void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *);
+int uld_active(struct adapter *, int);
 #endif
 #endif

Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/t4_main.c	Sun Feb  8 09:28:55 2015	(r278374)
@@ -7144,7 +7144,7 @@ set_filter_mode(struct adapter *sc, uint
 	}
 
 #ifdef TCP_OFFLOAD
-	if (sc->offload_map) {
+	if (uld_active(sc, ULD_TOM)) {
 		rc = EBUSY;
 		goto done;
 	}
@@ -8236,7 +8236,7 @@ toe_capability(struct port_info *pi, int
 		if (isset(&sc->offload_map, pi->port_id))
 			return (0);
 
-		if (!(sc->flags & TOM_INIT_DONE)) {
+		if (!uld_active(sc, ULD_TOM)) {
 			rc = t4_activate_uld(sc, ULD_TOM);
 			if (rc == EAGAIN) {
 				log(LOG_WARNING,
@@ -8247,16 +8247,22 @@ toe_capability(struct port_info *pi, int
 				return (rc);
 			KASSERT(sc->tom_softc != NULL,
 			    ("%s: TOM activated but softc NULL", __func__));
-			KASSERT(sc->flags & TOM_INIT_DONE,
+			KASSERT(uld_active(sc, ULD_TOM),
 			    ("%s: TOM activated but flag not set", __func__));
 		}
 
+		/* Activate iWARP and iSCSI too, if the modules are loaded. */
+		if (!uld_active(sc, ULD_IWARP))
+			(void) t4_activate_uld(sc, ULD_IWARP);
+		if (!uld_active(sc, ULD_ISCSI))
+			(void) t4_activate_uld(sc, ULD_ISCSI);
+
 		setbit(&sc->offload_map, pi->port_id);
 	} else {
 		if (!isset(&sc->offload_map, pi->port_id))
 			return (0);
 
-		KASSERT(sc->flags & TOM_INIT_DONE,
+		KASSERT(uld_active(sc, ULD_TOM),
 		    ("%s: TOM never initialized?", __func__));
 		clrbit(&sc->offload_map, pi->port_id);
 	}
@@ -8316,11 +8322,15 @@ done:
 int
 t4_activate_uld(struct adapter *sc, int id)
 {
-	int rc = EAGAIN;
+	int rc;
 	struct uld_info *ui;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
+	if (id < 0 || id > ULD_MAX)
+		return (EINVAL);
+	rc = EAGAIN;	/* kldoad the module with this ULD and try again. */
+
 	sx_slock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
@@ -8328,16 +8338,18 @@ t4_activate_uld(struct adapter *sc, int 
 			if (!(sc->flags & FULL_INIT_DONE)) {
 				rc = adapter_full_init(sc);
 				if (rc != 0)
-					goto done;
+					break;
 			}
 
 			rc = ui->activate(sc);
-			if (rc == 0)
+			if (rc == 0) {
+				setbit(&sc->active_ulds, id);
 				ui->refcount++;
-			goto done;
+			}
+			break;
 		}
 	}
-done:
+
 	sx_sunlock(&t4_uld_list_lock);
 
 	return (rc);
@@ -8346,26 +8358,41 @@ done:
 int
 t4_deactivate_uld(struct adapter *sc, int id)
 {
-	int rc = EINVAL;
+	int rc;
 	struct uld_info *ui;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
+	if (id < 0 || id > ULD_MAX)
+		return (EINVAL);
+	rc = ENXIO;
+
 	sx_slock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
 		if (ui->uld_id == id) {
 			rc = ui->deactivate(sc);
-			if (rc == 0)
+			if (rc == 0) {
+				clrbit(&sc->active_ulds, id);
 				ui->refcount--;
-			goto done;
+			}
+			break;
 		}
 	}
-done:
+
 	sx_sunlock(&t4_uld_list_lock);
 
 	return (rc);
 }
+
+int
+uld_active(struct adapter *sc, int uld_id)
+{
+
+	MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
+
+	return (isset(&sc->active_ulds, uld_id));
+}
 #endif
 
 /*

Modified: head/sys/dev/cxgbe/tom/t4_listen.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_listen.c	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/tom/t4_listen.c	Sun Feb  8 09:28:55 2015	(r278374)
@@ -523,7 +523,7 @@ t4_listen_start(struct toedev *tod, stru
 		goto done;
 	}
 
-	KASSERT(sc->flags & TOM_INIT_DONE,
+	KASSERT(uld_active(sc, ULD_TOM),
 	    ("%s: TOM not initialized", __func__));
 #endif
 

Modified: head/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.c	Sun Feb  8 09:25:06 2015	(r278373)
+++ head/sys/dev/cxgbe/tom/t4_tom.c	Sun Feb  8 09:28:55 2015	(r278374)
@@ -746,7 +746,7 @@ update_clip(struct adapter *sc, void *ar
 	if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomuc"))
 		return;
 
-	if (sc->flags & TOM_INIT_DONE)
+	if (uld_active(sc, ULD_TOM))
 		update_clip_table(sc, sc->tom_softc);
 
 	end_synchronized_op(sc, LOCK_HELD);
@@ -1025,7 +1025,6 @@ t4_tom_activate(struct adapter *sc)
 		TOEDEV(sc->port[i]->ifp) = &td->tod;
 
 	sc->tom_softc = td;
-	sc->flags |= TOM_INIT_DONE;
 	register_toedev(sc->tom_softc);
 
 done:
@@ -1048,6 +1047,9 @@ t4_tom_deactivate(struct adapter *sc)
 	if (sc->offload_map != 0)
 		return (EBUSY);	/* at least one port has IFCAP_TOE enabled */
 
+	if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI))
+		return (EBUSY);	/* both iWARP and iSCSI rely on the TOE. */
+
 	mtx_lock(&td->toep_list_lock);
 	if (!TAILQ_EMPTY(&td->toep_list))
 		rc = EBUSY;
@@ -1068,7 +1070,6 @@ t4_tom_deactivate(struct adapter *sc)
 		unregister_toedev(sc->tom_softc);
 		free_tom_data(sc, td);
 		sc->tom_softc = NULL;
-		sc->flags &= ~TOM_INIT_DONE;
 	}
 
 	return (rc);
@@ -1122,7 +1123,7 @@ tom_uninit(struct adapter *sc, void *arg
 		return;
 
 	/* Try to free resources (works only if no port has IFCAP_TOE) */
-	if (sc->flags & TOM_INIT_DONE)
+	if (uld_active(sc, ULD_TOM))
 		t4_deactivate_uld(sc, ULD_TOM);
 
 	end_synchronized_op(sc, 0);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201502080928.t189Subt053419>