Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 12 Nov 2011 09:52:28 +0000 (UTC)
From:      Alexander Motin <mav@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r227464 - in head: sbin/geom/class/multipath sys/geom/multipath
Message-ID:  <201111120952.pAC9qSUL036029@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: mav
Date: Sat Nov 12 09:52:27 2011
New Revision: 227464
URL: http://svn.freebsd.org/changeset/base/227464

Log:
  Major GEOM MULTIPATH class rewrite:
   - Improved locking and destruction process to fix crashes.
   - Improved "automatic" configuration method to make it consistent and safe
  by reading metadata back from all specified paths after writing to one.
   - Added provider size check to reduce chance of ordering conflict with
  other GEOM classes.
   - Added "manual" configuration method without using on-disk metadata.
   - Added "add" and "remove" commands to allow manage paths manually.
   - Failed paths are no longer dropped from geom, but only marked as FAIL
  and excluded from I/O operations.
   - Automatically restore failed paths when all others paths are marked
  as failed, for example, because of device-caused (not transport) errors.
   - Added "fail" and "restore" commands to manually control FAIL flag.
   - geom is now destroyed on last path disconnection.
   - Added optional Active/Active mode support. Unlike Active/Passive
  mode, load evenly distributed between all working paths. If supported by
  the device, it allows to significantly improve performance, utilizing
  bandwidth of all paths. It is controlled by -A option during creation.
  Disabled by default now.
   - Improved `status` and `list` commands output.
  
  Sponsored by:	iXsystems, inc.
  MFC after:	1 month

Modified:
  head/sbin/geom/class/multipath/geom_multipath.c
  head/sbin/geom/class/multipath/gmultipath.8
  head/sys/geom/multipath/g_multipath.c
  head/sys/geom/multipath/g_multipath.h

Modified: head/sbin/geom/class/multipath/geom_multipath.c
==============================================================================
--- head/sbin/geom/class/multipath/geom_multipath.c	Sat Nov 12 08:40:52 2011	(r227463)
+++ head/sbin/geom/class/multipath/geom_multipath.c	Sat Nov 12 09:52:27 2011	(r227464)
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <strings.h>
 #include <assert.h>
 #include <libgeom.h>
+#include <unistd.h>
 #include <uuid.h>
 #include <geom/multipath/g_multipath.h>
 
@@ -48,31 +49,58 @@ uint32_t version = G_MULTIPATH_VERSION;
 static void mp_main(struct gctl_req *, unsigned int);
 static void mp_label(struct gctl_req *);
 static void mp_clear(struct gctl_req *);
-static void mp_add(struct gctl_req *);
 
 struct g_command class_commands[] = {
 	{
-		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"add", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"add", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
-		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
-		"[-v] prov ..."
+		"remove", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"fail", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"restore", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
 		"rotate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"[-v] name"
 	},
 	{
 		"getactive", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"stop", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
 		"[-v] prov ..."
 	},
 	G_CMD_SENTINEL
@@ -90,8 +118,6 @@ mp_main(struct gctl_req *req, unsigned i
 	}
 	if (strcmp(name, "label") == 0) {
 		mp_label(req);
-	} else if (strcmp(name, "add") == 0) {
-		mp_add(req);
 	} else if (strcmp(name, "clear") == 0) {
 		mp_clear(req);
 	} else {
@@ -103,13 +129,13 @@ static void
 mp_label(struct gctl_req *req)
 {
 	struct g_multipath_metadata md;
-	off_t disksiz = 0, msize;
-	uint8_t *sector;
+	off_t disksize = 0, msize;
+	uint8_t *sector, *rsector;
 	char *ptr;
 	uuid_t uuid;
 	uint32_t secsize = 0, ssize, status;
-	const char *name, *mpname;
-	int error, i, nargs;
+	const char *name, *name2, *mpname;
+	int error, i, nargs, fd;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs < 2) {
@@ -132,14 +158,14 @@ mp_label(struct gctl_req *req)
 		}
 		if (i == 1) {
 			secsize = ssize;
-			disksiz	= msize;
+			disksize = msize;
 		} else {
 			if (secsize != ssize) {
 				gctl_error(req, "%s sector size %u different.",
 				    name, ssize);
 				return;
 			}
-			if (disksiz != msize) {
+			if (disksize != msize) {
 				gctl_error(req, "%s media size %ju different.",
 				    name, (intmax_t)msize);
 				return;
@@ -155,7 +181,7 @@ mp_label(struct gctl_req *req)
 	md.md_version = G_MULTIPATH_VERSION;
 	mpname = gctl_get_ascii(req, "arg0");
 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
-	md.md_size = disksiz;
+	md.md_size = disksize;
 	md.md_sectorsize = secsize;
 	uuid_create(&uuid, &status);
 	if (status != uuid_s_ok) {
@@ -168,19 +194,10 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	strlcpy(md.md_uuid, ptr, sizeof (md.md_uuid));
+	md.md_active_active = gctl_get_int(req, "active_active");
 	free(ptr);
 
 	/*
-	 * Clear metadata on initial provider first.
-	 */
-	name = gctl_get_ascii(req, "arg1");
-	error = g_metadata_clear(name, NULL);
-	if (error != 0) {
-		gctl_error(req, "cannot clear metadata on %s: %s.", name, strerror(error));
-		return;
-	}
-
-	/*
 	 * Allocate a sector to write as metadata.
 	 */
 	sector = malloc(secsize);
@@ -189,6 +206,12 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	memset(sector, 0, secsize);
+	rsector = malloc(secsize);
+	if (rsector == NULL) {
+		free(sector);
+		gctl_error(req, "unable to allocate metadata buffer");
+		return;
+	}
 
 	/*
 	 * encode the metadata
@@ -198,6 +221,7 @@ mp_label(struct gctl_req *req)
 	/*
 	 * Store metadata on the initial provider.
 	 */
+	name = gctl_get_ascii(req, "arg1");
 	error = g_metadata_store(name, sector, secsize);
 	if (error != 0) {
 		gctl_error(req, "cannot store metadata on %s: %s.", name, strerror(error));
@@ -205,20 +229,29 @@ mp_label(struct gctl_req *req)
 	}
 
 	/*
-	 * Now add the rest of the providers.
+	 * Now touch the rest of the providers to hint retaste.
 	 */
-	error = gctl_change_param(req, "verb", -1, "add");
-	if (error) {
-		gctl_error(req, "unable to change verb to \"add\": %s.", strerror(error));
-		return;
-	}
 	for (i = 2; i < nargs; i++) {
-		error = gctl_change_param(req, "arg1", -1, gctl_get_ascii(req, "arg%d", i));
-		if (error) {
-			gctl_error(req, "unable to add %s to %s: %s.", gctl_get_ascii(req, "arg%d", i), mpname, strerror(error));
+		name2 = gctl_get_ascii(req, "arg%d", i);
+		fd = g_open(name2, 1);
+		if (fd < 0) {
+			fprintf(stderr, "Unable to open %s: %s.\n",
+			    name2, strerror(errno));
+			continue;
+		}
+		if (pread(fd, rsector, secsize, disksize - secsize) !=
+		    secsize) {
+			fprintf(stderr, "Unable to read metadata from %s: %s.\n",
+			    name2, strerror(errno));
+			g_close(fd);
 			continue;
 		}
-		mp_add(req);
+		g_close(fd);
+		if (memcmp(sector, rsector, secsize)) {
+			fprintf(stderr, "No metadata found on %s."
+			    " It is not a path of %s.\n",
+			    name2, name);
+		}
 	}
 }
 
@@ -247,13 +280,3 @@ mp_clear(struct gctl_req *req)
 	}
 }
 
-static void
-mp_add(struct gctl_req *req)
-{
-	const char *errstr;
-
-	errstr = gctl_issue(req);
-	if (errstr != NULL && errstr[0] != '\0') {
-		gctl_error(req, "%s", errstr);
-	}
-}

Modified: head/sbin/geom/class/multipath/gmultipath.8
==============================================================================
--- head/sbin/geom/class/multipath/gmultipath.8	Sat Nov 12 08:40:52 2011	(r227463)
+++ head/sbin/geom/class/multipath/gmultipath.8	Sat Nov 12 09:52:27 2011	(r227464)
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 26, 2007
+.Dd October 31, 2011
 .Dt GMULTIPATH 8
 .Os
 .Sh NAME
@@ -32,11 +32,48 @@
 .Nd "disk multipath control utility"
 .Sh SYNOPSIS
 .Nm
+.Cm create
+.Op Fl Av
+.Ar name
+.Ar prov ...
+.Nm
 .Cm label
-.Op Fl hv
+.Op Fl Av
 .Ar name
 .Ar prov ...
 .Nm
+.Cm add
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm remove
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm fail
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm restore
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm rotate
+.Op Fl v
+.Ar name
+.Nm
+.Cm getactive
+.Op Fl v
+.Ar name
+.Nm
+.Cm destroy
+.Op Fl v
+.Ar name
+.Nm
+.Cm stop
+.Op Fl v
+.Ar name
+.Nm
 .Cm clear
 .Op Fl v
 .Ar prov ...
@@ -53,27 +90,79 @@ The
 .Nm
 utility is used for device multipath configuration.
 .Pp
-Only automatic configuration is supported at the present time via the
-.Cm label
-command.
-This operation writes a label on the last sector of the underlying
-disk device with a contained name and UUID.
-The UUID guarantees uniqueness
-in a shared storage environment but is in general too cumbersome to use.
+The multipath device can be configured using two different methods:
+.Dq manual
+or
+.Dq automatic .
+When using the
+.Dq manual
+method, no metadata are stored on the devices, so the multipath
+device has to be configured by hand every time it is needed.
+Additional device paths also won't be detected automatically.
+The
+.Dq automatic
+method uses on-disk metadata to detect device and all it's paths.
+Metadata use the last sector of the underlying disk device and
+include device name and UUID.
+The UUID guarantees uniqueness in a shared storage environment
+but is in general too cumbersome to use.
 The name is what is exported via the device interface.
 .Pp
 The first argument to
 .Nm
 indicates an action to be performed:
 .Bl -tag -width ".Cm destroy"
+.It Cm create
+Create multipath device with
+.Dq manual
+method without writing any on-disk metadata.
+It is up to administrator, how to properly identify device paths.
+Kernel will only check that all given providers have same media and
+sector sizes.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
 .It Cm label
-Label the given underlying device with the specified
+Create multipath device with
+.Dq automatic
+method.
+Label the first given provider with on-disk metadata using the specified
 .Ar name .
-The kernel module
-.Pa geom_multipath.ko
-will be loaded if it is not loaded already.
+The rest of given providers will be retasted to detect these metadata.
+It reliably protects against specifying unrelated providers.
+Providers with no matching metadata detected will not be added to the device.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
+.It Cm add
+Add the given provider as a path to the given multipath device.
+Should normally be used only for devices created with
+.Dq manual
+method, unless you know what you are doing (you are sure that it is another
+device path, but tasting its metadata in regular
+.Dq automatic
+way is not possible).
+.It Cm remove
+Remove the given provider as a path from the given multipath device.
+If the last path removed, the multipath device will be destroyed.
+.It Cm fail
+Mark specified provider as a path of the specified multipath device as failed.
+If there are other paths present, new requests will be forwarded there.
+.It Cm restore
+Mark specified provider as a path of the specified multipath device as
+operational, allowing it to handle requests.
+.It Cm rotate
+Change the active provider/path in Active/Passive mode.
+.It Cm getactive
+Get the currently active provider(s)/path(s).
+.It Cm destroy
+Destroy the given multipath device clearing metadata.
+.It Cm stop
+Stop the given multipath device without clearing metadata.
 .It Cm clear
-Clear metadata on the given device.
+Clear metadata on the given provider.
 .It Cm list
 See
 .Xr geom 8 .
@@ -101,14 +190,15 @@ Debug level of the
 GEOM class.
 This can be set to 0 (default) or 1 to disable or enable various
 forms of chattiness.
+.It Va kern.geom.multipath.exclusive : No 1
+Open underlying providers exclusively, preventing individual paths access.
 .El
 .Sh EXIT STATUS
 Exit status is 0 on success, and 1 if the command fails.
 .Sh MULTIPATH ARCHITECTURE
 .Pp
-This is an active/passive
-multiple path architecture with no device knowledge or presumptions other
-than size matching built in.
+This is a multiple path architecture with no device knowledge or
+presumptions other than size matching built in.
 Therefore the user must exercise some care
 in selecting providers that do indeed represent multiple paths to the
 same underlying disk device.
@@ -133,15 +223,16 @@ of multiple pathnames refer to the same 
 system operator who will use tools and knowledge of their own storage
 subsystem to make the correct configuration selection.
 .Pp
-As an active/passive architecture, only one path has I/O moving on it
+There are Active/Passive and Active/Active operation modes supported.
+In Active/Passive mode only one path has I/O moving on it
 at any point in time.
 This I/O continues until an I/O is returned with
 a generic I/O error or a "Nonexistent Device" error.
-When this occurs,
-the active device is kicked out of the
-.Nm MULTIPATH
-GEOM class and the next in a list is selected, the failed I/O reissued
-and the system proceeds.
+When this occurs, that path is marked FAIL, the next path
+in a list is selected as active and the failed I/O reissued.
+In Active/Active mode all paths not marked FAIL may handle I/O same time.
+Requests are distributed between paths to equalize load.
+For capable devices it allows to utilize bandwidth of all paths.
 .Pp
 When new devices are added to the system the
 .Nm MULTIPATH
@@ -149,9 +240,9 @@ GEOM class is given an opportunity to ta
 If a new
 device has a
 .Nm MULTIPATH
-label, the device is used to either create a new
+on-disk metadata label, the device is used to either create a new
 .Nm MULTIPATH
-GEOM, or to attach to the end of the list of devices for an existing
+GEOM, or been added the list of paths for an existing
 .Nm MULTIPATH
 GEOM.
 .Pp
@@ -176,7 +267,7 @@ of an RSCN event from the Fabric Domain 
 a rescan to occur and cause the attachment and configuration of any
 (now) new devices to occur, causing the taste event described above.
 .Pp
-This means that this active/passive architecture is not a one-shot path
+This means that this multipath architecture is not a one-shot path
 failover, but can be considered to be steady state as long as failed
 paths are repaired (automatically or otherwise).
 .Pp
@@ -184,7 +275,7 @@ Automatic rescanning is not a requiremen
 Nor is Fibre Channel.
 The
 same failover mechanisms work equally well for traditional "Parallel"
-SCSI but require manual intervention with
+SCSI but may require manual intervention with
 .Xr camcontrol 8
 to cause the reattachment of repaired device links.
 .Sh EXAMPLES
@@ -226,9 +317,9 @@ mount /dev/multipath/FREDa /mnt....
 .Pp
 The resultant console output looks something like:
 .Bd -literal -offset indent
-GEOM_MULTIPATH: adding da0 to Fred/b631385f-c61c-11db-b884-0011116ae789
-GEOM_MULTIPATH: da0 now active path in Fred
-GEOM_MULTIPATH: adding da2 to Fred/b631385f-c61c-11db-b884-0011116ae789
+GEOM_MULTIPATH: da0 added to FRED
+GEOM_MULTIPATH: da0 is now active path in FRED
+GEOM_MULTIPATH: da2 added to FRED
 .Ed
 .Sh SEE ALSO
 .Xr geom 4 ,
@@ -240,24 +331,6 @@ GEOM_MULTIPATH: adding da2 to Fred/b6313
 .Xr mount 8 ,
 .Xr newfs 8 ,
 .Xr sysctl 8
-.Sh BUGS
-The
-.Nm
-should allow for a manual method of pairing disks.
-.Pp
-There is currently no way for
-.Pa geom_multipath.ko
-to distinguish between various label instances of the same provider.
-That
-is devices such as
-.Ar da0
-and
-.Ar da0c
-can be tasted and instantiated as multiple paths for the same device.
-Technically, this is correct, but pretty useless.
-This will be fixed soon
-(I hope), but to avoid this it is a good idea to destroy any label on
-the disk object prior to labelling it with
-.Nm .
 .Sh AUTHOR
 .An Matthew Jacob Aq mjacob@FreeBSD.org
+.An Alexander Motin Aq mav@FreeBSD.org

Modified: head/sys/geom/multipath/g_multipath.c
==============================================================================
--- head/sys/geom/multipath/g_multipath.c	Sat Nov 12 08:40:52 2011	(r227463)
+++ head/sys/geom/multipath/g_multipath.c	Sat Nov 12 09:52:27 2011	(r227464)
@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2011 Alexander Motin <mav@FreeBSD.org>
  * Copyright (c) 2006-2007 Matthew Jacob <mjacob@FreeBSD.org>
  * All rights reserved.
  *
@@ -53,6 +54,9 @@ static SYSCTL_NODE(_kern_geom, OID_AUTO,
 static u_int g_multipath_debug = 0;
 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW,
     &g_multipath_debug, 0, "Debug level");
+static u_int g_multipath_exclusive = 1;
+SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
+    &g_multipath_exclusive, 0, "Exclusively open providers");
 
 static enum {
 	GKT_NIL,
@@ -79,6 +83,7 @@ static g_taste_t g_multipath_taste;
 static g_ctl_req_t g_multipath_config;
 static g_init_t g_multipath_init;
 static g_fini_t g_multipath_fini;
+static g_dumpconf_t g_multipath_dumpconf;
 
 struct g_class g_multipath_class = {
 	.name		= G_MULTIPATH_CLASS_NAME,
@@ -90,35 +95,144 @@ struct g_class g_multipath_class = {
 	.fini		= g_multipath_fini
 };
 
-#define	MP_BAD		0x1
-#define	MP_POSTED	0x2
+#define	MP_FAIL		0x00000001
+#define	MP_LOST		0x00000002
+#define	MP_NEW		0x00000004
+#define	MP_POSTED	0x00000008
+#define	MP_BAD		(MP_FAIL | MP_LOST | MP_NEW)
+#define MP_IDLE		0x00000010
+#define MP_IDLE_MASK	0xfffffff0
+
+static int
+g_multipath_good(struct g_geom *gp)
+{
+	struct g_consumer *cp;
+	int n = 0;
+
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if ((cp->index & MP_BAD) == 0)
+			n++;
+	}
+	return (n);
+}
+
+static void
+g_multipath_fault(struct g_consumer *cp, int cause)
+{
+	struct g_multipath_softc *sc;
+	struct g_consumer *lcp;
+	struct g_geom *gp;
+
+	gp = cp->geom;
+	sc = gp->softc;
+	cp->index |= cause;
+	if (g_multipath_good(gp) == 0 && sc->sc_ndisks > 0) {
+		LIST_FOREACH(lcp, &gp->consumer, consumer) {
+			if (lcp->provider == NULL ||
+			    (lcp->index & (MP_LOST | MP_NEW)))
+				continue;
+			if (sc->sc_ndisks > 1 && lcp == cp)
+				continue;
+			printf("GEOM_MULTIPATH: "
+			    "all paths in %s were marked FAIL, restore %s\n",
+			    sc->sc_name, lcp->provider->name);
+			lcp->index &= ~MP_FAIL;
+		}
+	}
+	if (cp != sc->sc_active)
+		return;
+	sc->sc_active = NULL;
+	LIST_FOREACH(lcp, &gp->consumer, consumer) {
+		if ((lcp->index & MP_BAD) == 0) {
+			sc->sc_active = lcp;
+			break;
+		}
+	}
+	if (sc->sc_active == NULL) {
+		printf("GEOM_MULTIPATH: out of providers for %s\n",
+		    sc->sc_name);
+	} else if (!sc->sc_active_active) {
+		printf("GEOM_MULTIPATH: %s is now active path in %s\n",
+		    sc->sc_active->provider->name, sc->sc_name);
+	}
+}
+
+static struct g_consumer *
+g_multipath_choose(struct g_geom *gp)
+{
+	struct g_multipath_softc *sc;
+	struct g_consumer *best, *cp;
+
+	sc = gp->softc;
+	if (!sc->sc_active_active)
+		return (sc->sc_active);
+	best = NULL;
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if (cp->index & MP_BAD)
+			continue;
+		cp->index += MP_IDLE;
+		if (best == NULL || cp->private < best->private ||
+		    (cp->private == best->private && cp->index > best->index))
+			best = cp;
+	}
+	if (best != NULL)
+		best->index &= ~MP_IDLE_MASK;
+	return (best);
+}
 
 static void
 g_mpd(void *arg, int flags __unused)
 {
+	struct g_geom *gp;
+	struct g_multipath_softc *sc;
 	struct g_consumer *cp;
+	int w;
 
 	g_topology_assert();
 	cp = arg;
-	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
+	gp = cp->geom;
+	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) {
+		w = cp->acw;
 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		if (w > 0 && cp->provider != NULL &&
+		    (cp->provider->geom->flags & G_GEOM_WITHER) == 0) {
+			g_post_event(g_mpd, cp, M_WAITOK, NULL);
+			return;
+		}
+	}
+	sc = gp->softc;
+	mtx_lock(&sc->sc_mtx);
 	if (cp->provider) {
 		printf("GEOM_MULTIPATH: %s removed from %s\n",
-		    cp->provider->name, cp->geom->name);
+		    cp->provider->name, gp->name);
 		g_detach(cp);
 	}
 	g_destroy_consumer(cp);
+	mtx_unlock(&sc->sc_mtx);
+	if (LIST_EMPTY(&gp->consumer))
+		g_multipath_destroy(gp);
 }
 
 static void
 g_multipath_orphan(struct g_consumer *cp)
 {
-	if ((cp->index & MP_POSTED) == 0) {
+	struct g_multipath_softc *sc;
+	uintptr_t *cnt;
+
+	g_topology_assert();
+	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
+	    cp->provider->name, cp->geom->name);
+	sc = cp->geom->softc;
+	cnt = (uintptr_t *)&cp->private;
+	mtx_lock(&sc->sc_mtx);
+	sc->sc_ndisks--;
+	g_multipath_fault(cp, MP_LOST);
+	if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
 		cp->index |= MP_POSTED;
-		printf("GEOM_MULTIPATH: %s orphaned in %s\n",
-		    cp->provider->name, cp->geom->name);
+		mtx_unlock(&sc->sc_mtx);
 		g_mpd(cp, 0);
-	}
+	} else
+		mtx_unlock(&sc->sc_mtx);
 }
 
 static void
@@ -128,20 +242,29 @@ g_multipath_start(struct bio *bp)
 	struct g_geom *gp;
 	struct g_consumer *cp;
 	struct bio *cbp;
+	uintptr_t *cnt;
 
 	gp = bp->bio_to->geom;
 	sc = gp->softc;
 	KASSERT(sc != NULL, ("NULL sc"));
-	cp = sc->cp_active;
-	if (cp == NULL) {
-		g_io_deliver(bp, ENXIO);
-		return;
-	}
 	cbp = g_clone_bio(bp);
 	if (cbp == NULL) {
 		g_io_deliver(bp, ENOMEM);
 		return;
 	}
+	mtx_lock(&sc->sc_mtx);
+	cp = g_multipath_choose(gp);
+	if (cp == NULL) {
+		mtx_unlock(&sc->sc_mtx);
+		g_destroy_bio(cbp);
+		g_io_deliver(bp, ENXIO);
+		return;
+	}
+	if ((uintptr_t)bp->bio_driver1 < sc->sc_ndisks)
+		bp->bio_driver1 = (void *)(uintptr_t)sc->sc_ndisks;
+	cnt = (uintptr_t *)&cp->private;
+	(*cnt)++;
+	mtx_unlock(&sc->sc_mtx);
 	cbp->bio_done = g_multipath_done;
 	g_io_request(cbp, cp);
 }
@@ -149,12 +272,27 @@ g_multipath_start(struct bio *bp)
 static void
 g_multipath_done(struct bio *bp)
 {
+	struct g_multipath_softc *sc;
+	struct g_consumer *cp;
+	uintptr_t *cnt;
+
 	if (bp->bio_error == ENXIO || bp->bio_error == EIO) {
 		mtx_lock(&gmtbq_mtx);
 		bioq_insert_tail(&gmtbq, bp);
-		wakeup(&g_multipath_kt_state);
 		mtx_unlock(&gmtbq_mtx);
+		wakeup(&g_multipath_kt_state);
 	} else {
+		cp = bp->bio_from;
+		sc = cp->geom->softc;
+		cnt = (uintptr_t *)&cp->private;
+		mtx_lock(&sc->sc_mtx);
+		(*cnt)--;
+		if (*cnt == 0 && (cp->index & MP_LOST)) {
+			cp->index |= MP_POSTED;
+			mtx_unlock(&sc->sc_mtx);
+			g_post_event(g_mpd, cp, M_WAITOK, NULL);
+		} else
+			mtx_unlock(&sc->sc_mtx);
 		g_std_done(bp);
 	}
 }
@@ -167,6 +305,7 @@ g_multipath_done_error(struct bio *bp)
 	struct g_multipath_softc *sc;
 	struct g_consumer *cp;
 	struct g_provider *pp;
+	uintptr_t *cnt;
 
 	/*
 	 * If we had a failure, we have to check first to see
@@ -176,47 +315,31 @@ g_multipath_done_error(struct bio *bp)
 	 * to the next available consumer.
 	 */
 
-	g_topology_lock();
 	pbp = bp->bio_parent;
 	gp = pbp->bio_to->geom;
 	sc = gp->softc;
 	cp = bp->bio_from;
 	pp = cp->provider;
+	cnt = (uintptr_t *)&cp->private;
 
-	cp->index |= MP_BAD;
-	if (cp->nend == cp->nstart && pp->nend == pp->nstart) {
+	mtx_lock(&sc->sc_mtx);
+	printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
+	    bp->bio_error, pp->name, sc->sc_name);
+	g_multipath_fault(cp, MP_FAIL);
+	(*cnt)--;
+	if (*cnt == 0 && (cp->index & (MP_LOST | MP_POSTED)) == MP_LOST) {
 		cp->index |= MP_POSTED;
-		g_post_event(g_mpd, cp, M_NOWAIT, NULL);
-	}
-	if (cp == sc->cp_active) {
-		struct g_consumer *lcp;
-		printf("GEOM_MULTIPATH: %s failed in %s\n",
-		    pp->name, sc->sc_name);
-		sc->cp_active = NULL;
-		LIST_FOREACH(lcp, &gp->consumer, consumer) {
-			if ((lcp->index & MP_BAD) == 0) {
-				sc->cp_active = lcp;
-				break;
-			}
-		}
-		if (sc->cp_active == NULL || sc->cp_active->provider == NULL) {
-			printf("GEOM_MULTIPATH: out of providers for %s\n",
-			    sc->sc_name);
-			g_topology_unlock();
-			return;
-		} else {
-			printf("GEOM_MULTIPATH: %s now active path in %s\n",
-			    sc->cp_active->provider->name, sc->sc_name);
-		}
-	}
-	g_topology_unlock();
+		mtx_unlock(&sc->sc_mtx);
+		g_post_event(g_mpd, cp, M_WAITOK, NULL);
+	} else
+		mtx_unlock(&sc->sc_mtx);
 
 	/*
 	 * If we can fruitfully restart the I/O, do so.
 	 */
-	if (sc->cp_active) {
+	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
+		pbp->bio_inbed++;
 		g_destroy_bio(bp);
-		pbp->bio_children--;
 		g_multipath_start(pbp);
 	} else {
 		g_std_done(bp);
@@ -254,6 +377,7 @@ g_multipath_access(struct g_provider *pp
 {
 	struct g_geom *gp;
 	struct g_consumer *cp, *badcp = NULL;
+	struct g_multipath_softc *sc;
 	int error;
 
 	gp = pp->geom;
@@ -265,6 +389,10 @@ g_multipath_access(struct g_provider *pp
 			goto fail;
 		}
 	}
+	sc = gp->softc;
+	sc->sc_opened += dr + dw + de;
+	if (sc->sc_stopping && sc->sc_opened == 0)
+		g_multipath_destroy(gp);
 	return (0);
 
 fail:
@@ -286,6 +414,9 @@ g_multipath_create(struct g_class *mp, s
 	g_topology_assert();
 
 	LIST_FOREACH(gp, &mp->geom, geom) {
+		sc = gp->softc;
+		if (sc == NULL || sc->sc_stopping)
+			continue;
 		if (strcmp(gp->name, md->md_name) == 0) {
 			printf("GEOM_MULTIPATH: name %s already exists\n",
 			    md->md_name);
@@ -295,19 +426,25 @@ g_multipath_create(struct g_class *mp, s
 
 	gp = g_new_geomf(mp, md->md_name);
 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
+	mtx_init(&sc->sc_mtx, "multipath", NULL, MTX_DEF);
+	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
+	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
+	sc->sc_active_active = md->md_active_active;
 	gp->softc = sc;
 	gp->start = g_multipath_start;
 	gp->orphan = g_multipath_orphan;
 	gp->access = g_multipath_access;
-	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
-	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
+	gp->dumpconf = g_multipath_dumpconf;
 
 	pp = g_new_providerf(gp, "multipath/%s", md->md_name);
-	/* limit the provider to not have it stomp on metadata */
-	pp->mediasize = md->md_size - md->md_sectorsize;
-	pp->sectorsize = md->md_sectorsize;
-	sc->pp = pp;
+	if (md->md_size != 0) {
+		pp->mediasize = md->md_size -
+		    ((md->md_uuid[0] != 0) ? md->md_sectorsize : 0);
+		pp->sectorsize = md->md_sectorsize;
+	}
+	sc->sc_pp = pp;
 	g_error_provider(pp, 0);
+	printf("GEOM_MULTIPATH: %s created\n", gp->name);
 	return (gp);
 }
 
@@ -316,7 +453,7 @@ g_multipath_add_disk(struct g_geom *gp, 
 {
 	struct g_multipath_softc *sc;
 	struct g_consumer *cp, *nxtcp;
-	int error;
+	int error, acr, acw, ace;
 
 	g_topology_assert();
 
@@ -337,6 +474,8 @@ g_multipath_add_disk(struct g_geom *gp, 
 	}
 	nxtcp = LIST_FIRST(&gp->consumer);
 	cp = g_new_consumer(gp);
+	cp->private = NULL;
+	cp->index = MP_NEW;
 	error = g_attach(cp, pp);
 	if (error != 0) {
 		printf("GEOM_MULTIPATH: cannot attach %s to %s",
@@ -344,29 +483,51 @@ g_multipath_add_disk(struct g_geom *gp, 
 		g_destroy_consumer(cp);
 		return (error);
 	}
-	cp->private = sc;
-	cp->index = 0;
 
 	/*
 	 * Set access permissions on new consumer to match other consumers
 	 */
-	if (nxtcp && (nxtcp->acr + nxtcp->acw +  nxtcp->ace)) {
-		error = g_access(cp, nxtcp->acr, nxtcp->acw, nxtcp->ace);
-		if (error) {
-			printf("GEOM_MULTIPATH: cannot set access in "
-			    "attaching %s to %s/%s (%d)\n",
-			    pp->name, sc->sc_name, sc->sc_uuid, error);
-			g_detach(cp);
-			g_destroy_consumer(cp);
-			return (error);
-		}
+	if (sc->sc_pp) {
+		acr = sc->sc_pp->acr;
+		acw = sc->sc_pp->acw;
+		ace = sc->sc_pp->ace;
+	} else
+		acr = acw = ace = 0;
+	if (g_multipath_exclusive) {
+		acr++;
+		acw++;
+		ace++;
+	}
+	error = g_access(cp, acr, acw, ace);
+	if (error) {
+		printf("GEOM_MULTIPATH: cannot set access in "
+		    "attaching %s to %s (%d)\n",
+		    pp->name, sc->sc_name, error);
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		return (error);
 	}
-	printf("GEOM_MULTIPATH: adding %s to %s/%s\n",
-	    pp->name, sc->sc_name, sc->sc_uuid);
-	if (sc->cp_active == NULL) {
-		sc->cp_active = cp;
-		printf("GEOM_MULTIPATH: %s now active path in %s\n",
-		    pp->name, sc->sc_name);
+	if (sc->sc_pp != NULL && sc->sc_pp->mediasize == 0) {
+		sc->sc_pp->mediasize = pp->mediasize -
+		    ((sc->sc_uuid[0] != 0) ? pp->sectorsize : 0);
+		sc->sc_pp->sectorsize = pp->sectorsize;
+	}
+	if (sc->sc_pp != NULL &&
+	    sc->sc_pp->stripesize == 0 && sc->sc_pp->stripeoffset == 0) {
+		sc->sc_pp->stripesize = pp->stripesize;
+		sc->sc_pp->stripeoffset = pp->stripeoffset;
+	}
+	mtx_lock(&sc->sc_mtx);
+	cp->index = 0;
+	sc->sc_ndisks++;
+	mtx_unlock(&sc->sc_mtx);
+	printf("GEOM_MULTIPATH: %s added to %s\n",
+	    pp->name, sc->sc_name);
+	if (sc->sc_active == NULL) {
+		sc->sc_active = cp;
+		if (!sc->sc_active_active)
+			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
+			    pp->name, sc->sc_name);
 	}
 	return (0);
 }
@@ -374,17 +535,41 @@ g_multipath_add_disk(struct g_geom *gp, 
 static int
 g_multipath_destroy(struct g_geom *gp)
 {
-	struct g_provider *pp;
+	struct g_multipath_softc *sc;
+	struct g_consumer *cp, *cp1;
 
 	g_topology_assert();
 	if (gp->softc == NULL)
 		return (ENXIO);
-	pp = LIST_FIRST(&gp->provider);
-	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0))
-		return (EBUSY);
-	printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
+	sc = gp->softc;
+	if (!sc->sc_stopping) {
+		printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
+		sc->sc_stopping = 1;
+	}
+	if (sc->sc_opened != 0) {
+		if (sc->sc_pp != NULL) {
+			g_wither_provider(sc->sc_pp, ENXIO);
+			sc->sc_pp = NULL;
+		}
+		return (EINPROGRESS);
+	}
+	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
+		mtx_lock(&sc->sc_mtx);
+		if ((cp->index & MP_POSTED) == 0) {
+			cp->index |= MP_POSTED;
+			mtx_unlock(&sc->sc_mtx);
+			g_mpd(cp, 0);
+			if (cp1 == NULL)
+				return(0);	/* Recursion happened. */
+		} else
+			mtx_unlock(&sc->sc_mtx);
+	}
+	if (!LIST_EMPTY(&gp->consumer))
+		return (EINPROGRESS);
+	mtx_destroy(&sc->sc_mtx);
 	g_free(gp->softc);
 	gp->softc = NULL;
+	printf("GEOM_MULTIPATH: %s destroyed\n", gp->name);
 	g_wither_geom(gp, ENXIO);
 	return (0);
 }

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201111120952.pAC9qSUL036029>