Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 15 Apr 2011 23:07:49 +0000 (UTC)
From:      Rick Macklem <rmacklem@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r220683 - in head/sys/fs: nfs nfsclient
Message-ID:  <201104152307.p3FN7nBe084937@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: rmacklem
Date: Fri Apr 15 23:07:48 2011
New Revision: 220683
URL: http://svn.freebsd.org/changeset/base/220683

Log:
  Change the experimental NFS client so that it creates nfsiod
  threads in the same manner as the regular NFS client after
  r214026 was committed. This resolves the lors fixed by r214026
  and its predecessors for the regular client.
  
  Reviewed by:	jhb
  MFC after:	2 weeks

Modified:
  head/sys/fs/nfs/nfs.h
  head/sys/fs/nfsclient/nfs.h
  head/sys/fs/nfsclient/nfs_clbio.c
  head/sys/fs/nfsclient/nfs_clnfsiod.c
  head/sys/fs/nfsclient/nfs_clsubs.c

Modified: head/sys/fs/nfs/nfs.h
==============================================================================
--- head/sys/fs/nfs/nfs.h	Fri Apr 15 21:47:10 2011	(r220682)
+++ head/sys/fs/nfs/nfs.h	Fri Apr 15 23:07:48 2011	(r220683)
@@ -70,8 +70,9 @@
 #define	NFS_WSIZE	8192		/* Def. write data size <= 8192 */
 #define	NFS_RSIZE	8192		/* Def. read data size <= 8192 */
 #define	NFS_READDIRSIZE	8192		/* Def. readdir size */
-#define	NFS_DEFRAHEAD	0		/* Def. read ahead # blocks */
-#define	NFS_MAXRAHEAD	32		/* Max. read ahead # blocks */
+#define	NFS_DEFRAHEAD	1		/* Def. read ahead # blocks */
+#define	NFS_MAXRAHEAD	16		/* Max. read ahead # blocks */
+#define	NFS_MAXASYNCDAEMON 	64	/* Max. number async_daemons runnable */
 #define	NFS_MAXUIDHASH	64		/* Max. # of hashed uid entries/mp */
 #ifndef	NFSRV_LEASE
 #define	NFSRV_LEASE		120	/* Lease time in seconds for V4 */

Modified: head/sys/fs/nfsclient/nfs.h
==============================================================================
--- head/sys/fs/nfsclient/nfs.h	Fri Apr 15 21:47:10 2011	(r220682)
+++ head/sys/fs/nfsclient/nfs.h	Fri Apr 15 23:07:48 2011	(r220683)
@@ -102,7 +102,8 @@ int ncl_fsinfo(struct nfsmount *, struct
 int ncl_init(struct vfsconf *);
 int ncl_uninit(struct vfsconf *);
 int ncl_mountroot(struct mount *);
-int ncl_nfsiodnew(int);
+void	ncl_nfsiodnew(void);
+void	ncl_nfsiodnew_tq(__unused void *, int);
 
 #endif	/* _KERNEL */
 

Modified: head/sys/fs/nfsclient/nfs_clbio.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clbio.c	Fri Apr 15 21:47:10 2011	(r220682)
+++ head/sys/fs/nfsclient/nfs_clbio.c	Fri Apr 15 23:07:48 2011	(r220683)
@@ -60,8 +60,8 @@ extern int newnfs_directio_allow_mmap;
 extern struct nfsstats newnfsstats;
 extern struct mtx ncl_iod_mutex;
 extern int ncl_numasync;
-extern enum nfsiod_state ncl_iodwant[NFS_MAXRAHEAD];
-extern struct nfsmount *ncl_iodmount[NFS_MAXRAHEAD];
+extern enum nfsiod_state ncl_iodwant[NFS_MAXASYNCDAEMON];
+extern struct nfsmount *ncl_iodmount[NFS_MAXASYNCDAEMON];
 extern int newnfs_directio_enable;
 
 int ncl_pbuf_freecnt = -1;	/* start out unlimited */
@@ -1354,15 +1354,6 @@ ncl_asyncio(struct nfsmount *nmp, struct
 	int error, error2;
 
 	/*
-	 * Unless iothreadcnt is set > 0, don't bother with async I/O
-	 * threads. For LAN environments, they don't buy any significant
-	 * performance improvement that you can't get with large block
-	 * sizes.
-	 */
-	if (nmp->nm_readahead == 0)
-		return (EPERM);
-
-	/*
 	 * Commits are usually short and sweet so lets save some cpu and
 	 * leave the async daemons for more important rpc's (such as reads
 	 * and writes).
@@ -1390,13 +1381,9 @@ again:
 	/*
 	 * Try to create one if none are free.
 	 */
-	if (!gotiod) {
-		iod = ncl_nfsiodnew(1);
-		if (iod != -1)
-			gotiod = TRUE;
-	}
-
-	if (gotiod) {
+	if (!gotiod)
+		ncl_nfsiodnew();
+	else {
 		/*
 		 * Found one, so wake it up and tell it which
 		 * mount to process.
@@ -1453,11 +1440,7 @@ again:
 			 * We might have lost our iod while sleeping,
 			 * so check and loop if nescessary.
 			 */
-			if (nmp->nm_bufqiods == 0) {
-				NFS_DPF(ASYNCIO,
-					("ncl_asyncio: no iods after mount %p queue was drained, looping\n", nmp));
-				goto again;
-			}
+			goto again;
 		}
 
 		/* We might have lost our nfsiod */

Modified: head/sys/fs/nfsclient/nfs_clnfsiod.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clnfsiod.c	Fri Apr 15 21:47:10 2011	(r220682)
+++ head/sys/fs/nfsclient/nfs_clnfsiod.c	Fri Apr 15 23:07:48 2011	(r220683)
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/lockf.h>
 #include <sys/mutex.h>
+#include <sys/taskqueue.h>
 
 #include <netinet/in.h>
 #include <netinet/tcp.h>
@@ -68,28 +69,32 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfs.h>
 #include <fs/nfsclient/nfsnode.h>
 
-extern struct mtx ncl_iod_mutex;
+extern struct mtx	ncl_iod_mutex;
+extern struct task	ncl_nfsiodnew_task;
 
 int ncl_numasync;
-enum nfsiod_state ncl_iodwant[NFS_MAXRAHEAD];
-struct nfsmount *ncl_iodmount[NFS_MAXRAHEAD];
+enum nfsiod_state ncl_iodwant[NFS_MAXASYNCDAEMON];
+struct nfsmount *ncl_iodmount[NFS_MAXASYNCDAEMON];
 
 static void	nfssvc_iod(void *);
 
-static int nfs_asyncdaemon[NFS_MAXRAHEAD];
+static int nfs_asyncdaemon[NFS_MAXASYNCDAEMON];
 
 SYSCTL_DECL(_vfs_newnfs);
 
 /* Maximum number of seconds a nfsiod kthread will sleep before exiting */
-static unsigned int ncl_iodmaxidle = 120;
-SYSCTL_UINT(_vfs_newnfs, OID_AUTO, iodmaxidle, CTLFLAG_RW, &ncl_iodmaxidle, 0, "");
+static unsigned int nfs_iodmaxidle = 120;
+SYSCTL_UINT(_vfs_newnfs, OID_AUTO, iodmaxidle, CTLFLAG_RW, &nfs_iodmaxidle, 0,
+    "Max number of seconds an nfsiod kthread will sleep before exiting");
 
 /* Maximum number of nfsiod kthreads */
-unsigned int ncl_iodmax = NFS_MAXRAHEAD;
+unsigned int ncl_iodmax = 20;
 
 /* Minimum number of nfsiod kthreads to keep as spares */
 static unsigned int nfs_iodmin = 0;
 
+static int nfs_nfsiodnew_sync(void);
+
 static int
 sysctl_iodmin(SYSCTL_HANDLER_ARGS)
 {
@@ -113,14 +118,14 @@ sysctl_iodmin(SYSCTL_HANDLER_ARGS)
 	 * than the new minimum, create some more.
 	 */
 	for (i = nfs_iodmin - ncl_numasync; i > 0; i--)
-		ncl_nfsiodnew(0);
+		nfs_nfsiodnew_sync();
 out:
 	mtx_unlock(&ncl_iod_mutex);	
 	return (0);
 }
 SYSCTL_PROC(_vfs_newnfs, OID_AUTO, iodmin, CTLTYPE_UINT | CTLFLAG_RW, 0,
-    sizeof (nfs_iodmin), sysctl_iodmin, "IU", "");
-
+    sizeof (nfs_iodmin), sysctl_iodmin, "IU",
+    "Min number of nfsiod kthreads to keep as spares");
 
 static int
 sysctl_iodmax(SYSCTL_HANDLER_ARGS)
@@ -132,7 +137,7 @@ sysctl_iodmax(SYSCTL_HANDLER_ARGS)
 	error = sysctl_handle_int(oidp, &newmax, 0, req);
 	if (error || (req->newptr == NULL))
 		return (error);
-	if (newmax > NFS_MAXRAHEAD)
+	if (newmax > NFS_MAXASYNCDAEMON)
 		return (EINVAL);
 	mtx_lock(&ncl_iod_mutex);
 	ncl_iodmax = newmax;
@@ -155,64 +160,79 @@ out:
 	return (0);
 }
 SYSCTL_PROC(_vfs_newnfs, OID_AUTO, iodmax, CTLTYPE_UINT | CTLFLAG_RW, 0,
-    sizeof (ncl_iodmax), sysctl_iodmax, "IU", "");
+    sizeof (ncl_iodmax), sysctl_iodmax, "IU",
+    "Max number of nfsiod kthreads");
 
-int
-ncl_nfsiodnew(int set_iodwant)
+static int
+nfs_nfsiodnew_sync(void)
 {
 	int error, i;
-	int newiod;
 
-	if (ncl_numasync >= ncl_iodmax)
-		return (-1);
-	newiod = -1;
-	for (i = 0; i < ncl_iodmax; i++)
+	mtx_assert(&ncl_iod_mutex, MA_OWNED);
+	for (i = 0; i < ncl_iodmax; i++) {
 		if (nfs_asyncdaemon[i] == 0) {
-			nfs_asyncdaemon[i]++;
-			newiod = i;
+			nfs_asyncdaemon[i] = 1;
 			break;
 		}
-	if (newiod == -1)
-		return (-1);
-	if (set_iodwant > 0)
-		ncl_iodwant[i] = NFSIOD_CREATED_FOR_NFS_ASYNCIO;
+	}
+	if (i == ncl_iodmax)
+		return (0);
 	mtx_unlock(&ncl_iod_mutex);
-	error = kproc_create(nfssvc_iod, nfs_asyncdaemon + i, NULL, RFHIGHPID,
-	    0, "nfsiod %d", newiod);
+	error = kproc_create(nfssvc_iod, nfs_asyncdaemon + i, NULL,
+	    RFHIGHPID, 0, "newnfs %d", i);
+	mtx_lock(&ncl_iod_mutex);
+	if (error == 0) {
+		ncl_numasync++;
+		ncl_iodwant[i] = NFSIOD_AVAILABLE;
+	} else
+		nfs_asyncdaemon[i] = 0;
+	return (error);
+}
+
+void
+ncl_nfsiodnew_tq(__unused void *arg, int pending)
+{
+
 	mtx_lock(&ncl_iod_mutex);
-	if (error) {
-		if (set_iodwant > 0)
-			ncl_iodwant[i] = NFSIOD_NOT_AVAILABLE;
-		return (-1);
+	while (pending > 0) {
+		pending--;
+		nfs_nfsiodnew_sync();
 	}
-	ncl_numasync++;
-	return (newiod);
+	mtx_unlock(&ncl_iod_mutex);
+}
+
+void
+ncl_nfsiodnew(void)
+{
+
+	mtx_assert(&ncl_iod_mutex, MA_OWNED);
+	taskqueue_enqueue(taskqueue_thread, &ncl_nfsiodnew_task);
 }
 
 static void
 nfsiod_setup(void *dummy)
 {
-	int i;
 	int error;
 
 	TUNABLE_INT_FETCH("vfs.newnfs.iodmin", &nfs_iodmin);
 	nfscl_init();
 	mtx_lock(&ncl_iod_mutex);
 	/* Silently limit the start number of nfsiod's */
-	if (nfs_iodmin > NFS_MAXRAHEAD)
-		nfs_iodmin = NFS_MAXRAHEAD;
+	if (nfs_iodmin > NFS_MAXASYNCDAEMON)
+		nfs_iodmin = NFS_MAXASYNCDAEMON;
 
-	for (i = 0; i < nfs_iodmin; i++) {
-		error = ncl_nfsiodnew(0);
+	while (ncl_numasync < nfs_iodmin) {
+		error = nfs_nfsiodnew_sync();
 		if (error == -1)
-			panic("newnfsiod_setup: ncl_nfsiodnew failed");
+			panic("nfsiod_setup: nfs_nfsiodnew failed");
 	}
 	mtx_unlock(&ncl_iod_mutex);
 }
 SYSINIT(newnfsiod, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, nfsiod_setup, NULL);
 
 static int nfs_defect = 0;
-SYSCTL_INT(_vfs_newnfs, OID_AUTO, defect, CTLFLAG_RW, &nfs_defect, 0, "");
+SYSCTL_INT(_vfs_newnfs, OID_AUTO, defect, CTLFLAG_RW, &nfs_defect, 0,
+    "Allow nfsiods to migrate serving different mounts");
 
 /*
  * Asynchronous I/O daemons for client nfs.
@@ -245,7 +265,7 @@ nfssvc_iod(void *instance)
 		/*
 		 * Always keep at least nfs_iodmin kthreads.
 		 */
-		timo = (myiod < nfs_iodmin) ? 0 : ncl_iodmaxidle * hz;
+		timo = (myiod < nfs_iodmin) ? 0 : nfs_iodmaxidle * hz;
 		error = msleep(&ncl_iodwant[myiod], &ncl_iod_mutex, PWAIT | PCATCH,
 		    "-", timo);
 		if (error) {
@@ -263,7 +283,6 @@ nfssvc_iod(void *instance)
 	    if (error)
 		    break;
 	    while ((bp = TAILQ_FIRST(&nmp->nm_bufq)) != NULL) {
-		    
 		/* Take one off the front of the list */
 		TAILQ_REMOVE(&nmp->nm_bufq, bp, b_freelist);
 		nmp->nm_bufqlen--;

Modified: head/sys/fs/nfsclient/nfs_clsubs.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clsubs.c	Fri Apr 15 21:47:10 2011	(r220682)
+++ head/sys/fs/nfsclient/nfs_clsubs.c	Fri Apr 15 23:07:48 2011	(r220683)
@@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysent.h>
 #include <sys/syscall.h>
 #include <sys/sysproto.h>
+#include <sys/taskqueue.h>
 
 #include <vm/vm.h>
 #include <vm/vm_object.h>
@@ -77,12 +78,14 @@ __FBSDID("$FreeBSD$");
 #include <machine/stdarg.h>
 
 extern struct mtx ncl_iod_mutex;
-extern enum nfsiod_state ncl_iodwant[NFS_MAXRAHEAD];
-extern struct nfsmount *ncl_iodmount[NFS_MAXRAHEAD];
+extern enum nfsiod_state ncl_iodwant[NFS_MAXASYNCDAEMON];
+extern struct nfsmount *ncl_iodmount[NFS_MAXASYNCDAEMON];
 extern int ncl_numasync;
 extern unsigned int ncl_iodmax;
 extern struct nfsstats newnfsstats;
 
+struct task	ncl_nfsiodnew_task;
+
 int
 ncl_uninit(struct vfsconf *vfsp)
 {
@@ -393,10 +396,11 @@ ncl_init(struct vfsconf *vfsp)
 	int i;
 
 	/* Ensure async daemons disabled */
-	for (i = 0; i < NFS_MAXRAHEAD; i++) {
+	for (i = 0; i < NFS_MAXASYNCDAEMON; i++) {
 		ncl_iodwant[i] = NFSIOD_NOT_AVAILABLE;
 		ncl_iodmount[i] = NULL;
 	}
+	TASK_INIT(&ncl_nfsiodnew_task, 0, ncl_nfsiodnew_tq, NULL);
 	ncl_nhinit();			/* Init the nfsnode table */
 
 	return (0);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201104152307.p3FN7nBe084937>