Date: Mon, 16 Oct 2017 23:43:18 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r324678 - projects/pnfs-planb-server/sys/fs/nfsserver Message-ID: <201710162343.v9GNhIdh009490@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Mon Oct 16 23:43:18 2017 New Revision: 324678 URL: https://svnweb.freebsd.org/changeset/base/324678 Log: Change the pNFS server to use the taskqueue code (common with the client) to do concurrent I/O operations to the mirrored DSs, instead of using kproc_create()/kproc_exit() for every RPC. Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Oct 16 23:40:24 2017 (r324677) +++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Oct 16 23:43:18 2017 (r324678) @@ -72,6 +72,7 @@ extern struct mtx nfsrv_dsrmlock_mtx; extern struct mtx nfsrv_dwrpclock_mtx; extern struct mtx nfsrv_dsrpclock_mtx; extern struct mtx nfsrv_darpclock_mtx; +extern int nfs_pnfsiothreads; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; NFSSTATESPINLOCK; @@ -127,6 +128,8 @@ static int nfsrv_putfhname(fhandle_t *, char *); static int nfsrv_pnfslookupds(struct vnode *, struct pnfsdsfile *, struct vnode *, NFSPROC_T *); +int nfs_pnfsio(task_fn_t *, void *); + SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW, 0, "NFS server"); SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW, &nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points"); @@ -4436,6 +4439,9 @@ nfsmout: * so that this function can be executed by a separate kernel process. */ struct nfsrvwritedsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; off_t off; int len; @@ -4443,7 +4449,6 @@ struct nfsrvwritedsdorpc { struct ucred *cred; NFSPROC_T *p; struct mbuf *m; - int haskproc; int err; }; @@ -4560,18 +4565,15 @@ nfsmout: * Start up the thread that will execute nfsrv_writedsdorpc(). */ static void -start_writedsdorpc(void *arg) +start_writedsdorpc(void *arg, int pending) { struct nfsrvwritedsdorpc *drpc; drpc = (struct nfsrvwritedsdorpc *)arg; drpc->err = nfsrv_writedsdorpc(drpc->nmp, &drpc->fh, drpc->off, drpc->len, NULL, drpc->m, drpc->cred, drpc->p); - NFSDWRPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDWRPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; + NFSD_DEBUG(4, "start_writedsdorpc: err=%d\n", drpc->err); } static int @@ -4582,7 +4584,7 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s struct nfsrvwritedsdorpc *drpc, *tdrpc; struct nfsvattr na; struct mbuf *m; - int error, haskproc, i, offs, ret; + int error, i, offs, ret, timo; NFSD_DEBUG(4, "in nfsrv_writedsrpc\n"); KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain")); @@ -4599,23 +4601,25 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s * Do the write RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; tdrpc->fh = *fhp; tdrpc->off = off; tdrpc->len = len; tdrpc->nmp = *nmpp; tdrpc->cred = cred; tdrpc->p = p; + tdrpc->inprog = 0; + tdrpc->err = 0; tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK); - tdrpc->haskproc = 1; - ret = kproc_create(start_writedsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdpw"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_writedsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_writedsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL, tdrpc->m, cred, p); if (error == 0 && ret != 0) @@ -4632,17 +4636,16 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s error = nfsrv_setextattr(vp, &na, p); NFSD_DEBUG(4, "nfsrv_writedsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDWRPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDWRPCLOCKMUTEXPTR, PVFS, - "nfspw", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDWRPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvwrds", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error); @@ -4732,6 +4735,9 @@ nfsmout: } struct nfsrvsetattrdsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; struct nfsmount *nmp; struct vnode *vp; @@ -4739,7 +4745,6 @@ struct nfsrvsetattrdsdorpc { NFSPROC_T *p; struct nfsvattr na; struct nfsvattr dsna; - int haskproc; int err; }; @@ -4747,18 +4752,14 @@ struct nfsrvsetattrdsdorpc { * Start up the thread that will execute nfsrv_setattrdsdorpc(). */ static void -start_setattrdsdorpc(void *arg) +start_setattrdsdorpc(void *arg, int pending) { struct nfsrvsetattrdsdorpc *drpc; drpc = (struct nfsrvsetattrdsdorpc *)arg; drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p, drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna); - NFSDSRPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDSRPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; } static int @@ -4768,7 +4769,7 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, { struct nfsrvsetattrdsdorpc *drpc, *tdrpc; struct nfsvattr na; - int error, haskproc, i, ret; + int error, i, ret, timo; NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n"); drpc = NULL; @@ -4780,22 +4781,24 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, * Do the setattr RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; + tdrpc->inprog = 0; tdrpc->fh = *fhp; tdrpc->nmp = *nmpp; tdrpc->vp = vp; tdrpc->cred = cred; tdrpc->p = p; tdrpc->na = *nap; - tdrpc->haskproc = 1; - ret = kproc_create(start_setattrdsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdps"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + tdrpc->err = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_setattrdsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_setattrdsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap, &na); if (error == 0 && ret != 0) @@ -4810,17 +4813,16 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, if (error == 0) error = nfsrv_setextattr(vp, &na, p); NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDSRPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDSRPCLOCKMUTEXPTR, PVFS, - "nfsps", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDSRPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvsads", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error); @@ -4877,13 +4879,15 @@ nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred } struct nfsrvsetacldsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; struct nfsmount *nmp; struct vnode *vp; struct ucred *cred; NFSPROC_T *p; struct acl *aclp; - int haskproc; int err; }; @@ -4891,18 +4895,14 @@ struct nfsrvsetacldsdorpc { * Start up the thread that will execute nfsrv_setacldsdorpc(). */ static void -start_setacldsdorpc(void *arg) +start_setacldsdorpc(void *arg, int pending) { struct nfsrvsetacldsdorpc *drpc; drpc = (struct nfsrvsetacldsdorpc *)arg; drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p, drpc->vp, drpc->nmp, drpc->aclp); - NFSDARPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDARPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; } static int @@ -4910,7 +4910,7 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, struct acl *aclp) { struct nfsrvsetacldsdorpc *drpc, *tdrpc; - int error, haskproc, i, ret; + int error, i, ret, timo; NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n"); drpc = NULL; @@ -4922,22 +4922,24 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, * Do the setattr RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; + tdrpc->inprog = 0; tdrpc->fh = *fhp; tdrpc->nmp = *nmpp; tdrpc->vp = vp; tdrpc->cred = cred; tdrpc->p = p; tdrpc->aclp = aclp; - tdrpc->haskproc = 1; - ret = kproc_create(start_setacldsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdpa"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + tdrpc->err = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_setacldsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_setacldsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp, aclp); if (error == 0 && ret != 0) @@ -4950,17 +4952,16 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, if (error == 0 && ret != 0) error = ret; NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDARPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDARPCLOCKMUTEXPTR, PVFS, - "nfspa", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDARPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvacds", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201710162343.v9GNhIdh009490>