Date: Wed, 11 Oct 2017 13:33:12 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r324530 - in projects/pnfs-planb-server-stable11/sys/fs: nfs nfsclient nfsserver Message-ID: <201710111333.v9BDXCck095142@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Wed Oct 11 13:33:11 2017 New Revision: 324530 URL: https://svnweb.freebsd.org/changeset/base/324530 Log: Update the stable11 pNFS client/server. The main changes are: - Make the pNFS client use a taskqueue to do writes and commits concurrently. - Convert the pNFS server to using this taskqueue instead of kproc_create(). Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c Wed Oct 11 13:33:11 2017 (r324530) @@ -1119,9 +1119,29 @@ nfsmout: int newnfs_nmcancelreqs(struct nfsmount *nmp) { + struct nfsclds *dsp; + struct __rpc_client *cl; if (nmp->nm_sockreq.nr_client != NULL) CLNT_CLOSE(nmp->nm_sockreq.nr_client); +lookformore: + NFSLOCKMNT(nmp); + TAILQ_FOREACH(dsp, &nmp->nm_sess, nfsclds_list) { + NFSLOCKDS(dsp); + if (dsp != TAILQ_FIRST(&nmp->nm_sess) && + (dsp->nfsclds_flags & NFSCLDS_CLOSED) == 0 && + dsp->nfsclds_sockp != NULL && + dsp->nfsclds_sockp->nr_client != NULL) { + dsp->nfsclds_flags |= NFSCLDS_CLOSED; + cl = dsp->nfsclds_sockp->nr_client; + NFSUNLOCKDS(dsp); + NFSUNLOCKMNT(nmp); + CLNT_CLOSE(cl); + goto lookformore; + } + NFSUNLOCKDS(dsp); + } + NFSUNLOCKMNT(nmp); return (0); } Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c Wed Oct 11 13:33:11 2017 (r324530) @@ -40,7 +40,9 @@ __FBSDID("$FreeBSD$"); * to this BSD variant. */ #include <fs/nfs/nfsport.h> +#include <sys/smp.h> #include <sys/sysctl.h> +#include <sys/taskqueue.h> #include <vm/vm.h> #include <vm/vm_object.h> #include <vm/vm_page.h> @@ -74,6 +76,8 @@ struct nfsdevicehead nfsrv_devidhead; void (*nfsd_call_servertimer)(void) = NULL; void (*ncl_call_invalcaches)(struct vnode *) = NULL; +int nfs_pnfsio(task_fn_t *, void *); + static int nfs_realign_test; static int nfs_realign_count; static struct ext_nfsstats oldnfsstats; @@ -90,6 +94,9 @@ SYSCTL_INT(_vfs_nfs, OID_AUTO, debuglevel, CTLFLAG_RW, 0, "Debug level for NFS client"); SYSCTL_INT(_vfs_nfs, OID_AUTO, userhashsize, CTLFLAG_RDTUN, &nfsrv_lughashsize, 0, "Size of hash tables for uid/name mapping"); +int nfs_pnfsiothreads = 0; +SYSCTL_INT(_vfs_nfs, OID_AUTO, pnfsiothreads, CTLFLAG_RW, &nfs_pnfsiothreads, + 0, "Number of pNFS mirror I/O threads"); /* * Defines for malloc @@ -676,6 +683,50 @@ nfs_supportsnfsv4acls(struct vnode *vp) if (error == 0 && retval != 0) return (1); return (0); +} + +/* + * These are the first fields of all the context structures passed into + * nfs_pnfsio(). + */ +struct pnfsio { + int done; + int inprog; + struct task tsk; +}; + +/* + * Do a mirror I/O on a pNFS thread. + */ +int +nfs_pnfsio(task_fn_t *func, void *context) +{ + struct pnfsio *pio; + int ret; + static struct taskqueue *pnfsioq = NULL; + + pio = (struct pnfsio *)context; + if (pnfsioq == NULL) { + if (nfs_pnfsiothreads == 0) + nfs_pnfsiothreads = mp_ncpus * 8; + pnfsioq = taskqueue_create("pnfsioq", M_WAITOK, + taskqueue_thread_enqueue, &pnfsioq); + if (pnfsioq == NULL) + return (ENOMEM); + ret = taskqueue_start_threads(&pnfsioq, nfs_pnfsiothreads, + 0, "pnfsiot"); + if (ret != 0) { + taskqueue_free(pnfsioq); + pnfsioq = NULL; + return (ret); + } + } + pio->inprog = 1; + TASK_INIT(&pio->tsk, 0, func, context); + ret = taskqueue_enqueue(pnfsioq, &pio->tsk); + if (ret != 0) + pio->inprog = 0; + return (ret); } extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *); Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h Wed Oct 11 13:33:11 2017 (r324530) @@ -91,6 +91,7 @@ struct nfsclds { #define NFSCLDS_HASWRITEVERF 0x0001 #define NFSCLDS_MDS 0x0002 #define NFSCLDS_DS 0x0004 +#define NFSCLDS_CLOSED 0x0008 struct nfsclclient { LIST_ENTRY(nfsclclient) nfsc_list; Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c Wed Oct 11 13:33:11 2017 (r324530) @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$"); #include <fs/nfs/nfsport.h> #include <sys/sysctl.h> +#include <sys/taskqueue.h> SYSCTL_DECL(_vfs_nfs); @@ -64,6 +65,7 @@ extern nfstype nfsv34_type[9]; extern int nfsrv_useacl; extern char nfsv4_callbackaddr[INET6_ADDRSTRLEN]; extern int nfscl_debuglevel; +extern int nfs_pnfsiothreads; NFSCLSTATEMUTEX; int nfstest_outofseq = 0; int nfscl_assumeposixlocks = 1; @@ -87,6 +89,30 @@ enum nfsclds_state { NFSDSP_NOTFOUND = 2, }; +/* + * Do a write RPC on a DS data file, using this structure for the arguments, + * so that this function can be executed by a separate kernel process. + */ +struct nfsclwritedsdorpc { + int done; + int inprog; + struct task tsk; + struct vnode *vp; + int iomode; + int must_commit; + nfsv4stateid_t *stateidp; + struct nfsclds *dsp; + uint64_t off; + int len; + struct nfsfh *fhp; + struct mbuf *m; + int vers; + int minorvers; + struct ucred *cred; + NFSPROC_T *p; + int err; +}; + static int nfsrpc_setattrrpc(vnode_t , struct vattr *, nfsv4stateid_t *, struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, void *); static int nfsrpc_readrpc(vnode_t , struct uio *, struct ucred *, @@ -119,7 +145,7 @@ static int nfscl_doflayoutio(vnode_t, struct uio *, in static int nfscl_dofflayoutio(vnode_t, struct uio *, int *, int *, int *, nfsv4stateid_t *, int, struct nfscldevinfo *, struct nfscllayout *, struct nfsclflayout *, uint64_t, uint64_t, int, int, struct mbuf *, - struct ucred *, NFSPROC_T *); + struct nfsclwritedsdorpc *, struct ucred *, NFSPROC_T *); static struct mbuf *nfsm_copym(struct mbuf *, int, int); static int nfsrpc_readds(vnode_t, struct uio *, nfsv4stateid_t *, int *, struct nfsclds *, uint64_t, int, struct nfsfh *, int, int, int, @@ -127,11 +153,17 @@ static int nfsrpc_readds(vnode_t, struct uio *, nfsv4s static int nfsrpc_writeds(vnode_t, struct uio *, int *, int *, nfsv4stateid_t *, struct nfsclds *, uint64_t, int, struct nfsfh *, int, int, int, int, struct ucred *, NFSPROC_T *); +static int nfsio_writedsmir(vnode_t, int *, int *, nfsv4stateid_t *, + struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int, + struct nfsclwritedsdorpc *, struct ucred *, NFSPROC_T *); static int nfsrpc_writedsmir(vnode_t, int *, int *, nfsv4stateid_t *, struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int, struct ucred *, NFSPROC_T *); static enum nfsclds_state nfscl_getsameserver(struct nfsmount *, struct nfsclds *, struct nfsclds **); +static int nfsio_commitds(vnode_t, uint64_t, int, struct nfsclds *, + struct nfsfh *, int, int, struct nfsclwritedsdorpc *, struct ucred *, + NFSPROC_T *); static int nfsrpc_commitds(vnode_t, uint64_t, int, struct nfsclds *, struct nfsfh *, int, int, struct ucred *, NFSPROC_T *); static void nfsrv_setuplayoutget(struct nfsrv_descript *, int, uint64_t, @@ -163,6 +195,8 @@ static int nfsrpc_layoutgetres(struct nfsmount *, vnod int, nfsv4stateid_t *, int, uint32_t *, struct nfscllayout **, struct nfsclflayouthead *, int, int, int *, struct ucred *, NFSPROC_T *); +int nfs_pnfsio(task_fn_t *, void *); + /* * nfs null call from vfs. */ @@ -5539,10 +5573,11 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode struct nfscldevinfo *dip; struct nfsclflayout *rflp; struct mbuf *m; + struct nfsclwritedsdorpc *drpc, *tdrpc; nfsv4stateid_t stateid; struct ucred *newcred; uint64_t lastbyte, len, off, oresid, xfer; - int eof, error, firstmirror, i, iolaymode, mirrorcnt, recalled; + int eof, error, firstmirror, i, iolaymode, mirrorcnt, recalled, timo; void *lckp; uint8_t *dev; void *iovbase; @@ -5622,6 +5657,7 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode * do all mirrors. */ m = NULL; + drpc = NULL; firstmirror = 0; mirrorcnt = 1; if ((layp->nfsly_flags & NFSLY_FLEXFILE) != 0 && @@ -5629,17 +5665,24 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode if (rwaccess == NFSV4OPEN_ACCESSREAD) { firstmirror = arc4random() % mirrorcnt; mirrorcnt = firstmirror + 1; - } else if (docommit == 0) { - /* - * Save values, so uiop can be rolled - * back upon a write error. - */ - offs = uiop->uio_offset; - resid = uiop->uio_resid; - iovbase = uiop->uio_iov->iov_base; - iovlen = uiop->uio_iov->iov_len; - m = nfsm_uiombuflist(uiop, len, NULL, - NULL); + } else { + if (docommit == 0) { + /* + * Save values, so uiop can be + * rolled back upon a write + * error. + */ + offs = uiop->uio_offset; + resid = uiop->uio_resid; + iovbase = + uiop->uio_iov->iov_base; + iovlen = uiop->uio_iov->iov_len; + m = nfsm_uiombuflist(uiop, len, + NULL, NULL); + } + tdrpc = drpc = malloc(sizeof(*drpc) * + (mirrorcnt - 1), M_TEMP, M_WAITOK | + M_ZERO); } } for (i = firstmirror; i < mirrorcnt && error == 0; i++){ @@ -5656,8 +5699,8 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode uiop, iomode, must_commit, &eof, &stateid, rwaccess, dip, layp, rflp, off, xfer, - i, docommit, m, newcred, - p); + i, docommit, m, tdrpc, + newcred, p); else error = nfscl_doflayoutio(vp, uiop, iomode, must_commit, @@ -5667,9 +5710,27 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode nfscl_reldevinfo(dip); } else error = EIO; + tdrpc++; } if (m != NULL) m_freem(m); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = firstmirror; i < mirrorcnt - 1 && + tdrpc != NULL; i++, tdrpc++) { + /* + * For the unused drpc entries, both inprog and + * err == 0, so this loop won't break. + */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "clrpcio", + timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; + } + free(drpc, M_TEMP); if (error == 0) { if (mirrorcnt > 1 && rwaccess == NFSV4OPEN_ACCESSWRITE && docommit == 0) { @@ -5893,8 +5954,8 @@ static int nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int *iomode, int *must_commit, int *eofp, nfsv4stateid_t *stateidp, int rwflag, struct nfscldevinfo *dp, struct nfscllayout *lyp, struct nfsclflayout *flp, uint64_t off, - uint64_t len, int mirror, int docommit, struct mbuf *mp, struct ucred *cred, - NFSPROC_T *p) + uint64_t len, int mirror, int docommit, struct mbuf *mp, + struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p) { uint64_t transfer, xfer; int error, rel_off; @@ -5935,13 +5996,23 @@ nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int * else xfer = len; if (docommit != 0) { - if (error == 0) - error = nfsrpc_commitds(vp, off, xfer, *dspp, - fhp, dp->nfsdi_vers, dp->nfsdi_minorvers, - tcred, p); - NFSCL_DEBUG(4, "aft nfsrpc_commitds=%d\n", error); if (error == 0) { /* + * Do last mirrored DS commit with this thread. + */ + if (mirror < flp->nfsfl_mirrorcnt - 1) + error = nfsio_commitds(vp, off, xfer, + *dspp, fhp, dp->nfsdi_vers, + dp->nfsdi_minorvers, drpc, tcred, + p); + else + error = nfsrpc_commitds(vp, off, xfer, + *dspp, fhp, dp->nfsdi_vers, + dp->nfsdi_minorvers, tcred, p); + } + NFSCL_DEBUG(4, "aft nfsio_commitds=%d\n", error); + if (error == 0) { + /* * Set both eof and uio_resid = 0 to end any * loops. */ @@ -5971,11 +6042,22 @@ nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int * m = nfsm_copym(mp, rel_off, xfer); NFSCL_DEBUG(4, "mcopy reloff=%d xfer=%jd\n", rel_off, (uintmax_t)xfer); - error = nfsrpc_writedsmir(vp, iomode, - must_commit, stateidp, *dspp, off, xfer, - fhp, m, dp->nfsdi_vers, dp->nfsdi_minorvers, - tcred, p); - NFSCL_DEBUG(4, "nfsrpc_writedsmir=%d\n", error); + /* + * Do last write to a mirrored DS with this + * thread. + */ + if (mirror < flp->nfsfl_mirrorcnt - 1) + error = nfsio_writedsmir(vp, iomode, + must_commit, stateidp, *dspp, off, + xfer, fhp, m, dp->nfsdi_vers, + dp->nfsdi_minorvers, drpc, tcred, + p); + else + error = nfsrpc_writedsmir(vp, iomode, + must_commit, stateidp, *dspp, off, + xfer, fhp, m, dp->nfsdi_vers, + dp->nfsdi_minorvers, tcred, p); + NFSCL_DEBUG(4, "nfsio_writedsmir=%d\n", error); } } NFSCL_DEBUG(4, "aft read/writeds=%d\n", error); @@ -6302,6 +6384,62 @@ nfsmout: } /* + * Start up the thread that will execute nfsrpc_writedsmir(). + */ +static void +start_writedsmir(void *arg, int pending) +{ + struct nfsclwritedsdorpc *drpc; + + drpc = (struct nfsclwritedsdorpc *)arg; + drpc->err = nfsrpc_writedsmir(drpc->vp, &drpc->iomode, + &drpc->must_commit, drpc->stateidp, drpc->dsp, drpc->off, drpc->len, + drpc->fhp, drpc->m, drpc->vers, drpc->minorvers, drpc->cred, + drpc->p); + drpc->done = 1; + NFSCL_DEBUG(4, "start_writedsmir: err=%d\n", drpc->err); +} + +/* + * Set up the write DS mirror call for the pNFS I/O thread. + */ +static int +nfsio_writedsmir(vnode_t vp, int *iomode, int *must_commit, + nfsv4stateid_t *stateidp, struct nfsclds *dsp, uint64_t off, int len, + struct nfsfh *fhp, struct mbuf *m, int vers, int minorvers, + struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p) +{ + int error, ret; + + error = 0; + drpc->done = 0; + drpc->vp = vp; + drpc->iomode = *iomode; + drpc->must_commit = *must_commit; + drpc->stateidp = stateidp; + drpc->dsp = dsp; + drpc->off = off; + drpc->len = len; + drpc->fhp = fhp; + drpc->m = m; + drpc->vers = vers; + drpc->minorvers = minorvers; + drpc->cred = cred; + drpc->p = p; + drpc->inprog = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_writedsmir, drpc); + NFSCL_DEBUG(4, "nfsio_writedsmir: nfs_pnfsio=%d\n", ret); + } + if (ret != 0) + error = nfsrpc_writedsmir(vp, iomode, must_commit, stateidp, + dsp, off, len, fhp, m, vers, minorvers, cred, p); + NFSCL_DEBUG(4, "nfsio_writedsmir: error=%d\n", error); + return (error); +} + +/* * Free up the nfsclds structure. */ void @@ -6427,6 +6565,56 @@ nfsmout: if (error == 0 && nd->nd_repstat != 0) error = nd->nd_repstat; mbuf_freem(nd->nd_mrep); + return (error); +} + +/* + * Start up the thread that will execute nfsrpc_commitds(). + */ +static void +start_commitds(void *arg, int pending) +{ + struct nfsclwritedsdorpc *drpc; + + drpc = (struct nfsclwritedsdorpc *)arg; + drpc->err = nfsrpc_commitds(drpc->vp, drpc->off, drpc->len, + drpc->dsp, drpc->fhp, drpc->vers, drpc->minorvers, drpc->cred, + drpc->p); + drpc->done = 1; + NFSCL_DEBUG(4, "start_commitds: err=%d\n", drpc->err); +} + +/* + * Set up the commit DS mirror call for the pNFS I/O thread. + */ +static int +nfsio_commitds(vnode_t vp, uint64_t offset, int cnt, struct nfsclds *dsp, + struct nfsfh *fhp, int vers, int minorvers, + struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p) +{ + int error, ret; + + error = 0; + drpc->done = 0; + drpc->vp = vp; + drpc->off = offset; + drpc->len = cnt; + drpc->dsp = dsp; + drpc->fhp = fhp; + drpc->vers = vers; + drpc->minorvers = minorvers; + drpc->cred = cred; + drpc->p = p; + drpc->inprog = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_commitds, drpc); + NFSCL_DEBUG(4, "nfsio_commitds: nfs_pnfsio=%d\n", ret); + } + if (ret != 0) + error = nfsrpc_commitds(vp, offset, cnt, dsp, fhp, vers, + minorvers, cred, p); + NFSCL_DEBUG(4, "nfsio_commitds: error=%d\n", error); return (error); } Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c Wed Oct 11 13:33:11 2017 (r324530) @@ -72,6 +72,7 @@ extern struct mtx nfsrv_dsrmlock_mtx; extern struct mtx nfsrv_dwrpclock_mtx; extern struct mtx nfsrv_dsrpclock_mtx; extern struct mtx nfsrv_darpclock_mtx; +extern int nfs_pnfsiothreads; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; NFSSTATESPINLOCK; @@ -127,6 +128,8 @@ static int nfsrv_putfhname(fhandle_t *, char *); static int nfsrv_pnfslookupds(struct vnode *, struct pnfsdsfile *, struct vnode *, NFSPROC_T *); +int nfs_pnfsio(task_fn_t *, void *); + SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW, 0, "NFS server"); SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW, &nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points"); @@ -4419,6 +4422,9 @@ nfsmout: * so that this function can be executed by a separate kernel process. */ struct nfsrvwritedsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; off_t off; int len; @@ -4426,7 +4432,6 @@ struct nfsrvwritedsdorpc { struct ucred *cred; NFSPROC_T *p; struct mbuf *m; - int haskproc; int err; }; @@ -4543,18 +4548,15 @@ nfsmout: * Start up the thread that will execute nfsrv_writedsdorpc(). */ static void -start_writedsdorpc(void *arg) +start_writedsdorpc(void *arg, int pending) { struct nfsrvwritedsdorpc *drpc; drpc = (struct nfsrvwritedsdorpc *)arg; drpc->err = nfsrv_writedsdorpc(drpc->nmp, &drpc->fh, drpc->off, drpc->len, NULL, drpc->m, drpc->cred, drpc->p); - NFSDWRPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDWRPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; + NFSD_DEBUG(4, "start_writedsdorpc: err=%d\n", drpc->err); } static int @@ -4565,7 +4567,7 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s struct nfsrvwritedsdorpc *drpc, *tdrpc; struct nfsvattr na; struct mbuf *m; - int error, haskproc, i, offs, ret; + int error, i, offs, ret, timo; NFSD_DEBUG(4, "in nfsrv_writedsrpc\n"); KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain")); @@ -4582,23 +4584,25 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s * Do the write RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; tdrpc->fh = *fhp; tdrpc->off = off; tdrpc->len = len; tdrpc->nmp = *nmpp; tdrpc->cred = cred; tdrpc->p = p; + tdrpc->inprog = 0; + tdrpc->err = 0; tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK); - tdrpc->haskproc = 1; - ret = kproc_create(start_writedsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdpw"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_writedsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_writedsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL, tdrpc->m, cred, p); if (error == 0 && ret != 0) @@ -4615,17 +4619,16 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s error = nfsrv_setextattr(vp, &na, p); NFSD_DEBUG(4, "nfsrv_writedsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDWRPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDWRPCLOCKMUTEXPTR, PVFS, - "nfspw", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDWRPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvwrds", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error); @@ -4715,6 +4718,9 @@ nfsmout: } struct nfsrvsetattrdsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; struct nfsmount *nmp; struct vnode *vp; @@ -4722,7 +4728,6 @@ struct nfsrvsetattrdsdorpc { NFSPROC_T *p; struct nfsvattr na; struct nfsvattr dsna; - int haskproc; int err; }; @@ -4730,18 +4735,14 @@ struct nfsrvsetattrdsdorpc { * Start up the thread that will execute nfsrv_setattrdsdorpc(). */ static void -start_setattrdsdorpc(void *arg) +start_setattrdsdorpc(void *arg, int pending) { struct nfsrvsetattrdsdorpc *drpc; drpc = (struct nfsrvsetattrdsdorpc *)arg; drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p, drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna); - NFSDSRPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDSRPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; } static int @@ -4751,7 +4752,7 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, { struct nfsrvsetattrdsdorpc *drpc, *tdrpc; struct nfsvattr na; - int error, haskproc, i, ret; + int error, i, ret, timo; NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n"); drpc = NULL; @@ -4763,22 +4764,24 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, * Do the setattr RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; + tdrpc->inprog = 0; tdrpc->fh = *fhp; tdrpc->nmp = *nmpp; tdrpc->vp = vp; tdrpc->cred = cred; tdrpc->p = p; tdrpc->na = *nap; - tdrpc->haskproc = 1; - ret = kproc_create(start_setattrdsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdps"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + tdrpc->err = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_setattrdsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_setattrdsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap, &na); if (error == 0 && ret != 0) @@ -4793,17 +4796,16 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, if (error == 0) error = nfsrv_setextattr(vp, &na, p); NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDSRPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDSRPCLOCKMUTEXPTR, PVFS, - "nfsps", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDSRPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvsads", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error); @@ -4860,13 +4862,15 @@ nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred } struct nfsrvsetacldsdorpc { + int done; + int inprog; + struct task tsk; fhandle_t fh; struct nfsmount *nmp; struct vnode *vp; struct ucred *cred; NFSPROC_T *p; struct acl *aclp; - int haskproc; int err; }; @@ -4874,18 +4878,14 @@ struct nfsrvsetacldsdorpc { * Start up the thread that will execute nfsrv_setacldsdorpc(). */ static void -start_setacldsdorpc(void *arg) +start_setacldsdorpc(void *arg, int pending) { struct nfsrvsetacldsdorpc *drpc; drpc = (struct nfsrvsetacldsdorpc *)arg; drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p, drpc->vp, drpc->nmp, drpc->aclp); - NFSDARPCLOCK(); - drpc->haskproc = 0; - wakeup(drpc); - NFSDARPCUNLOCK(); - kproc_exit(0); + drpc->done = 1; } static int @@ -4893,7 +4893,7 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, struct acl *aclp) { struct nfsrvsetacldsdorpc *drpc, *tdrpc; - int error, haskproc, i, ret; + int error, i, ret, timo; NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n"); drpc = NULL; @@ -4905,22 +4905,24 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, * Do the setattr RPC for every DS, using a separate kernel process * for every DS except the last one. */ - haskproc = 0; error = 0; for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; + tdrpc->inprog = 0; tdrpc->fh = *fhp; tdrpc->nmp = *nmpp; tdrpc->vp = vp; tdrpc->cred = cred; tdrpc->p = p; tdrpc->aclp = aclp; - tdrpc->haskproc = 1; - ret = kproc_create(start_setacldsdorpc, (void *)tdrpc, NULL, 0, - 0, "nfsdpa"); - if (ret == 0) - haskproc = 1; - else { - tdrpc->haskproc = 0; + tdrpc->err = 0; + ret = EIO; + if (nfs_pnfsiothreads > 0) { + ret = nfs_pnfsio(start_setacldsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_setacldsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp, aclp); if (error == 0 && ret != 0) @@ -4933,17 +4935,16 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, if (error == 0 && ret != 0) error = ret; NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error); - if (haskproc != 0) { - /* Wait for kernel proc(s) to complete. */ - NFSDARPCLOCK(); - for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { - while (tdrpc->haskproc != 0) - mtx_sleep(tdrpc, NFSDARPCLOCKMUTEXPTR, PVFS, - "nfspa", 0); - if (error == 0 && tdrpc->err != 0) - error = tdrpc->err; - } - NFSDARPCUNLOCK(); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvacds", timo); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; } free(drpc, M_TEMP); return (error); Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c ============================================================================== --- projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c Wed Oct 11 13:20:24 2017 (r324529) +++ projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c Wed Oct 11 13:33:11 2017 (r324530) @@ -1000,7 +1000,7 @@ nfsrvd_write(struct nfsrv_descript *nd, __unused int i nd->nd_md, nd->nd_dpos, nd->nd_cred, p); error = nfsm_advance(nd, NFSM_RNDUP(retlen), -1); if (error) - panic("nfsrv_write mbuf"); + goto nfsmout; } if (nd->nd_flag & ND_NFSV4) aftat_ret = 0;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201710111333.v9BDXCck095142>