Date: Mon, 14 Aug 2017 20:53:01 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r322519 - in projects/pnfs-planb-server/sys/fs: nfs nfsserver Message-ID: <201708142053.v7EKr1Sp058857@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Mon Aug 14 20:53:01 2017 New Revision: 322519 URL: https://svnweb.freebsd.org/changeset/base/322519 Log: Modify nfsrv_pnfsremovesetup() and nfsrv_pnfsremove(), factoring out the part that does the Lookup and Remove RPCs against the DS server to remove the DS data file, so that it can be done by a separate kernel process. Put all the arguments to this factored out function (nfsrv_dsremove()) in a structure, so that it can be passed through kproc_create(). This allows removal of the data file on mirrors to be done concurrently. In future, it might be better to avoid the kproc_create() call for every file remove and have a set of these kernel processes waiting to do the RPCs. Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c projects/pnfs-planb-server/sys/fs/nfs/nfsport.h projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c Mon Aug 14 20:47:14 2017 (r322518) +++ projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c Mon Aug 14 20:53:01 2017 (r322519) @@ -67,6 +67,7 @@ struct callout newnfsd_callout; int nfsrv_lughashsize = 100; struct mtx nfsrv_dslock_mtx; struct mtx nfsrv_dsclock_mtx; +struct mtx nfsrv_dsrmlock_mtx; struct nfsdevicehead nfsrv_devidhead; void (*nfsd_call_servertimer)(void) = NULL; void (*ncl_call_invalcaches)(struct vnode *) = NULL; @@ -717,7 +718,8 @@ nfscommon_modevent(module_t mod, int type, void *data) mtx_init(&nfsrv_nfsuserdsock.nr_mtx, "nfsuserd", NULL, MTX_DEF); mtx_init(&nfsrv_dslock_mtx, "nfs4ds", NULL, MTX_DEF); - mtx_init(&nfsrv_dsclock_mtx, "nfs4ds", NULL, MTX_DEF); + mtx_init(&nfsrv_dsclock_mtx, "nfsdsc", NULL, MTX_DEF); + mtx_init(&nfsrv_dsrmlock_mtx, "nfsdsrm", NULL, MTX_DEF); TAILQ_INIT(&nfsrv_devidhead); callout_init(&newnfsd_callout, 1); newnfs_init(); @@ -747,6 +749,7 @@ nfscommon_modevent(module_t mod, int type, void *data) mtx_destroy(&nfsrv_nfsuserdsock.nr_mtx); mtx_destroy(&nfsrv_dslock_mtx); mtx_destroy(&nfsrv_dsclock_mtx); + mtx_destroy(&nfsrv_dsrmlock_mtx); loaded = 0; break; default: Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsport.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfsport.h Mon Aug 14 20:47:14 2017 (r322518) +++ projects/pnfs-planb-server/sys/fs/nfs/nfsport.h Mon Aug 14 20:53:01 2017 (r322519) @@ -716,6 +716,9 @@ void nfsrvd_rcv(struct socket *, void *, int); #define NFSDSCLOCKMUTEXPTR (&nfsrv_dsclock_mtx) #define NFSDSCLOCK() mtx_lock(&nfsrv_dsclock_mtx) #define NFSDSCUNLOCK() mtx_unlock(&nfsrv_dsclock_mtx) +#define NFSDSRMLOCKMUTEXPTR (&nfsrv_dsrmlock_mtx) +#define NFSDSRMLOCK() mtx_lock(&nfsrv_dsrmlock_mtx) +#define NFSDSRMUNLOCK() mtx_unlock(&nfsrv_dsrmlock_mtx) /* * Use these macros to initialize/free a mutex. Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Aug 14 20:47:14 2017 (r322518) +++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Aug 14 20:53:01 2017 (r322519) @@ -68,6 +68,7 @@ extern struct nfslayouthash *nfslayouthash; extern int nfsrv_layouthashsize; extern struct mtx nfsrv_dslock_mtx; extern struct mtx nfsrv_dsclock_mtx; +extern struct mtx nfsrv_dsrmlock_mtx; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; NFSSTATESPINLOCK; @@ -99,8 +100,8 @@ extern struct nfsdevicehead nfsrv_devidhead; static void nfsrv_pnfscreate(struct vnode *, struct vattr *, struct ucred *, NFSPROC_T *); static void nfsrv_pnfsremovesetup(struct vnode *, NFSPROC_T *, struct vnode **, - fhandle_t *, char *); -static void nfsrv_pnfsremove(struct vnode *, fhandle_t *, char *, NFSPROC_T *); + int *, char *); +static void nfsrv_pnfsremove(struct vnode **, int, char *, NFSPROC_T *); static int nfsrv_proxyds(struct nfsrv_descript *, struct vnode *, off_t, int, struct ucred *, struct thread *, int, struct mbuf **, char *, struct mbuf **, struct nfsvattr *, struct acl *); @@ -1199,25 +1200,25 @@ int nfsvno_removesub(struct nameidata *ndp, int is_v4, struct ucred *cred, struct thread *p, struct nfsexstuff *exp) { - struct vnode *vp, *dsdvp; - fhandle_t fh; - int error = 0; + struct vnode *vp, *dsdvp[NFSDEV_MAXMIRRORS]; + int error = 0, i, mirrorcnt; char fname[PNFS_FILENAME_LEN + 1]; vp = ndp->ni_vp; - dsdvp = NULL; + dsdvp[0] = NULL; if (vp->v_type == VDIR) error = NFSERR_ISDIR; else if (is_v4) error = nfsrv_checkremove(vp, 1, p); if (error == 0) - nfsrv_pnfsremovesetup(vp, p, &dsdvp, &fh, fname); + nfsrv_pnfsremovesetup(vp, p, dsdvp, &mirrorcnt, fname); if (!error) error = VOP_REMOVE(ndp->ni_dvp, vp, &ndp->ni_cnd); - if (dsdvp != NULL) { + if (dsdvp[0] != NULL) { if (error == 0) - nfsrv_pnfsremove(dsdvp, &fh, fname, p); - NFSVOPUNLOCK(dsdvp, 0); + nfsrv_pnfsremove(dsdvp, mirrorcnt, fname, p); + for (i = 0; i < mirrorcnt; i++) + NFSVOPUNLOCK(dsdvp[i], 0); } if (ndp->ni_dvp == vp) vrele(ndp->ni_dvp); @@ -1278,12 +1279,11 @@ int nfsvno_rename(struct nameidata *fromndp, struct nameidata *tondp, u_int32_t ndstat, u_int32_t ndflag, struct ucred *cred, struct thread *p) { - struct vnode *fvp, *tvp, *tdvp, *dsdvp; - fhandle_t fh; - int error = 0; + struct vnode *fvp, *tvp, *tdvp, *dsdvp[NFSDEV_MAXMIRRORS]; + int error = 0, i, mirrorcnt; char fname[PNFS_FILENAME_LEN + 1]; - dsdvp = NULL; + dsdvp[0] = NULL; fvp = fromndp->ni_vp; if (ndstat) { vrele(fromndp->ni_dvp); @@ -1359,9 +1359,9 @@ nfsvno_rename(struct nameidata *fromndp, struct nameid nfsd_recalldelegation(fvp, p); } if (error == 0 && tvp != NULL) { - nfsrv_pnfsremovesetup(tvp, p, &dsdvp, &fh, fname); + nfsrv_pnfsremovesetup(tvp, p, dsdvp, &mirrorcnt, fname); NFSD_DEBUG(4, "nfsvno_rename: pnfsremovesetup" - " dsdvp=%p\n", dsdvp); + " dsdvp=%p\n", dsdvp[0]); } out: if (!error) { @@ -1382,16 +1382,17 @@ out: } /* - * If dsdvp != NULL, it was set up by nfsrv_pnfsremovesetup() and + * If dsdvp[0] != NULL, it was set up by nfsrv_pnfsremovesetup() and * if the rename succeeded, the DS file for the tvp needs to be * removed. */ - if (dsdvp != NULL) { + if (dsdvp[0] != NULL) { if (error == 0) { - nfsrv_pnfsremove(dsdvp, &fh, fname, p); + nfsrv_pnfsremove(dsdvp, mirrorcnt, fname, p); NFSD_DEBUG(4, "nfsvno_rename: pnfsremove\n"); } - NFSVOPUNLOCK(dsdvp, 0); + for (i = 0; i < mirrorcnt; i++) + NFSVOPUNLOCK(dsdvp[i], 0); } vrele(tondp->ni_startdir); @@ -3866,16 +3867,15 @@ nfsrv_pnfscreate(struct vnode *vp, struct vattr *vap, */ static void nfsrv_pnfsremovesetup(struct vnode *vp, NFSPROC_T *p, struct vnode **dvpp, - fhandle_t *fhp, char *fname) + int *mirrorcntp, char *fname) { - struct vnode *dvp[NFSDEV_MAXMIRRORS]; struct nfsmount *nmp[NFSDEV_MAXMIRRORS]; struct vattr va; struct ucred *tcred; char *buf; - int buflen, error, mirrorcnt; + int buflen, error; - *dvpp = NULL; + dvpp[0] = NULL; /* If not an exported regular file or not a pNFS server, just return. */ NFSDDSLOCK(); if (vp->v_type != VREG || (vp->v_mount->mnt_flag & MNT_EXPORTED) == 0 || @@ -3899,61 +3899,136 @@ nfsrv_pnfsremovesetup(struct vnode *vp, NFSPROC_T *p, buflen = 1024; buf = malloc(buflen, M_TEMP, M_WAITOK); /* Get the directory vnode for the DS mount and the file handle. */ - error = nfsrv_dsgetsockmnt(vp, LK_EXCLUSIVE, buf, buflen, &mirrorcnt, p, - dvp, nmp, NULL, NULL, fname); - if (error == 0) { - error = nfsvno_getfh(vp, fhp, p); - if (error != 0) { - NFSVOPUNLOCK(dvp[0], 0); - printf("pNFS: nfsrv_pnfsremovesetup getfh=%d\n", error); - } - } else - printf("pNFS: nfsrv_pnfsremovesetup getsockmnt=%d\n", error); + error = nfsrv_dsgetsockmnt(vp, LK_EXCLUSIVE, buf, buflen, mirrorcntp, p, + dvpp, nmp, NULL, NULL, fname); free(buf, M_TEMP); - if (error == 0) - *dvpp = dvp[0]; + if (error != 0) + printf("pNFS: nfsrv_pnfsremovesetup getsockmnt=%d\n", error); } /* - * Remove a pNFS data file from a Data Server. - * nfsrv_pnfsremovesetup() must have been called before the MDS file was - * removed to set up the dvp and fill in the FH. + * Remove a DS data file for nfsrv_pnfsremove(). Called for each mirror. + * The arguments are in a structure, so that they can be passed through + * kproc_create() for a kernel process to execute this function. */ +struct nfsrvdsremove { + struct ucred *tcred; + struct vnode *dvp; + NFSPROC_T *p; + int haskproc; + char fname[PNFS_FILENAME_LEN + 1]; +}; + static void -nfsrv_pnfsremove(struct vnode *dvp, fhandle_t *fhp, char *fname, NFSPROC_T *p) +nfsrv_dsremove(struct nfsrvdsremove *dsrm) { - struct vnode *nvp; struct nameidata named; - struct ucred *tcred; + struct vnode *nvp; char *bufp; u_long *hashp; int error; - /* Look up the data file and remove it. */ - tcred = newnfs_getcred(); named.ni_cnd.cn_nameiop = DELETE; named.ni_cnd.cn_lkflags = LK_EXCLUSIVE | LK_RETRY; - named.ni_cnd.cn_cred = tcred; - named.ni_cnd.cn_thread = p; - named.ni_cnd.cn_flags = ISLASTCN | LOCKPARENT | LOCKLEAF | SAVENAME; + named.ni_cnd.cn_cred = dsrm->tcred; + named.ni_cnd.cn_thread = dsrm->p; + named.ni_cnd.cn_flags = ISLASTCN | LOCKPARENT | LOCKLEAF | + SAVENAME; nfsvno_setpathbuf(&named, &bufp, &hashp); named.ni_cnd.cn_nameptr = bufp; - named.ni_cnd.cn_namelen = strlen(fname); - strlcpy(bufp, fname, NAME_MAX); + named.ni_cnd.cn_namelen = strlen(dsrm->fname); + strlcpy(bufp, dsrm->fname, NAME_MAX); NFSD_DEBUG(4, "nfsrv_pnfsremove: filename=%s\n", bufp); - error = VOP_LOOKUP(dvp, &nvp, &named.ni_cnd); + error = VOP_LOOKUP(dsrm->dvp, &nvp, &named.ni_cnd); NFSD_DEBUG(4, "nfsrv_pnfsremove: aft LOOKUP=%d\n", error); if (error == 0) { - error = VOP_REMOVE(dvp, nvp, &named.ni_cnd); + error = VOP_REMOVE(dsrm->dvp, nvp, &named.ni_cnd); vput(nvp); } - NFSFREECRED(tcred); nfsvno_relpathbuf(&named); if (error != 0) printf("pNFS: nfsrv_pnfsremove failed=%d\n", error); } /* + * Start up the thread that will execute nfsrv_dsremove(). + */ +static void +start_dsremove(void *arg) +{ + struct nfsrvdsremove *dsrm; + + dsrm = (struct nfsrvdsremove *)arg; + nfsrv_dsremove(dsrm); + NFSDSRMLOCK(); + dsrm->haskproc = 0; + wakeup(dsrm); + NFSDSRMUNLOCK(); + kproc_exit(0); +} + +/* + * Remove a pNFS data file from a Data Server. + * nfsrv_pnfsremovesetup() must have been called before the MDS file was + * removed to set up the dvp and fill in the FH. + */ +static void +nfsrv_pnfsremove(struct vnode **dvp, int mirrorcnt, char *fname, NFSPROC_T *p) +{ + struct ucred *tcred; + struct nfsrvdsremove *dsrm, *tdsrm; + int haskproc, i, ret; + + tcred = newnfs_getcred(); + tdsrm = dsrm = malloc(sizeof(*dsrm) * NFSDEV_MAXMIRRORS, M_TEMP, + M_WAITOK); + /* + * Remove the file on each DS mirror, using kernel process(es) for the + * additional mirrors. + */ + haskproc = 0; + for (i = 0; i < mirrorcnt; i++, tdsrm++) { + tdsrm->tcred = tcred; + tdsrm->p = p; + tdsrm->dvp = dvp[i]; + strlcpy(tdsrm->fname, fname, PNFS_FILENAME_LEN + 1); + + /* + * Do the last remove outselves instead of forking a process. + * (This avoids creating kernel processes unless mirrors are + * in use.) + * tdsrm->haskproc marks a create with a kernel process. + * haskproc is set non-zero to indicate at least one kernel + * process has been created. + */ + ret = ENXIO; + if (i < mirrorcnt - 1) { + tdsrm->haskproc = 1; + ret = kproc_create(start_dsremove, (void *)tdsrm, NULL, + 0, 0, "nfsdpnfs"); + } + if (ret == 0) + haskproc = 1; + else { + tdsrm->haskproc = 0; + nfsrv_dsremove(tdsrm); + } + } + if (haskproc != 0) { + /* Wait for kernel proc(s) to complete. */ + NFSDSRMLOCK(); + for (tdsrm = dsrm, i = 0; i < mirrorcnt; i++, tdsrm++) { + while (tdsrm->haskproc != 0) + mtx_sleep(tdsrm, NFSDSRMLOCKMUTEXPTR, PVFS, + "nfsprm", 0); + } + NFSDSRMUNLOCK(); + } + NFSFREECRED(tcred); + free(dsrm, M_TEMP); +} + +/* * Generate a file name based on the file handle and put it in *bufp. * Return the number of bytes generated. */ @@ -4211,8 +4286,10 @@ nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char * the one(s) prior to the failure will have locked dvp's that * need to be unlocked. */ - for (j = 1; j < i; j++) - NFSVOPUNLOCK(*dvpp++, 0); + for (j = 1; j < i; j++) { + NFSVOPUNLOCK(*dvpp, 0); + *dvpp++ = NULL; + } } return (error); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201708142053.v7EKr1Sp058857>