Date: Thu, 17 Aug 2017 00:20:50 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r322603 - in projects/pnfs-planb-server/sys/fs: nfs nfsserver Message-ID: <201708170020.v7H0KoZa014126@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Thu Aug 17 00:20:50 2017 New Revision: 322603 URL: https://svnweb.freebsd.org/changeset/base/322603 Log: Add support for mirrors to the Setattr RPC proxies to the DS data file. Also, fix the arguments to the RPC functions from nfsrv_proxyds() so that mirrors are handled. Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c projects/pnfs-planb-server/sys/fs/nfs/nfsport.h projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c Thu Aug 17 00:15:41 2017 (r322602) +++ projects/pnfs-planb-server/sys/fs/nfs/nfs_commonport.c Thu Aug 17 00:20:50 2017 (r322603) @@ -69,6 +69,8 @@ struct mtx nfsrv_dslock_mtx; struct mtx nfsrv_dsclock_mtx; struct mtx nfsrv_dsrmlock_mtx; struct mtx nfsrv_dwrpclock_mtx; +struct mtx nfsrv_dsrpclock_mtx; +struct mtx nfsrv_darpclock_mtx; struct nfsdevicehead nfsrv_devidhead; void (*nfsd_call_servertimer)(void) = NULL; void (*ncl_call_invalcaches)(struct vnode *) = NULL; @@ -722,6 +724,8 @@ nfscommon_modevent(module_t mod, int type, void *data) mtx_init(&nfsrv_dsclock_mtx, "nfsdsc", NULL, MTX_DEF); mtx_init(&nfsrv_dsrmlock_mtx, "nfsdsrm", NULL, MTX_DEF); mtx_init(&nfsrv_dwrpclock_mtx, "nfsdwrpc", NULL, MTX_DEF); + mtx_init(&nfsrv_dsrpclock_mtx, "nfsdsrpc", NULL, MTX_DEF); + mtx_init(&nfsrv_darpclock_mtx, "nfsdarpc", NULL, MTX_DEF); TAILQ_INIT(&nfsrv_devidhead); callout_init(&newnfsd_callout, 1); newnfs_init(); @@ -753,6 +757,8 @@ nfscommon_modevent(module_t mod, int type, void *data) mtx_destroy(&nfsrv_dsclock_mtx); mtx_destroy(&nfsrv_dsrmlock_mtx); mtx_destroy(&nfsrv_dwrpclock_mtx); + mtx_destroy(&nfsrv_dsrpclock_mtx); + mtx_destroy(&nfsrv_darpclock_mtx); loaded = 0; break; default: Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsport.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfsport.h Thu Aug 17 00:15:41 2017 (r322602) +++ projects/pnfs-planb-server/sys/fs/nfs/nfsport.h Thu Aug 17 00:20:50 2017 (r322603) @@ -722,6 +722,12 @@ void nfsrvd_rcv(struct socket *, void *, int); #define NFSDWRPCLOCKMUTEXPTR (&nfsrv_dwrpclock_mtx) #define NFSDWRPCLOCK() mtx_lock(&nfsrv_dwrpclock_mtx) #define NFSDWRPCUNLOCK() mtx_unlock(&nfsrv_dwrpclock_mtx) +#define NFSDSRPCLOCKMUTEXPTR (&nfsrv_dsrpclock_mtx) +#define NFSDSRPCLOCK() mtx_lock(&nfsrv_dsrpclock_mtx) +#define NFSDSRPCUNLOCK() mtx_unlock(&nfsrv_dsrpclock_mtx) +#define NFSDARPCLOCKMUTEXPTR (&nfsrv_darpclock_mtx) +#define NFSDARPCLOCK() mtx_lock(&nfsrv_darpclock_mtx) +#define NFSDARPCUNLOCK() mtx_unlock(&nfsrv_darpclock_mtx) /* * Use these macros to initialize/free a mutex. Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Thu Aug 17 00:15:41 2017 (r322602) +++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Thu Aug 17 00:20:50 2017 (r322603) @@ -70,6 +70,8 @@ extern struct mtx nfsrv_dslock_mtx; extern struct mtx nfsrv_dsclock_mtx; extern struct mtx nfsrv_dsrmlock_mtx; extern struct mtx nfsrv_dwrpclock_mtx; +extern struct mtx nfsrv_dsrpclock_mtx; +extern struct mtx nfsrv_darpclock_mtx; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; NFSSTATESPINLOCK; @@ -113,11 +115,12 @@ static int nfsrv_setextattr(struct vnode *, struct nfs static int nfsrv_readdsrpc(fhandle_t *, off_t, int, struct ucred *, NFSPROC_T *, struct nfsmount *, struct mbuf **, struct mbuf **); static int nfsrv_writedsrpc(fhandle_t *, off_t, int, struct ucred *, - NFSPROC_T *, struct vnode *, struct nfsmount *, struct mbuf **, char *); + NFSPROC_T *, struct vnode *, struct nfsmount **, int, struct mbuf **, + char *); static int nfsrv_setacldsrpc(fhandle_t *, struct ucred *, NFSPROC_T *, - struct vnode *, struct nfsmount *, struct acl *); + struct vnode *, struct nfsmount **, int, struct acl *); static int nfsrv_setattrdsrpc(fhandle_t *, struct ucred *, NFSPROC_T *, - struct vnode *, struct nfsmount *, struct nfsvattr *); + struct vnode *, struct nfsmount **, int, struct nfsvattr *); static int nfsrv_getattrdsrpc(fhandle_t *, struct ucred *, NFSPROC_T *, struct vnode *, struct nfsmount *, struct nfsvattr *); static int nfsrv_putfhname(fhandle_t *, char *); @@ -4078,7 +4081,7 @@ nfsrv_proxyds(struct nfsrv_descript *nd, struct vnode struct vnode *dvp[NFSDEV_MAXMIRRORS]; struct pnfsdsattr dsattr; char *buf; - int buflen, error, mirrorcnt; + int buflen, error, i, mirrorcnt; NFSD_DEBUG(4, "in nfsrv_proxyds\n"); /* @@ -4151,17 +4154,18 @@ nfsrv_proxyds(struct nfsrv_descript *nd, struct vnode mpp, mpp2); else if (ioproc == NFSPROC_WRITEDS) error = nfsrv_writedsrpc(fh, off, cnt, cred, p, vp, - nmp[0], mpp, cp); + &nmp[0], mirrorcnt, mpp, cp); else if (ioproc == NFSPROC_SETATTR) - error = nfsrv_setattrdsrpc(fh, cred, p, vp, nmp[0], - nap); + error = nfsrv_setattrdsrpc(fh, cred, p, vp, &nmp[0], + mirrorcnt, nap); else if (ioproc == NFSPROC_SETACL) - error = nfsrv_setacldsrpc(fh, cred, p, vp, nmp[0], - aclp); + error = nfsrv_setacldsrpc(fh, cred, p, vp, &nmp[0], + mirrorcnt, aclp); else - error = nfsrv_getattrdsrpc(fh, cred, p, vp, nmp[0], - nap); - NFSVOPUNLOCK(dvp[0], 0); + error = nfsrv_getattrdsrpc(&fh[mirrorcnt - 1], cred, p, + vp, nmp[mirrorcnt - 1], nap); + for (i = 0; i < mirrorcnt; i++) + NFSVOPUNLOCK(dvp[i], 0); NFSD_DEBUG(4, "nfsrv_proxyds: aft RPC=%d\n", error); } else { /* Return ENOENT for any Extended Attribute error. */ @@ -4553,13 +4557,13 @@ start_writedsdorpc(void *arg) static int nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, struct ucred *cred, - NFSPROC_T *p, struct vnode *vp, struct nfsmount *nmp, struct mbuf **mpp, - char *cp) + NFSPROC_T *p, struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, + struct mbuf **mpp, char *cp) { struct nfsrvwritedsdorpc *drpc, *tdrpc; struct nfsvattr na; struct mbuf *m; - int error, haskproc, i, offs, ret, mirrorcnt = 1; + int error, haskproc, i, offs, ret; NFSD_DEBUG(4, "in nfsrv_writedsrpc\n"); KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain")); @@ -4582,7 +4586,7 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s tdrpc->fh = *fhp; tdrpc->off = off; tdrpc->len = len; - tdrpc->nmp = nmp; + tdrpc->nmp = *nmpp; tdrpc->cred = cred; tdrpc->p = p; tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK); @@ -4593,14 +4597,16 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s haskproc = 1; else { tdrpc->haskproc = 0; - ret = nfsrv_writedsdorpc(nmp, fhp, off, len, NULL, + ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL, tdrpc->m, cred, p); if (error == 0 && ret != 0) error = ret; } + nmpp++; + fhp++; } m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK); - ret = nfsrv_writedsdorpc(nmp, fhp, off, len, &na, m, cred, p); + ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, &na, m, cred, p); if (error == 0 && ret != 0) error = ret; if (error == 0) @@ -4624,17 +4630,17 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s } static int -nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, - struct vnode *vp, struct nfsmount *nmp, struct nfsvattr *nap) +nfsrv_setattrdsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, + struct vnode *vp, struct nfsmount *nmp, struct nfsvattr *nap, + struct nfsvattr *dsnap) { uint32_t *tl; struct nfsrv_descript *nd; nfsv4stateid_t st; nfsattrbit_t attrbits; - struct nfsvattr na; int error; - NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n"); + NFSD_DEBUG(4, "in nfsrv_setattrdsdorpc\n"); nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK | M_ZERO); /* * Use a stateid where other is an alternating 01010 pattern and @@ -4666,15 +4672,14 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, free(nd, M_TEMP); return (error); } - NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setattrrpc=%d\n", + NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: aft setattrrpc=%d\n", nd->nd_repstat); /* Get rid of weak cache consistency data for now. */ if ((nd->nd_flag & (ND_NOMOREDATA | ND_NFSV4 | ND_V4WCCATTR)) == (ND_NFSV4 | ND_V4WCCATTR)) { - error = nfsv4_loadattr(nd, NULL, &na, NULL, - NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, NULL); - NFSD_DEBUG(4, "nfsrv_setattrdsrpc: wcc attr=%d\n", error); + error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL, + NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL); + NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: wcc attr=%d\n", error); if (error != 0) goto nfsmout; /* @@ -4696,26 +4701,117 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, */ if (error == 0) { NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); - error = nfsv4_loadattr(nd, NULL, &na, NULL, NULL, 0, - NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, - NULL, NULL); + error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL, + NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL); } - NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setattr loadattr=%d\n", error); - if (error == 0) - error = nfsrv_setextattr(vp, &na, p); - NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error); + NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: aft setattr loadattr=%d\n", error); nfsmout: m_freem(nd->nd_mrep); free(nd, M_TEMP); - NFSD_DEBUG(4, "nfsrv_setattrdsrpc error=%d\n", error); + NFSD_DEBUG(4, "nfsrv_setattrdsdorpc error=%d\n", error); return (error); } +struct nfsrvsetattrdsdorpc { + fhandle_t fh; + struct nfsmount *nmp; + struct vnode *vp; + struct ucred *cred; + NFSPROC_T *p; + struct nfsvattr na; + struct nfsvattr dsna; + int haskproc; + int err; +}; + /* + * Start up the thread that will execute nfsrv_setattrdsdorpc(). + */ +static void +start_setattrdsdorpc(void *arg) +{ + struct nfsrvsetattrdsdorpc *drpc; + + drpc = (struct nfsrvsetattrdsdorpc *)arg; + drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p, + drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna); + NFSDSRPCLOCK(); + drpc->haskproc = 0; + wakeup(drpc); + NFSDSRPCUNLOCK(); + kproc_exit(0); +} + +static int +nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, + struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, + struct nfsvattr *nap) +{ + struct nfsrvsetattrdsdorpc *drpc, *tdrpc; + struct nfsvattr na; + int error, haskproc, i, ret; + + NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n"); + drpc = NULL; + if (mirrorcnt > 1) + tdrpc = drpc = malloc(sizeof(*drpc) * (mirrorcnt - 1), M_TEMP, + M_WAITOK); + + /* + * Do the setattr RPC for every DS, using a separate kernel process + * for every DS except the last one. + */ + haskproc = 0; + error = 0; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->fh = *fhp; + tdrpc->nmp = *nmpp; + tdrpc->vp = vp; + tdrpc->cred = cred; + tdrpc->p = p; + tdrpc->na = *nap; + tdrpc->haskproc = 1; + ret = kproc_create(start_setattrdsdorpc, (void *)tdrpc, NULL, 0, + 0, "nfsdps"); + if (ret == 0) + haskproc = 1; + else { + tdrpc->haskproc = 0; + ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap, + &na); + if (error == 0 && ret != 0) + error = ret; + } + nmpp++; + fhp++; + } + ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap, &na); + if (error == 0 && ret != 0) + error = ret; + if (error == 0) + error = nfsrv_setextattr(vp, &na, p); + NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error); + if (haskproc != 0) { + /* Wait for kernel proc(s) to complete. */ + NFSDSRPCLOCK(); + for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + while (tdrpc->haskproc != 0) + mtx_sleep(tdrpc, NFSDSRPCLOCKMUTEXPTR, PVFS, + "nfsps", 0); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; + } + NFSDSRPCUNLOCK(); + } + free(drpc, M_TEMP); + return (error); +} + +/* * Do a Setattr of an NFSv4 ACL on the DS file. */ static int -nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, +nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, struct vnode *vp, struct nfsmount *nmp, struct acl *aclp) { struct nfsrv_descript *nd; @@ -4723,7 +4819,7 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, nfsattrbit_t attrbits; int error; - NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n"); + NFSD_DEBUG(4, "in nfsrv_setacldsdorpc\n"); nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK | M_ZERO); /* * Use a stateid where other is an alternating 01010 pattern and @@ -4753,11 +4849,101 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, free(nd, M_TEMP); return (error); } - NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setaclrpc=%d\n", + NFSD_DEBUG(4, "nfsrv_setacldsdorpc: aft setaclrpc=%d\n", nd->nd_repstat); error = nd->nd_repstat; m_freem(nd->nd_mrep); free(nd, M_TEMP); + return (error); +} + +struct nfsrvsetacldsdorpc { + fhandle_t fh; + struct nfsmount *nmp; + struct vnode *vp; + struct ucred *cred; + NFSPROC_T *p; + struct acl *aclp; + int haskproc; + int err; +}; + +/* + * Start up the thread that will execute nfsrv_setacldsdorpc(). + */ +static void +start_setacldsdorpc(void *arg) +{ + struct nfsrvsetacldsdorpc *drpc; + + drpc = (struct nfsrvsetacldsdorpc *)arg; + drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p, + drpc->vp, drpc->nmp, drpc->aclp); + NFSDARPCLOCK(); + drpc->haskproc = 0; + wakeup(drpc); + NFSDARPCUNLOCK(); + kproc_exit(0); +} + +static int +nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, + struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, struct acl *aclp) +{ + struct nfsrvsetacldsdorpc *drpc, *tdrpc; + int error, haskproc, i, ret; + + NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n"); + drpc = NULL; + if (mirrorcnt > 1) + tdrpc = drpc = malloc(sizeof(*drpc) * (mirrorcnt - 1), M_TEMP, + M_WAITOK); + + /* + * Do the setattr RPC for every DS, using a separate kernel process + * for every DS except the last one. + */ + haskproc = 0; + error = 0; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->fh = *fhp; + tdrpc->nmp = *nmpp; + tdrpc->vp = vp; + tdrpc->cred = cred; + tdrpc->p = p; + tdrpc->aclp = aclp; + tdrpc->haskproc = 1; + ret = kproc_create(start_setacldsdorpc, (void *)tdrpc, NULL, 0, + 0, "nfsdpa"); + if (ret == 0) + haskproc = 1; + else { + tdrpc->haskproc = 0; + ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp, + aclp); + if (error == 0 && ret != 0) + error = ret; + } + nmpp++; + fhp++; + } + ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp, aclp); + if (error == 0 && ret != 0) + error = ret; + NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error); + if (haskproc != 0) { + /* Wait for kernel proc(s) to complete. */ + NFSDARPCLOCK(); + for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + while (tdrpc->haskproc != 0) + mtx_sleep(tdrpc, NFSDARPCLOCKMUTEXPTR, PVFS, + "nfspa", 0); + if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; + } + NFSDARPCUNLOCK(); + } + free(drpc, M_TEMP); return (error); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201708170020.v7H0KoZa014126>