From owner-svn-src-projects@freebsd.org Sun Nov 17 03:54:33 2019 Return-Path: Delivered-To: svn-src-projects@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 1E5B21B5930 for ; Sun, 17 Nov 2019 03:54:33 +0000 (UTC) (envelope-from rmacklem@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47Fyrm4kt4z45v0; Sun, 17 Nov 2019 03:54:32 +0000 (UTC) (envelope-from rmacklem@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4FE25C138; Sun, 17 Nov 2019 03:54:32 +0000 (UTC) (envelope-from rmacklem@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xAH3sWnZ078842; Sun, 17 Nov 2019 03:54:32 GMT (envelope-from rmacklem@FreeBSD.org) Received: (from rmacklem@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xAH3sVcK078838; Sun, 17 Nov 2019 03:54:31 GMT (envelope-from rmacklem@FreeBSD.org) Message-Id: <201911170354.xAH3sVcK078838@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: rmacklem set sender to rmacklem@FreeBSD.org using -f From: Rick Macklem Date: Sun, 17 Nov 2019 03:54:31 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r354784 - in projects/nfsv42/sys/fs: nfs nfsserver X-SVN-Group: projects X-SVN-Commit-Author: rmacklem X-SVN-Commit-Paths: in projects/nfsv42/sys/fs: nfs nfsserver X-SVN-Commit-Revision: 354784 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 17 Nov 2019 03:54:33 -0000 Author: rmacklem Date: Sun Nov 17 03:54:31 2019 New Revision: 354784 URL: https://svnweb.freebsd.org/changeset/base/354784 Log: Add support for Allocate to the pNFS server. The Allocate operation needed to be proxied to the DSs for the pNFS server. This patch adds Allocate proxying to the pNFS server for NFSv4.2. Modified: projects/nfsv42/sys/fs/nfs/nfs_var.h projects/nfsv42/sys/fs/nfsserver/nfs_nfsdport.c projects/nfsv42/sys/fs/nfsserver/nfs_nfsdserv.c Modified: projects/nfsv42/sys/fs/nfs/nfs_var.h ============================================================================== --- projects/nfsv42/sys/fs/nfs/nfs_var.h Sun Nov 17 01:01:02 2019 (r354783) +++ projects/nfsv42/sys/fs/nfs/nfs_var.h Sun Nov 17 03:54:31 2019 (r354784) @@ -745,6 +745,7 @@ void nfsrv_killrpcs(struct nfsmount *); int nfsrv_setacl(struct vnode *, NFSACL_T *, struct ucred *, NFSPROC_T *); int nfsvno_seek(struct nfsrv_descript *, struct vnode *, u_long, off_t *, int, bool *, struct ucred *, NFSPROC_T *); +int nfsvno_allocate(struct vnode *, off_t, off_t, struct ucred *, NFSPROC_T *); int nfsvno_getxattr(struct vnode *, char *, uint32_t, struct ucred *, struct thread *, struct mbuf **, struct mbuf **, int *); int nfsvno_setxattr(struct vnode *, char *, int, struct mbuf *, char *, Modified: projects/nfsv42/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/nfsv42/sys/fs/nfsserver/nfs_nfsdport.c Sun Nov 17 01:01:02 2019 (r354783) +++ projects/nfsv42/sys/fs/nfsserver/nfs_nfsdport.c Sun Nov 17 03:54:31 2019 (r354784) @@ -125,6 +125,8 @@ static int nfsrv_readdsrpc(fhandle_t *, off_t, int, st static int nfsrv_writedsrpc(fhandle_t *, off_t, int, struct ucred *, NFSPROC_T *, struct vnode *, struct nfsmount **, int, struct mbuf **, char *, int *); +static int nfsrv_allocatedsrpc(fhandle_t *, off_t, off_t, struct ucred *, + NFSPROC_T *, struct vnode *, struct nfsmount **, int, int *); static int nfsrv_setacldsrpc(fhandle_t *, struct ucred *, NFSPROC_T *, struct vnode *, struct nfsmount **, int, struct acl *, int *); static int nfsrv_setattrdsrpc(fhandle_t *, struct ucred *, NFSPROC_T *, @@ -4580,7 +4582,10 @@ tryagain: error = 0; trycnt++; } - } else { + } else if (ioproc == NFSPROC_ALLOCATE) + error = nfsrv_allocatedsrpc(fh, off, *offp, cred, p, vp, + &nmp[0], mirrorcnt, &failpos); + else { error = nfsrv_getattrdsrpc(&fh[mirrorcnt - 1], cred, p, vp, nmp[mirrorcnt - 1], nap); if (nfsds_failerr(error) && mirrorcnt > 1) { @@ -5202,7 +5207,166 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s return (error); } +/* + * Do a allocate RPC on a DS data file, using this structure for the arguments, + * so that this function can be executed by a separate kernel process. + */ +struct nfsrvallocatedsdorpc { + int done; + int inprog; + struct task tsk; + fhandle_t fh; + off_t off; + off_t len; + struct nfsmount *nmp; + struct ucred *cred; + NFSPROC_T *p; + int err; +}; + static int +nfsrv_allocatedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, + off_t len, struct nfsvattr *nap, struct ucred *cred, NFSPROC_T *p) +{ + uint32_t *tl; + struct nfsrv_descript *nd; + nfsattrbit_t attrbits; + nfsv4stateid_t st; + int error; + + nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK | M_ZERO); + nfscl_reqstart(nd, NFSPROC_ALLOCATE, nmp, (u_int8_t *)fhp, + sizeof(fhandle_t), NULL, NULL, 0, 0); + + /* + * Use a stateid where other is an alternating 01010 pattern and + * seqid is 0xffffffff. This value is not defined as special by + * the RFC and is used by the FreeBSD NFS server to indicate an + * MDS->DS proxy operation. + */ + st.other[0] = 0x55555555; + st.other[1] = 0x55555555; + st.other[2] = 0x55555555; + st.seqid = 0xffffffff; + nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID); + NFSM_BUILD(tl, uint32_t *, 2 * NFSX_HYPER + NFSX_UNSIGNED); + txdr_hyper(off, tl); tl += 2; + txdr_hyper(len, tl); tl += 2; + NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: len=%jd\n", (intmax_t)len); + + *tl = txdr_unsigned(NFSV4OP_GETATTR); + NFSGETATTR_ATTRBIT(&attrbits); + nfsrv_putattrbit(nd, &attrbits); + error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, + cred, NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL); + if (error != 0) { + free(nd, M_TEMP); + return (error); + } + NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: aft allocaterpc=%d\n", + nd->nd_repstat); + if (nd->nd_repstat == 0) { + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL); + } else + error = nd->nd_repstat; + NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: aft loadattr=%d\n", error); +nfsmout: + m_freem(nd->nd_mrep); + free(nd, M_TEMP); + NFSD_DEBUG(4, "nfsrv_allocatedsdorpc error=%d\n", error); + return (error); +} + +/* + * Start up the thread that will execute nfsrv_allocatedsdorpc(). + */ +static void +start_allocatedsdorpc(void *arg, int pending) +{ + struct nfsrvallocatedsdorpc *drpc; + + drpc = (struct nfsrvallocatedsdorpc *)arg; + drpc->err = nfsrv_allocatedsdorpc(drpc->nmp, &drpc->fh, drpc->off, + drpc->len, NULL, drpc->cred, drpc->p); + drpc->done = 1; + NFSD_DEBUG(4, "start_allocatedsdorpc: err=%d\n", drpc->err); +} + +static int +nfsrv_allocatedsrpc(fhandle_t *fhp, off_t off, off_t len, struct ucred *cred, + NFSPROC_T *p, struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, + int *failposp) +{ + struct nfsrvallocatedsdorpc *drpc, *tdrpc; + struct nfsvattr na; + int error, i, ret, timo; + + NFSD_DEBUG(4, "in nfsrv_allocatedsrpc\n"); + drpc = NULL; + if (mirrorcnt > 1) + tdrpc = drpc = malloc(sizeof(*drpc) * (mirrorcnt - 1), M_TEMP, + M_WAITOK); + + /* + * Do the allocate RPC for every DS, using a separate kernel process + * for every DS except the last one. + */ + error = 0; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + tdrpc->done = 0; + NFSBCOPY(fhp, &tdrpc->fh, sizeof(*fhp)); + tdrpc->off = off; + tdrpc->len = len; + tdrpc->nmp = *nmpp; + tdrpc->cred = cred; + tdrpc->p = p; + tdrpc->inprog = 0; + tdrpc->err = 0; + ret = EIO; + if (nfs_pnfsiothreads != 0) { + ret = nfs_pnfsio(start_allocatedsdorpc, tdrpc); + NFSD_DEBUG(4, "nfsrv_allocatedsrpc: nfs_pnfsio=%d\n", + ret); + } + if (ret != 0) { + ret = nfsrv_allocatedsdorpc(*nmpp, fhp, off, len, NULL, + cred, p); + if (nfsds_failerr(ret) && *failposp == -1) + *failposp = i; + else if (error == 0 && ret != 0) + error = ret; + } + nmpp++; + fhp++; + } + ret = nfsrv_allocatedsdorpc(*nmpp, fhp, off, len, &na, cred, p); + if (nfsds_failerr(ret) && *failposp == -1 && mirrorcnt > 1) + *failposp = mirrorcnt - 1; + else if (error == 0 && ret != 0) + error = ret; + if (error == 0) + error = nfsrv_setextattr(vp, &na, p); + NFSD_DEBUG(4, "nfsrv_allocatedsrpc: aft setextat=%d\n", error); + tdrpc = drpc; + timo = hz / 50; /* Wait for 20msec. */ + if (timo < 1) + timo = 1; + for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) { + /* Wait for RPCs on separate threads to complete. */ + while (tdrpc->inprog != 0 && tdrpc->done == 0) + tsleep(&tdrpc->tsk, PVFS, "srvalds", timo); + if (nfsds_failerr(tdrpc->err) && *failposp == -1) + *failposp = i; + else if (error == 0 && tdrpc->err != 0) + error = tdrpc->err; + } + free(drpc, M_TEMP); + return (error); +} + +static int nfsrv_setattrdsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, struct vnode *vp, struct nfsmount *nmp, struct nfsvattr *nap, struct nfsvattr *dsnap) @@ -5910,6 +6074,39 @@ nfsvno_seek(struct nfsrv_descript *nd, struct vnode *v error = ret; } vrele(vp); + NFSEXITCODE(error); + return (error); +} + +/* + * Allocate vnode op call. + */ +int +nfsvno_allocate(struct vnode *vp, off_t off, off_t len, struct ucred *cred, + NFSPROC_T *p) +{ + int error, trycnt; + + ASSERT_VOP_ELOCKED(vp, "nfsvno_allocate vp"); + /* + * Attempt to allocate on a DS file. A return of ENOENT implies + * there is no DS file to allocate on. + */ + error = nfsrv_proxyds(vp, off, 0, cred, p, NFSPROC_ALLOCATE, NULL, + NULL, NULL, NULL, NULL, &len, 0, NULL); + if (error != ENOENT) + return (error); + error = 0; + + /* + * Do the actual VOP_ALLOCATE(), looping a reasonable number of + * times to achieve completion. + */ + trycnt = 0; + while (error == 0 && len > 0 && trycnt++ < 20) + error = VOP_ALLOCATE(vp, &off, &len); + if (error == 0 && len > 0) + error = NFSERR_IO; NFSEXITCODE(error); return (error); } Modified: projects/nfsv42/sys/fs/nfsserver/nfs_nfsdserv.c ============================================================================== --- projects/nfsv42/sys/fs/nfsserver/nfs_nfsdserv.c Sun Nov 17 01:01:02 2019 (r354783) +++ projects/nfsv42/sys/fs/nfsserver/nfs_nfsdserv.c Sun Nov 17 03:54:31 2019 (r354784) @@ -5104,7 +5104,7 @@ nfsrvd_allocate(struct nfsrv_descript *nd, __unused in { uint32_t *tl; struct nfsvattr forat; - int error = 0, forat_ret = 1, trycnt; + int error = 0, forat_ret = 1, gotproxystateid; off_t off, len; struct nfsstate st, *stp = &st; struct nfslock lo, *lop = &lo; @@ -5116,6 +5116,7 @@ nfsrvd_allocate(struct nfsrv_descript *nd, __unused in nd->nd_repstat = NFSERR_WRONGSEC; goto nfsmout; } + gotproxystateid = 0; NFSM_DISSECT(tl, uint32_t *, NFSX_STATEID + 2 * NFSX_HYPER); stp->ls_flags = (NFSLCK_CHECK | NFSLCK_WRITEACCESS); lop->lo_flags = NFSLCK_WRITE; @@ -5142,6 +5143,12 @@ nfsrvd_allocate(struct nfsrv_descript *nd, __unused in */ if ((nd->nd_flag & ND_DSSERVER) != 0) nd->nd_repstat = NFSERR_NOTSUPP; + /* However, allow the proxy stateid. */ + if (stp->ls_stateid.seqid == 0xffffffff && + stp->ls_stateid.other[0] == 0x55555555 && + stp->ls_stateid.other[1] == 0x55555555 && + stp->ls_stateid.other[2] == 0x55555555) + gotproxystateid = 1; off = fxdr_hyper(tl); tl += 2; lop->lo_first = off; len = fxdr_hyper(tl); @@ -5150,7 +5157,7 @@ nfsrvd_allocate(struct nfsrv_descript *nd, __unused in * Paranoia, just in case it wraps around, which shouldn't * ever happen anyhow. */ - if (nd->nd_repstat == 0 && lop->lo_end < lop->lo_first) + if (nd->nd_repstat == 0 && (lop->lo_end < lop->lo_first || len <= 0)) nd->nd_repstat = NFSERR_INVAL; if (nd->nd_repstat == 0 && vnode_vtype(vp) != VREG) @@ -5165,19 +5172,13 @@ nfsrvd_allocate(struct nfsrv_descript *nd, __unused in nd->nd_repstat = nfsvno_accchk(vp, VWRITE, nd->nd_cred, exp, curthread, NFSACCCHK_ALLOWOWNER, NFSACCCHK_VPISLOCKED, NULL); - if (nd->nd_repstat == 0) + if (nd->nd_repstat == 0 && gotproxystateid == 0) nd->nd_repstat = nfsrv_lockctrl(vp, &stp, &lop, NULL, clientid, &stateid, exp, nd, curthread); - /* - * Do the actual VOP_ALLOCATE(), looping a reasonable number of - * times to achieve completion. - */ - trycnt = 0; - while (nd->nd_repstat == 0 && len > 0 && trycnt++ < 20) - nd->nd_repstat = VOP_ALLOCATE(vp, &off, &len); - if (nd->nd_repstat == 0 && len > 0) - nd->nd_repstat = NFSERR_IO; + if (nd->nd_repstat == 0) + nd->nd_repstat = nfsvno_allocate(vp, off, len, nd->nd_cred, + curthread); vput(vp); NFSEXITCODE2(0, nd); return (0);