From owner-svn-src-stable-9@FreeBSD.ORG Thu Jan 23 00:42:09 2014 Return-Path: Delivered-To: svn-src-stable-9@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id B935CBC3; Thu, 23 Jan 2014 00:42:09 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 9929016BF; Thu, 23 Jan 2014 00:42:09 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id s0N0g9qu050469; Thu, 23 Jan 2014 00:42:09 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.7/8.14.7/Submit) id s0N0g8CY050465; Thu, 23 Jan 2014 00:42:08 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201401230042.s0N0g8CY050465@svn.freebsd.org> From: Alexander Motin Date: Thu, 23 Jan 2014 00:42:08 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org Subject: svn commit: r261061 - in stable/9/sys: fs/nfsserver nfs nfsserver X-SVN-Group: stable-9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable-9@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: SVN commit messages for only the 9-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 23 Jan 2014 00:42:09 -0000 Author: mav Date: Thu Jan 23 00:42:08 2014 New Revision: 261061 URL: http://svnweb.freebsd.org/changeset/base/261061 Log: MFC r259765: Fix RPC server threads file handle affinity to work better with ZFS. Instead of taking 8 specific bytes of file handle to identify file during RPC thread affitinity handling, use trivial hash of the full file handle. ZFS's struct zfid_short does not have padding field after the length field, as result, originally picked 8 bytes are loosing lower 16 bits of object ID, causing many false matches and unneeded requests affinity to same thread. This fix substantially improves NFS server latency and scalability in SPEC NFS benchmark by more flexible use of multiple NFS threads. Modified: stable/9/sys/fs/nfsserver/nfs_fha_new.c stable/9/sys/nfs/nfs_fha.c stable/9/sys/nfs/nfs_fha.h stable/9/sys/nfsserver/nfs_fha_old.c Directory Properties: stable/9/ (props changed) stable/9/sys/ (props changed) stable/9/sys/fs/ (props changed) Modified: stable/9/sys/fs/nfsserver/nfs_fha_new.c ============================================================================== --- stable/9/sys/fs/nfsserver/nfs_fha_new.c Thu Jan 23 00:41:23 2014 (r261060) +++ stable/9/sys/fs/nfsserver/nfs_fha_new.c Thu Jan 23 00:42:08 2014 (r261061) @@ -41,7 +41,7 @@ static void fhanew_init(void *foo); static void fhanew_uninit(void *foo); rpcproc_t fhanew_get_procnum(rpcproc_t procnum); int fhanew_realign(struct mbuf **mb, int malloc_flags); -int fhanew_get_fh(fhandle_t *fh, int v3, struct mbuf **md, caddr_t *dpos); +int fhanew_get_fh(uint64_t *fh, int v3, struct mbuf **md, caddr_t *dpos); int fhanew_is_read(rpcproc_t procnum); int fhanew_is_write(rpcproc_t procnum); int fhanew_get_offset(struct mbuf **md, caddr_t *dpos, int v3, @@ -128,11 +128,13 @@ fhanew_realign(struct mbuf **mb, int mal } int -fhanew_get_fh(fhandle_t *fh, int v3, struct mbuf **md, caddr_t *dpos) +fhanew_get_fh(uint64_t *fh, int v3, struct mbuf **md, caddr_t *dpos) { struct nfsrv_descript lnd, *nd; uint32_t *tl; - int error, len; + uint8_t *buf; + uint64_t t; + int error, len, i; error = 0; len = 0; @@ -151,11 +153,13 @@ fhanew_get_fh(fhandle_t *fh, int v3, str len = NFSX_V2FH; } + t = 0; if (len != 0) { - NFSM_DISSECT_NONBLOCK(tl, uint32_t *, len); - bcopy(tl, fh, len); - } else - bzero(fh, sizeof(*fh)); + NFSM_DISSECT_NONBLOCK(buf, uint8_t *, len); + for (i = 0; i < len; i++) + t ^= ((uint64_t)buf[i] << (i & 7) * 8); + } + *fh = t; nfsmout: *md = nd->nd_md; Modified: stable/9/sys/nfs/nfs_fha.c ============================================================================== --- stable/9/sys/nfs/nfs_fha.c Thu Jan 23 00:41:23 2014 (r261060) +++ stable/9/sys/nfs/nfs_fha.c Thu Jan 23 00:42:08 2014 (r261061) @@ -130,7 +130,6 @@ fha_extract_info(struct svc_req *req, st struct fha_callbacks *cb) { struct mbuf *md; - fhandle_t fh; caddr_t dpos; static u_int64_t random_fh = 0; int error; @@ -177,12 +176,10 @@ fha_extract_info(struct svc_req *req, st dpos = mtod(md, caddr_t); /* Grab the filehandle. */ - error = cb->get_fh(&fh, v3, &md, &dpos); + error = cb->get_fh(&i->fh, v3, &md, &dpos); if (error) goto out; - bcopy(fh.fh_fid.fid_data, &i->fh, sizeof(i->fh)); - /* Content ourselves with zero offset for all but reads. */ if (cb->is_read(procnum) || cb->is_write(procnum)) cb->get_offset(&md, &dpos, v3, i); Modified: stable/9/sys/nfs/nfs_fha.h ============================================================================== --- stable/9/sys/nfs/nfs_fha.h Thu Jan 23 00:41:23 2014 (r261060) +++ stable/9/sys/nfs/nfs_fha.h Thu Jan 23 00:42:08 2014 (r261061) @@ -82,7 +82,7 @@ struct fha_info { struct fha_callbacks { rpcproc_t (*get_procnum)(rpcproc_t procnum); int (*realign)(struct mbuf **mb, int malloc_flags); - int (*get_fh)(fhandle_t *fh, int v3, struct mbuf **md, caddr_t *dpos); + int (*get_fh)(uint64_t *fh, int v3, struct mbuf **md, caddr_t *dpos); int (*is_read)(rpcproc_t procnum); int (*is_write)(rpcproc_t procnum); int (*get_offset)(struct mbuf **md, caddr_t *dpos, int v3, struct Modified: stable/9/sys/nfsserver/nfs_fha_old.c ============================================================================== --- stable/9/sys/nfsserver/nfs_fha_old.c Thu Jan 23 00:41:23 2014 (r261060) +++ stable/9/sys/nfsserver/nfs_fha_old.c Thu Jan 23 00:42:08 2014 (r261061) @@ -49,7 +49,7 @@ static void fhaold_init(void *foo); static void fhaold_uninit(void *foo); rpcproc_t fhaold_get_procnum(rpcproc_t procnum); int fhaold_realign(struct mbuf **mb, int malloc_flags); -int fhaold_get_fh(fhandle_t *fh, int v3, struct mbuf **md, caddr_t *dpos); +int fhaold_get_fh(uint64_t *fh, int v3, struct mbuf **md, caddr_t *dpos); int fhaold_is_read(rpcproc_t procnum); int fhaold_is_write(rpcproc_t procnum); int fhaold_get_offset(struct mbuf **md, caddr_t *dpos, int v3, @@ -135,9 +135,33 @@ fhaold_realign(struct mbuf **mb, int mal } int -fhaold_get_fh(fhandle_t *fh, int v3, struct mbuf **md, caddr_t *dpos) +fhaold_get_fh(uint64_t *fh, int v3, struct mbuf **md, caddr_t *dpos) { - return (nfsm_srvmtofh_xx(fh, v3, md, dpos)); + u_int32_t *tl; + uint8_t *buf; + uint64_t t; + int fhlen, i; + + if (v3) { + tl = nfsm_dissect_xx_nonblock(NFSX_UNSIGNED, md, dpos); + if (tl == NULL) + return EBADRPC; + fhlen = fxdr_unsigned(int, *tl); + if (fhlen != 0 && fhlen != NFSX_V3FH) + return EBADRPC; + } else { + fhlen = NFSX_V2FH; + } + t = 0; + if (fhlen != 0) { + buf = nfsm_dissect_xx_nonblock(fhlen, md, dpos); + if (buf == NULL) + return EBADRPC; + for (i = 0; i < fhlen; i++) + t ^= ((uint64_t)buf[i] << (i & 7) * 8); + } + *fh = t; + return 0; } int