From owner-svn-src-all@FreeBSD.ORG Wed Jan 28 19:05:18 2009 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id AAF6B10656C4; Wed, 28 Jan 2009 19:05:18 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 979BF8FC12; Wed, 28 Jan 2009 19:05:18 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n0SJ5It0029607; Wed, 28 Jan 2009 19:05:18 GMT (envelope-from jhb@svn.freebsd.org) Received: (from jhb@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n0SJ5IT8029606; Wed, 28 Jan 2009 19:05:18 GMT (envelope-from jhb@svn.freebsd.org) Message-Id: <200901281905.n0SJ5IT8029606@svn.freebsd.org> From: John Baldwin Date: Wed, 28 Jan 2009 19:05:18 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r187839 - head/sys/kern X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Jan 2009 19:05:19 -0000 Author: jhb Date: Wed Jan 28 19:05:18 2009 New Revision: 187839 URL: http://svn.freebsd.org/changeset/base/187839 Log: Convert the global mutex protecting the directory lookup name cache from a mutex to a reader/writer lock. Lookup operations first grab a read lock and perform the lookup. If the operation results in a need to modify the cache, then it tries to do an upgrade. If that fails, it drops the read lock, obtains a write lock, and redoes the lookup. Modified: head/sys/kern/vfs_cache.c Modified: head/sys/kern/vfs_cache.c ============================================================================== --- head/sys/kern/vfs_cache.c Wed Jan 28 18:54:56 2009 (r187838) +++ head/sys/kern/vfs_cache.c Wed Jan 28 19:05:18 2009 (r187839) @@ -42,9 +42,9 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include +#include #include #include #include @@ -109,11 +109,14 @@ SYSCTL_ULONG(_debug, OID_AUTO, numcachep #endif struct nchstats nchstats; /* cache effectiveness statistics */ -static struct mtx cache_lock; -MTX_SYSINIT(vfscache, &cache_lock, "Name Cache", MTX_DEF); +static struct rwlock cache_lock; +RW_SYSINIT(vfscache, &cache_lock, "Name Cache"); -#define CACHE_LOCK() mtx_lock(&cache_lock) -#define CACHE_UNLOCK() mtx_unlock(&cache_lock) +#define CACHE_UPGRADE_LOCK() rw_try_upgrade(&cache_lock) +#define CACHE_RLOCK() rw_rlock(&cache_lock) +#define CACHE_RUNLOCK() rw_runlock(&cache_lock) +#define CACHE_WLOCK() rw_wlock(&cache_lock) +#define CACHE_WUNLOCK() rw_wunlock(&cache_lock) /* * UMA zones for the VFS cache. @@ -162,6 +165,7 @@ static u_long numposzaps; STATNODE(CTLFL static u_long numposhits; STATNODE(CTLFLAG_RD, numposhits, &numposhits); static u_long numnegzaps; STATNODE(CTLFLAG_RD, numnegzaps, &numnegzaps); static u_long numneghits; STATNODE(CTLFLAG_RD, numneghits, &numneghits); +static u_long numupgrades; STATNODE(CTLFLAG_RD, numupgrades, &numupgrades); SYSCTL_OPAQUE(_vfs_cache, OID_AUTO, nchstats, CTLFLAG_RD | CTLFLAG_MPSAFE, &nchstats, sizeof(nchstats), "LU", "VFS cache effectiveness statistics"); @@ -200,12 +204,12 @@ sysctl_debug_hashstat_rawnchash(SYSCTL_H /* Scan hash tables for applicable entries */ for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { - CACHE_LOCK(); + CACHE_RLOCK(); count = 0; LIST_FOREACH(ncp, ncpp, nc_hash) { count++; } - CACHE_UNLOCK(); + CACHE_RUNLOCK(); error = SYSCTL_OUT(req, &count, sizeof(count)); if (error) return (error); @@ -235,11 +239,11 @@ sysctl_debug_hashstat_nchash(SYSCTL_HAND /* Scan hash tables for applicable entries */ for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { count = 0; - CACHE_LOCK(); + CACHE_RLOCK(); LIST_FOREACH(ncp, ncpp, nc_hash) { count++; } - CACHE_UNLOCK(); + CACHE_RUNLOCK(); if (count) used++; if (maxlength < count) @@ -277,7 +281,7 @@ cache_zap(ncp) { struct vnode *vp; - mtx_assert(&cache_lock, MA_OWNED); + rw_assert(&cache_lock, RA_WLOCKED); CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, ncp->nc_vp); vp = NULL; LIST_REMOVE(ncp, nc_hash); @@ -324,16 +328,19 @@ cache_lookup(dvp, vpp, cnp) { struct namecache *ncp; u_int32_t hash; - int error, ltype; + int error, ltype, wlocked; if (!doingcache) { cnp->cn_flags &= ~MAKEENTRY; return (0); } retry: - CACHE_LOCK(); + CACHE_RLOCK(); + wlocked = 0; numcalls++; + error = 0; +retry_wlocked: if (cnp->cn_nameptr[0] == '.') { if (cnp->cn_namelen == 1) { *vpp = dvp; @@ -346,8 +353,7 @@ retry: dotdothits++; if (dvp->v_dd == NULL || (cnp->cn_flags & MAKEENTRY) == 0) { - CACHE_UNLOCK(); - return (0); + goto unlock; } *vpp = dvp->v_dd; CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..", @@ -366,23 +372,24 @@ retry: } /* We failed to find an entry */ - if (ncp == 0) { + if (ncp == NULL) { if ((cnp->cn_flags & MAKEENTRY) == 0) { nummisszap++; } else { nummiss++; } nchstats.ncs_miss++; - CACHE_UNLOCK(); - return (0); + goto unlock; } /* We don't want to have an entry, so dump it */ if ((cnp->cn_flags & MAKEENTRY) == 0) { numposzaps++; nchstats.ncs_badhits++; + if (!wlocked && !CACHE_UPGRADE_LOCK()) + goto wlock; cache_zap(ncp); - CACHE_UNLOCK(); + CACHE_WUNLOCK(); return (0); } @@ -400,11 +407,15 @@ retry: if (cnp->cn_nameiop == CREATE) { numnegzaps++; nchstats.ncs_badhits++; + if (!wlocked && !CACHE_UPGRADE_LOCK()) + goto wlock; cache_zap(ncp); - CACHE_UNLOCK(); + CACHE_WUNLOCK(); return (0); } + if (!wlocked && !CACHE_UPGRADE_LOCK()) + goto wlock; numneghits++; /* * We found a "negative" match, so we shift it to the end of @@ -417,9 +428,20 @@ retry: nchstats.ncs_neghits++; if (ncp->nc_flag & NCF_WHITE) cnp->cn_flags |= ISWHITEOUT; - CACHE_UNLOCK(); + CACHE_WUNLOCK(); return (ENOENT); +wlock: + /* + * We need to update the cache after our lookup, so upgrade to + * a write lock and retry the operation. + */ + CACHE_RUNLOCK(); + CACHE_WLOCK(); + numupgrades++; + wlocked = 1; + goto retry_wlocked; + success: /* * On success we return a locked and ref'd vnode as per the lookup @@ -427,7 +449,10 @@ success: */ if (dvp == *vpp) { /* lookup on "." */ VREF(*vpp); - CACHE_UNLOCK(); + if (wlocked) + CACHE_WUNLOCK(); + else + CACHE_RUNLOCK(); /* * When we lookup "." we still can be asked to lock it * differently... @@ -453,7 +478,10 @@ success: VOP_UNLOCK(dvp, 0); } VI_LOCK(*vpp); - CACHE_UNLOCK(); + if (wlocked) + CACHE_WUNLOCK(); + else + CACHE_RUNLOCK(); error = vget(*vpp, cnp->cn_lkflags | LK_INTERLOCK, cnp->cn_thread); if (cnp->cn_flags & ISDOTDOT) vn_lock(dvp, ltype | LK_RETRY); @@ -466,6 +494,13 @@ success: ASSERT_VOP_ELOCKED(*vpp, "cache_lookup"); } return (-1); + +unlock: + if (wlocked) + CACHE_WUNLOCK(); + else + CACHE_RUNLOCK(); + return (0); } /* @@ -509,10 +544,10 @@ cache_enter(dvp, vp, cnp) * cache_purge() time. */ if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') { - CACHE_LOCK(); + CACHE_WLOCK(); if (!TAILQ_EMPTY(&dvp->v_cache_dst)) dvp->v_dd = vp; - CACHE_UNLOCK(); + CACHE_WUNLOCK(); return; } } @@ -531,7 +566,7 @@ cache_enter(dvp, vp, cnp) hash = fnv_32_buf(cnp->cn_nameptr, len, FNV1_32_INIT); bcopy(cnp->cn_nameptr, ncp->nc_name, len); hash = fnv_32_buf(&dvp, sizeof(dvp), hash); - CACHE_LOCK(); + CACHE_WLOCK(); /* * See if this vnode or negative entry is already in the cache @@ -543,7 +578,7 @@ cache_enter(dvp, vp, cnp) if (n2->nc_dvp == dvp && n2->nc_nlen == cnp->cn_namelen && !bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) { - CACHE_UNLOCK(); + CACHE_WUNLOCK(); cache_free(ncp); return; } @@ -587,7 +622,7 @@ cache_enter(dvp, vp, cnp) vhold(dvp); if (zap) cache_zap(ncp); - CACHE_UNLOCK(); + CACHE_WUNLOCK(); } /* @@ -618,13 +653,13 @@ cache_purge(vp) { CTR1(KTR_VFS, "cache_purge(%p)", vp); - CACHE_LOCK(); + CACHE_WLOCK(); while (!LIST_EMPTY(&vp->v_cache_src)) cache_zap(LIST_FIRST(&vp->v_cache_src)); while (!TAILQ_EMPTY(&vp->v_cache_dst)) cache_zap(TAILQ_FIRST(&vp->v_cache_dst)); vp->v_dd = NULL; - CACHE_UNLOCK(); + CACHE_WUNLOCK(); } /* @@ -638,14 +673,14 @@ cache_purgevfs(mp) struct namecache *ncp, *nnp; /* Scan hash tables for applicable entries */ - CACHE_LOCK(); + CACHE_WLOCK(); for (ncpp = &nchashtbl[nchash]; ncpp >= nchashtbl; ncpp--) { LIST_FOREACH_SAFE(ncp, ncpp, nc_hash, nnp) { if (ncp->nc_dvp->v_mount == mp) cache_zap(ncp); } } - CACHE_UNLOCK(); + CACHE_WUNLOCK(); } /* @@ -845,7 +880,7 @@ vn_vptocnp(struct vnode **vp, char **bp, int error, vfslocked; vhold(*vp); - CACHE_UNLOCK(); + CACHE_RUNLOCK(); vfslocked = VFS_LOCK_GIANT((*vp)->v_mount); vn_lock(*vp, LK_SHARED | LK_RETRY); error = VOP_VPTOCNP(*vp, &dvp, buf, buflen); @@ -858,10 +893,10 @@ vn_vptocnp(struct vnode **vp, char **bp, } *bp = buf + *buflen; *vp = dvp; - CACHE_LOCK(); + CACHE_RLOCK(); if ((*vp)->v_iflag & VI_DOOMED) { /* forced unmount */ - CACHE_UNLOCK(); + CACHE_RUNLOCK(); vdrop(*vp); return (ENOENT); } @@ -887,7 +922,7 @@ vn_fullpath1(struct thread *td, struct v error = 0; slash_prefixed = 0; - CACHE_LOCK(); + CACHE_RLOCK(); numfullpathcalls++; if (vp->v_type != VDIR) { ncp = TAILQ_FIRST(&vp->v_cache_dst); @@ -896,7 +931,7 @@ vn_fullpath1(struct thread *td, struct v *--bp = ncp->nc_name[i]; if (bp == buf) { numfullpathfail4++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); return (ENOMEM); } vp = ncp->nc_dvp; @@ -910,7 +945,7 @@ vn_fullpath1(struct thread *td, struct v buflen--; if (buflen < 0) { numfullpathfail4++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); return (ENOMEM); } slash_prefixed = 1; @@ -918,7 +953,7 @@ vn_fullpath1(struct thread *td, struct v while (vp != rdir && vp != rootvnode) { if (vp->v_vflag & VV_ROOT) { if (vp->v_iflag & VI_DOOMED) { /* forced unmount */ - CACHE_UNLOCK(); + CACHE_RUNLOCK(); error = EBADF; break; } @@ -927,7 +962,7 @@ vn_fullpath1(struct thread *td, struct v } if (vp->v_type != VDIR) { numfullpathfail1++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); error = ENOTDIR; break; } @@ -939,7 +974,7 @@ vn_fullpath1(struct thread *td, struct v *--bp = ncp->nc_name[i]; if (bp == buf) { numfullpathfail4++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); error = ENOMEM; break; } @@ -954,7 +989,7 @@ vn_fullpath1(struct thread *td, struct v buflen--; if (buflen < 0) { numfullpathfail4++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); error = ENOMEM; break; } @@ -965,14 +1000,14 @@ vn_fullpath1(struct thread *td, struct v if (!slash_prefixed) { if (bp == buf) { numfullpathfail4++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); return (ENOMEM); } else { *--bp = '/'; } } numfullpathfound++; - CACHE_UNLOCK(); + CACHE_RUNLOCK(); *retbuf = bp; return (0); @@ -984,15 +1019,15 @@ vn_commname(struct vnode *vp, char *buf, struct namecache *ncp; int l; - CACHE_LOCK(); + CACHE_RLOCK(); ncp = TAILQ_FIRST(&vp->v_cache_dst); if (!ncp) { - CACHE_UNLOCK(); + CACHE_RUNLOCK(); return (ENOENT); } l = min(ncp->nc_nlen, buflen - 1); memcpy(buf, ncp->nc_name, l); - CACHE_UNLOCK(); + CACHE_RUNLOCK(); buf[l] = '\0'; return (0); }