Date: Sat, 20 Jul 2013 08:38:34 +0000 (UTC) From: Attilio Rao <attilio@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r253487 - in user/attilio/vmobj-fullread/sys: cddl/contrib/opensolaris/uts/common/fs/zfs dev/drm2/i915 fs/tmpfs kern vm Message-ID: <201307200838.r6K8cYhv073944@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: attilio Date: Sat Jul 20 08:38:34 2013 New Revision: 253487 URL: http://svnweb.freebsd.org/changeset/base/253487 Log: Rework vm_page_sleep_if_busy() and vm_page_grab() to be possibly called with the object lock in read-mode. With the object lock held in read-mode the busy operation must be time-invariant, which means VM_ALLOC_NOBUSY cannot be really passed. Also, in the vm_page_sleep_if_busy() the VM_ALLOC_* flags don't make much sense, maybe these 2 interfaces should grow their own set of flags (and vm_page_grab() internally can play with VM_ALLOC_* flags as appropriate). Sponsored by: EMC / Isilon storage division Tested by: pho Modified: user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c user/attilio/vmobj-fullread/sys/kern/subr_uio.c user/attilio/vmobj-fullread/sys/kern/uipc_shm.c user/attilio/vmobj-fullread/sys/kern/vfs_bio.c user/attilio/vmobj-fullread/sys/vm/vm_fault.c user/attilio/vmobj-fullread/sys/vm/vm_object.c user/attilio/vmobj-fullread/sys/vm/vm_object.h user/attilio/vmobj-fullread/sys/vm/vm_page.c user/attilio/vmobj-fullread/sys/vm/vm_page.h Modified: user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Sat Jul 20 08:38:34 2013 (r253487) @@ -336,19 +336,9 @@ page_busy(vnode_t *vp, int64_t start, in for (;;) { if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && pp->valid) { - if (vm_page_busy_wlocked(pp)) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_reference(pp); - vm_page_lock(pp); - zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb"); - zfs_vmobject_wlock(obj); + if (vm_page_sleep_if_busy(pp, "zfsmwb", + VM_ALLOC_NOBUSY, TRUE)) continue; - } vm_page_busy_rlock(pp); } else if (pp == NULL) { if (!alloc) Modified: user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c ============================================================================== --- user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c Sat Jul 20 08:38:34 2013 (r253487) @@ -2333,7 +2333,8 @@ retry: m = vm_page_lookup(devobj, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, "915unm")) + if (vm_page_sleep_if_busy(m, "915unm", VM_ALLOC_NOBUSY, + FALSE)) goto retry; cdev_pager_free_page(devobj, m); } Modified: user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c ============================================================================== --- user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c Sat Jul 20 08:38:34 2013 (r253487) @@ -1331,7 +1331,8 @@ tmpfs_reg_resize(struct vnode *vp, off_t retry: m = vm_page_lookup(uobj, idx); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "tmfssz")) + if (vm_page_sleep_if_busy(m, "tmfssz", + VM_ALLOC_NOBUSY, FALSE)) goto retry; MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { Modified: user/attilio/vmobj-fullread/sys/kern/subr_uio.c ============================================================================== --- user/attilio/vmobj-fullread/sys/kern/subr_uio.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/kern/subr_uio.c Sat Jul 20 08:38:34 2013 (r253487) @@ -107,7 +107,8 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { - if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco")) + if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco", + VM_ALLOC_NOBUSY, FALSE)) goto retry; vm_page_lock(user_pg); pmap_remove_all(user_pg); Modified: user/attilio/vmobj-fullread/sys/kern/uipc_shm.c ============================================================================== --- user/attilio/vmobj-fullread/sys/kern/uipc_shm.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/kern/uipc_shm.c Sat Jul 20 08:38:34 2013 (r253487) @@ -281,7 +281,8 @@ shm_dotruncate(struct shmfd *shmfd, off_ retry: m = vm_page_lookup(object, idx); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "shmtrc")) + if (vm_page_sleep_if_busy(m, "shmtrc", + VM_ALLOC_NOBUSY, FALSE)) goto retry; } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL); Modified: user/attilio/vmobj-fullread/sys/kern/vfs_bio.c ============================================================================== --- user/attilio/vmobj-fullread/sys/kern/vfs_bio.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/kern/vfs_bio.c Sat Jul 20 08:38:34 2013 (r253487) @@ -3444,7 +3444,7 @@ allocbuf(struct buf *bp, int size) KASSERT(m != bogus_page, ("allocbuf: bogus page found")); while (vm_page_sleep_if_busy(m, - "biodep")) + "biodep", VM_ALLOC_NOBUSY, FALSE)) continue; bp->b_pages[i] = NULL; Modified: user/attilio/vmobj-fullread/sys/vm/vm_fault.c ============================================================================== --- user/attilio/vmobj-fullread/sys/vm/vm_fault.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/vm/vm_fault.c Sat Jul 20 08:38:34 2013 (r253487) @@ -392,7 +392,8 @@ RetryFault:; unlock_map(&fs); if (fs.m == vm_page_lookup(fs.object, fs.pindex)) { - vm_page_sleep_if_busy(fs.m, "vmpfw"); + vm_page_sleep_if_busy(fs.m, "vmpfw", + VM_ALLOC_NOBUSY, FALSE); } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); Modified: user/attilio/vmobj-fullread/sys/vm/vm_object.c ============================================================================== --- user/attilio/vmobj-fullread/sys/vm/vm_object.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/vm/vm_object.c Sat Jul 20 08:38:34 2013 (r253487) @@ -870,7 +870,8 @@ rescan: np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; - if (vm_page_sleep_if_busy(p, "vpcwai")) { + if (vm_page_sleep_if_busy(p, "vpcwai", VM_ALLOC_NOBUSY, + FALSE)) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) goto rescan; Modified: user/attilio/vmobj-fullread/sys/vm/vm_object.h ============================================================================== --- user/attilio/vmobj-fullread/sys/vm/vm_object.h Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/vm/vm_object.h Sat Jul 20 08:38:34 2013 (r253487) @@ -226,6 +226,8 @@ extern struct vm_object kmem_object_stor rw_assert(&(object)->lock, RA_WLOCKED) #define VM_OBJECT_LOCK_DOWNGRADE(object) \ rw_downgrade(&(object)->lock) +#define VM_OBJECT_LOCK_TRYUPGRADE(object) \ + rw_try_upgrade(&(object)->lock) #define VM_OBJECT_RLOCK(object) \ rw_rlock(&(object)->lock) #define VM_OBJECT_RUNLOCK(object) \ @@ -238,6 +240,8 @@ extern struct vm_object kmem_object_stor rw_try_wlock(&(object)->lock) #define VM_OBJECT_WLOCK(object) \ rw_wlock(&(object)->lock) +#define VM_OBJECT_WOWNED(object) \ + rw_wowned(&(object)->lock) #define VM_OBJECT_WUNLOCK(object) \ rw_wunlock(&(object)->lock) Modified: user/attilio/vmobj-fullread/sys/vm/vm_page.c ============================================================================== --- user/attilio/vmobj-fullread/sys/vm/vm_page.c Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/vm/vm_page.c Sat Jul 20 08:38:34 2013 (r253487) @@ -869,26 +869,53 @@ vm_page_readahead_finish(vm_page_t m) * be locked. */ int -vm_page_sleep_if_busy(vm_page_t m, const char *msg) +vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags, + boolean_t pref) { vm_object_t obj; + int cond, iswowned; vm_page_lock_assert(m, MA_NOTOWNED); - VM_OBJECT_ASSERT_WLOCKED(m->object); - if (vm_page_busy_locked(m)) { + /* + * The page-specific object must be cached because page + * identity can change during the sleep, causing the + * re-lock of a different object. + * It is assumed that a reference to the object is already + * held by the callers. + */ + obj = m->object; + VM_OBJECT_ASSERT_LOCKED(obj); + iswowned = VM_OBJECT_WOWNED(obj); + KASSERT((busyflags & VM_ALLOC_NOBUSY) == 0 || iswowned, + ("vm_page_sleep_if_busy: VM_ALLOC_NOBUSY with read object lock")); + + if ((busyflags & VM_ALLOC_NOBUSY) != 0) { + cond = (busyflags & VM_ALLOC_IGN_RBUSY) != 0 ? + vm_page_busy_wlocked(m) : vm_page_busy_locked(m); + } else if ((busyflags & VM_ALLOC_RBUSY) != 0) + cond = !vm_page_busy_tryrlock(m); + else + cond = !vm_page_busy_trywlock(m); + if (cond) { + /* - * The page-specific object must be cached because page - * identity can change during the sleep, causing the - * re-lock of a different object. - * It is assumed that a reference to the object is already - * held by the callers. + * Some consumers may want to reference the page before + * unlocking and sleeping so that the page daemon is less + * likely to reclaim it. */ - obj = m->object; + if (pref) + vm_page_aflag_set(m, PGA_REFERENCED); vm_page_lock(m); - VM_OBJECT_WUNLOCK(obj); + if (iswowned) + VM_OBJECT_WUNLOCK(obj); + else + VM_OBJECT_RUNLOCK(obj); vm_page_busy_sleep(m, msg); - VM_OBJECT_WLOCK(obj); + if (iswowned) + VM_OBJECT_WLOCK(obj); + else + VM_OBJECT_RLOCK(obj); return (TRUE); } return (FALSE); @@ -2500,44 +2527,44 @@ vm_page_t vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { vm_page_t m; - int sleep; + int origwlock; VM_OBJECT_ASSERT_WLOCKED(object); + origwlock = VM_OBJECT_WOWNED(object); KASSERT((allocflags & VM_ALLOC_RETRY) != 0, ("vm_page_grab: VM_ALLOC_RETRY is required")); KASSERT((allocflags & VM_ALLOC_RBUSY) == 0 || (allocflags & VM_ALLOC_IGN_RBUSY) != 0, ("vm_page_grab: VM_ALLOC_RBUSY/VM_ALLOC_IGN_RBUSY mismatch")); + KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || origwlock != 0, + ("vm_page_grab: VM_ALLOC_NOBUSY with object read lock")); retrylookup: if ((m = vm_page_lookup(object, pindex)) != NULL) { - sleep = (allocflags & VM_ALLOC_IGN_RBUSY) != 0 ? - vm_page_busy_wlocked(m) : vm_page_busy_locked(m); - if (sleep) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_aflag_set(m, PGA_REFERENCED); - vm_page_lock(m); - VM_OBJECT_WUNLOCK(object); - vm_page_busy_sleep(m, "pgrbwt"); - VM_OBJECT_WLOCK(object); + if (vm_page_sleep_if_busy(m, "pgrbwt", allocflags & + (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY | VM_ALLOC_IGN_RBUSY), + TRUE)) goto retrylookup; - } else { + else { if ((allocflags & VM_ALLOC_WIRED) != 0) { vm_page_lock(m); vm_page_wire(m); vm_page_unlock(m); } - if ((allocflags & - (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0) - vm_page_busy_wlock(m); - if ((allocflags & VM_ALLOC_RBUSY) != 0) - vm_page_busy_rlock(m); + + /* + * If the lock state changed in the meanwhile, + * unwind back. + */ + if (VM_OBJECT_WOWNED(object) != origwlock) + VM_OBJECT_LOCK_DOWNGRADE(object); return (m); } } + if (!VM_OBJECT_WOWNED(object) && !VM_OBJECT_LOCK_TRYUPGRADE(object)) { + VM_OBJECT_RUNLOCK(object); + VM_OBJECT_WLOCK(object); + goto retrylookup; + } m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY | VM_ALLOC_IGN_RBUSY)); if (m == NULL) { @@ -2545,7 +2572,12 @@ retrylookup: VM_WAIT; VM_OBJECT_WLOCK(object); goto retrylookup; - } else if (m->valid != 0) + } + + /* If the lock state changed in the meanwhile, unwind back. */ + if (VM_OBJECT_WOWNED(object) != origwlock) + VM_OBJECT_LOCK_DOWNGRADE(object); + if (m->valid != 0) return (m); if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); Modified: user/attilio/vmobj-fullread/sys/vm/vm_page.h ============================================================================== --- user/attilio/vmobj-fullread/sys/vm/vm_page.h Sat Jul 20 08:25:03 2013 (r253486) +++ user/attilio/vmobj-fullread/sys/vm/vm_page.h Sat Jul 20 08:38:34 2013 (r253487) @@ -439,7 +439,8 @@ void vm_page_rename (vm_page_t, vm_objec void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); -int vm_page_sleep_if_busy(vm_page_t m, const char *msg); +int vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags, + boolean_t pref); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307200838.r6K8cYhv073944>