Date: Tue, 15 Oct 2019 03:45:42 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r353539 - in head/sys: amd64/sgx cddl/contrib/opensolaris/uts/common/fs/zfs compat/linuxkpi/common/src dev/drm2/ttm dev/md dev/netmap dev/xen/gntdev dev/xen/privcmd fs/nfsclient fs/smbf... Message-ID: <201910150345.x9F3jgYw028767@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Tue Oct 15 03:45:41 2019 New Revision: 353539 URL: https://svnweb.freebsd.org/changeset/base/353539 Log: (4/6) Protect page valid with the busy lock. Atomics are used for page busy and valid state when the shared busy is held. The details of the locking protocol and valid and dirty synchronization are in the updated vm_page.h comments. Reviewed by: kib, markj Tested by: pho Sponsored by: Netflix, Intel Differential Revision: https://reviews.freebsd.org/D21594 Modified: head/sys/amd64/sgx/sgx.c head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c head/sys/compat/linuxkpi/common/src/linux_compat.c head/sys/dev/drm2/ttm/ttm_bo_vm.c head/sys/dev/drm2/ttm/ttm_tt.c head/sys/dev/md/md.c head/sys/dev/netmap/netmap_freebsd.c head/sys/dev/xen/gntdev/gntdev.c head/sys/dev/xen/privcmd/privcmd.c head/sys/fs/nfsclient/nfs_clbio.c head/sys/fs/smbfs/smbfs_io.c head/sys/fs/tmpfs/tmpfs_subr.c head/sys/kern/kern_exec.c head/sys/kern/uipc_shm.c head/sys/kern/vfs_bio.c head/sys/kern/vfs_cluster.c head/sys/vm/device_pager.c head/sys/vm/phys_pager.c head/sys/vm/sg_pager.c head/sys/vm/swap_pager.c head/sys/vm/vm_fault.c head/sys/vm/vm_map.c head/sys/vm/vm_mmap.c head/sys/vm/vm_object.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h head/sys/vm/vm_pageout.c head/sys/vm/vm_swapout.c head/sys/vm/vnode_pager.c Modified: head/sys/amd64/sgx/sgx.c ============================================================================== --- head/sys/amd64/sgx/sgx.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/amd64/sgx/sgx.c Tue Oct 15 03:45:41 2019 (r353539) @@ -220,8 +220,8 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_obj page = PHYS_TO_VM_PAGE(epc->phys); - vm_page_insert(page, object, idx); page->valid = VM_PAGE_BITS_ALL; + vm_page_insert(page, object, idx); } return (0); @@ -610,8 +610,8 @@ sgx_insert_epc_page_by_index(vm_page_t page, vm_object VM_OBJECT_ASSERT_WLOCKED(object); - vm_page_insert(page, object, pidx); page->valid = VM_PAGE_BITS_ALL; + vm_page_insert(page, object, pidx); } static void Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1731,11 +1731,13 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ db = dbp[0]; for (i = 0; i < *rbehind; i++) { m = vm_page_grab(vmobj, ma[0]->pindex - 1 - i, - VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_NOBUSY); + VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | + VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY); if (m == NULL) break; - if (m->valid != 0) { + if (!vm_page_none_valid(m)) { ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); + vm_page_sunbusy(m); break; } ASSERT(m->dirty == 0); @@ -1746,13 +1748,14 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ va = zfs_map_page(m, &sf); bcopy((char *)db->db_data + bufoff, va, PAGESIZE); zfs_unmap_page(sf); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); vm_page_lock(m); if ((m->busy_lock & VPB_BIT_WAITERS) != 0) vm_page_activate(m); else vm_page_deactivate(m); vm_page_unlock(m); + vm_page_sunbusy(m); } *rbehind = i; @@ -1763,7 +1766,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ m = ma[mi]; if (m != bogus_page) { vm_page_assert_xbusied(m); - ASSERT(m->valid == 0); + ASSERT(vm_page_none_valid(m)); ASSERT(m->dirty == 0); ASSERT(!pmap_page_is_mapped(m)); va = zfs_map_page(m, &sf); @@ -1791,7 +1794,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ if (pgoff == PAGESIZE) { if (m != bogus_page) { zfs_unmap_page(sf); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); } ASSERT(mi < count); mi++; @@ -1840,16 +1843,18 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ ASSERT(m != bogus_page); bzero(va + pgoff, PAGESIZE - pgoff); zfs_unmap_page(sf); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); } for (i = 0; i < *rahead; i++) { m = vm_page_grab(vmobj, ma[count - 1]->pindex + 1 + i, - VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_NOBUSY); + VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | + VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY); if (m == NULL) break; - if (m->valid != 0) { + if (!vm_page_none_valid(m)) { ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); + vm_page_sunbusy(m); break; } ASSERT(m->dirty == 0); @@ -1866,13 +1871,14 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_ bzero(va + tocpy, PAGESIZE - tocpy); } zfs_unmap_page(sf); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); vm_page_lock(m); if ((m->busy_lock & VPB_BIT_WAITERS) != 0) vm_page_activate(m); else vm_page_deactivate(m); vm_page_unlock(m); + vm_page_sunbusy(m); } *rahead = i; zfs_vmobject_wunlock(vmobj); Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Oct 15 03:45:41 2019 (r353539) @@ -534,7 +534,7 @@ mappedread_sf(vnode_t *vp, int nbytes, uio_t *uio) pp = vm_page_grab(obj, OFF_TO_IDX(start), VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); - if (pp->valid == 0) { + if (vm_page_none_valid(pp)) { zfs_vmobject_wunlock(obj); va = zfs_map_page(pp, &sf); error = dmu_read(os, zp->z_id, start, bytes, va, @@ -543,17 +543,16 @@ mappedread_sf(vnode_t *vp, int nbytes, uio_t *uio) bzero(va + bytes, PAGESIZE - bytes); zfs_unmap_page(sf); zfs_vmobject_wlock(obj); - vm_page_sunbusy(pp); - if (error) { - if (!vm_page_busied(pp) && !vm_page_wired(pp) && - pp->valid == 0) - vm_page_free(pp); - } else { - pp->valid = VM_PAGE_BITS_ALL; + if (error == 0) { + vm_page_valid(pp); vm_page_lock(pp); vm_page_activate(pp); vm_page_unlock(pp); } + vm_page_sunbusy(pp); + if (error != 0 && !vm_page_wired(pp) == 0 && + pp->valid == 0 && vm_page_tryxbusy(pp)) + vm_page_free(pp); } else { ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); vm_page_sunbusy(pp); Modified: head/sys/compat/linuxkpi/common/src/linux_compat.c ============================================================================== --- head/sys/compat/linuxkpi/common/src/linux_compat.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/compat/linuxkpi/common/src/linux_compat.c Tue Oct 15 03:45:41 2019 (r353539) @@ -514,7 +514,7 @@ linux_cdev_pager_fault(vm_object_t vm_obj, vm_ooffset_ vm_page_free(*mres); *mres = page; } - page->valid = VM_PAGE_BITS_ALL; + vm_page_valid(page); return (VM_PAGER_OK); } return (VM_PAGER_FAIL); Modified: head/sys/dev/drm2/ttm/ttm_bo_vm.c ============================================================================== --- head/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Oct 15 03:45:41 2019 (r353539) @@ -252,7 +252,7 @@ reserve: ("inconsistent insert bo %p m %p m1 %p offset %jx", bo, m, m1, (uintmax_t)offset)); } - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); if (*mres != NULL) { KASSERT(*mres != m, ("losing %p %p", *mres, m)); vm_page_free(*mres); Modified: head/sys/dev/drm2/ttm/ttm_tt.c ============================================================================== --- head/sys/dev/drm2/ttm/ttm_tt.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/drm2/ttm/ttm_tt.c Tue Oct 15 03:45:41 2019 (r353539) @@ -344,7 +344,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t per continue; to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); pmap_copy_page(from_page, to_page); - to_page->valid = VM_PAGE_BITS_ALL; + vm_page_valid(to_page); vm_page_dirty(to_page); vm_page_xunbusy(to_page); } Modified: head/sys/dev/md/md.c ============================================================================== --- head/sys/dev/md/md.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/md/md.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1074,7 +1074,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp) len = ((i == lastp) ? lastend : PAGE_SIZE) - offs; m = vm_page_grab(sc->object, i, VM_ALLOC_SYSTEM); if (bp->bio_cmd == BIO_READ) { - if (m->valid == VM_PAGE_BITS_ALL) + if (vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, @@ -1090,7 +1090,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp) * can be recreated if thrown out. */ pmap_zero_page(m); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); } if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(&m, offs, bp->bio_ma, @@ -1104,7 +1104,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp) cpu_flush_dcache(p, len); } } else if (bp->bio_cmd == BIO_WRITE) { - if (len == PAGE_SIZE || m->valid == VM_PAGE_BITS_ALL) + if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, @@ -1125,13 +1125,13 @@ mdstart_swap(struct md_s *sc, struct bio *bp) physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); } - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); if (m->dirty != VM_PAGE_BITS_ALL) { vm_page_dirty(m); vm_pager_page_unswapped(m); } } else if (bp->bio_cmd == BIO_DELETE) { - if (len == PAGE_SIZE || m->valid == VM_PAGE_BITS_ALL) + if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, Modified: head/sys/dev/netmap/netmap_freebsd.c ============================================================================== --- head/sys/dev/netmap/netmap_freebsd.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/netmap/netmap_freebsd.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1056,7 +1056,7 @@ netmap_dev_pager_fault(vm_object_t object, vm_ooffset_ *mres = page; vm_page_insert(page, object, pidx); } - page->valid = VM_PAGE_BITS_ALL; + vm_page_valid(page); return (VM_PAGER_OK); } Modified: head/sys/dev/xen/gntdev/gntdev.c ============================================================================== --- head/sys/dev/xen/gntdev/gntdev.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/xen/gntdev/gntdev.c Tue Oct 15 03:45:41 2019 (r353539) @@ -836,8 +836,8 @@ gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t } vm_page_busy_acquire(page, 0); + vm_page_valid(page); vm_page_insert(page, object, pidx); - page->valid = VM_PAGE_BITS_ALL; *mres = page; return (VM_PAGER_OK); } Modified: head/sys/dev/xen/privcmd/privcmd.c ============================================================================== --- head/sys/dev/xen/privcmd/privcmd.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/dev/xen/privcmd/privcmd.c Tue Oct 15 03:45:41 2019 (r353539) @@ -179,8 +179,8 @@ privcmd_pg_fault(vm_object_t object, vm_ooffset_t offs } vm_page_busy_acquire(page, 0); + vm_page_valid(page); vm_page_insert(page, object, pidx); - page->valid = VM_PAGE_BITS_ALL; *mres = page; return (VM_PAGER_OK); } Modified: head/sys/fs/nfsclient/nfs_clbio.c ============================================================================== --- head/sys/fs/nfsclient/nfs_clbio.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/fs/nfsclient/nfs_clbio.c Tue Oct 15 03:45:41 2019 (r353539) @@ -174,7 +174,7 @@ ncl_getpages(struct vop_getpages_args *ap) * XXXGL: is that true for NFS, where short read can occur??? */ VM_OBJECT_WLOCK(object); - if (pages[npages - 1]->valid != 0 && --npages == 0) + if (!vm_page_none_valid(pages[npages - 1]) && --npages == 0) goto out; VM_OBJECT_WUNLOCK(object); @@ -227,14 +227,14 @@ ncl_getpages(struct vop_getpages_args *ap) /* * Read operation filled an entire page */ - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); KASSERT(m->dirty == 0, ("nfs_getpages: page %p is dirty", m)); } else if (size > toff) { /* * Read operation filled a partial page. */ - m->valid = 0; + vm_page_invalid(m); vm_page_set_valid_range(m, 0, size - toff); KASSERT(m->dirty == 0, ("nfs_getpages: page %p is dirty", m)); Modified: head/sys/fs/smbfs/smbfs_io.c ============================================================================== --- head/sys/fs/smbfs/smbfs_io.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/fs/smbfs/smbfs_io.c Tue Oct 15 03:45:41 2019 (r353539) @@ -457,7 +457,7 @@ smbfs_getpages(ap) * XXXGL: is that true for SMB filesystem? */ VM_OBJECT_WLOCK(object); - if (pages[npages - 1]->valid != 0 && --npages == 0) + if (!vm_page_none_valid(pages[npages - 1]) && --npages == 0) goto out; VM_OBJECT_WUNLOCK(object); @@ -505,14 +505,14 @@ smbfs_getpages(ap) /* * Read operation filled an entire page */ - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); KASSERT(m->dirty == 0, ("smbfs_getpages: page %p is dirty", m)); } else if (size > toff) { /* * Read operation filled a partial page. */ - m->valid = 0; + vm_page_invalid(m); vm_page_set_valid_range(m, 0, size - toff); KASSERT(m->dirty == 0, ("smbfs_getpages: page %p is dirty", m)); Modified: head/sys/fs/tmpfs/tmpfs_subr.c ============================================================================== --- head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1408,7 +1408,7 @@ tmpfs_reg_resize(struct vnode *vp, off_t newsize, bool retry: m = vm_page_grab(uobj, idx, VM_ALLOC_NOCREAT); if (m != NULL) { - MPASS(m->valid == VM_PAGE_BITS_ALL); + MPASS(vm_page_all_valid(m)); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); Modified: head/sys/kern/kern_exec.c ============================================================================== --- head/sys/kern/kern_exec.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/kern/kern_exec.c Tue Oct 15 03:45:41 2019 (r353539) @@ -979,11 +979,15 @@ exec_map_first_page(struct image_params *imgp) retry: ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); - if (ma[0]->valid != VM_PAGE_BITS_ALL) { + if (!vm_page_all_valid(ma[0])) { if (vm_page_busy_acquire(ma[0], VM_ALLOC_WAITFAIL) == 0) { vm_page_unwire_noq(ma[0]); goto retry; } + if (vm_page_all_valid(ma[0])) { + vm_page_xunbusy(ma[0]); + goto out; + } if (!vm_pager_has_page(object, 0, NULL, &after)) { if (vm_page_unwire_noq(ma[0])) vm_page_free(ma[0]); @@ -1029,6 +1033,8 @@ retry: for (i = 1; i < initial_pagein; i++) vm_page_readahead_finish(ma[i]); } + +out: VM_OBJECT_WUNLOCK(object); imgp->firstpage = sf_buf_alloc(ma[0], 0); Modified: head/sys/kern/uipc_shm.c ============================================================================== --- head/sys/kern/uipc_shm.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/kern/uipc_shm.c Tue Oct 15 03:45:41 2019 (r353539) @@ -459,7 +459,7 @@ shm_dotruncate_locked(struct shmfd *shmfd, off_t lengt retry: m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT); if (m != NULL) { - MPASS(m->valid == VM_PAGE_BITS_ALL); + MPASS(vm_page_all_valid(m)); } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); @@ -485,7 +485,7 @@ retry: } if (m != NULL) { pmap_zero_page_area(m, base, PAGE_SIZE - base); - KASSERT(m->valid == VM_PAGE_BITS_ALL, + KASSERT(vm_page_all_valid(m), ("shm_dotruncate: page %p is invalid", m)); vm_page_dirty(m); vm_page_xunbusy(m); Modified: head/sys/kern/vfs_bio.c ============================================================================== --- head/sys/kern/vfs_bio.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/kern/vfs_bio.c Tue Oct 15 03:45:41 2019 (r353539) @@ -956,6 +956,12 @@ vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, { VM_OBJECT_ASSERT_LOCKED(m->object); + + /* + * This function and its results are protected by higher level + * synchronization requiring vnode and buf locks to page in and + * validate pages. + */ if (bp->b_flags & B_CACHE) { int base = (foff + off) & PAGE_MASK; if (vm_page_is_valid(m, base, size) == 0) @@ -4640,7 +4646,7 @@ vfs_busy_pages(struct buf *bp, int clear_modify) if (clear_modify) { pmap_remove_write(m); vfs_page_set_validclean(bp, foff, m); - } else if (m->valid == VM_PAGE_BITS_ALL && + } else if (vm_page_all_valid(m) && (bp->b_flags & B_CACHE) == 0) { bp->b_pages[i] = bogus_page; bogus = true; @@ -4681,6 +4687,14 @@ vfs_bio_set_valid(struct buf *bp, int base, int size) n = PAGE_SIZE - (base & PAGE_MASK); VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); + + /* + * Busy may not be strictly necessary here because the pages are + * unlikely to be fully valid and the vnode lock will synchronize + * their access via getpages. It is grabbed for consistency with + * other page validation. + */ + vfs_busy_pages_acquire(bp); for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) { m = bp->b_pages[i]; if (n > size) @@ -4690,6 +4704,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size) size -= n; n = PAGE_SIZE; } + vfs_busy_pages_release(bp); VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } @@ -4717,6 +4732,7 @@ vfs_bio_clrbuf(struct buf *bp) bp->b_flags &= ~B_INVAL; bp->b_ioflags &= ~BIO_ERROR; VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); + vfs_busy_pages_acquire(bp); if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && (bp->b_offset & PAGE_MASK) == 0) { if (bp->b_pages[0] == bogus_page) @@ -4758,6 +4774,7 @@ vfs_bio_clrbuf(struct buf *bp) bp->b_pages[i]->valid |= mask; } unlock: + vfs_busy_pages_release(bp); VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); bp->b_resid = 0; } @@ -5189,7 +5206,7 @@ again: * the end of the function catches the race in a * reliable way (protected by the object lock). */ - if (m->valid == VM_PAGE_BITS_ALL) + if (vm_page_all_valid(m)) continue; poff = IDX_TO_OFF(m->pindex); @@ -5219,7 +5236,7 @@ again: * cache pressure. */ if (buf_pager_relbuf || - m->valid != VM_PAGE_BITS_ALL) + !vm_page_all_valid(m)) bp->b_flags |= B_RELBUF; bp->b_flags &= ~B_NOCACHE; @@ -5229,12 +5246,12 @@ again: } } KASSERT(1 /* racy, enable for debugging */ || - m->valid == VM_PAGE_BITS_ALL || i == count - 1, + vm_page_all_valid(m) || i == count - 1, ("buf %d %p invalid", i, m)); if (i == count - 1 && lpart) { VM_OBJECT_WLOCK(object); - if (m->valid != 0 && - m->valid != VM_PAGE_BITS_ALL) + if (!vm_page_none_valid(m) && + !vm_page_all_valid(m)) vm_page_zero_invalid(m, TRUE); VM_OBJECT_WUNLOCK(object); } @@ -5261,7 +5278,7 @@ end_pages: * invalidated or removed, so we must restart for * safety as well. */ - if (ma[i]->valid != VM_PAGE_BITS_ALL) + if (!vm_page_all_valid(ma[i])) redo = true; } if (redo && error == 0) Modified: head/sys/kern/vfs_cluster.c ============================================================================== --- head/sys/kern/vfs_cluster.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/kern/vfs_cluster.c Tue Oct 15 03:45:41 2019 (r353539) @@ -465,11 +465,13 @@ cluster_rbuild(struct vnode *vp, u_quad_t filesize, da if (toff + tinc > PAGE_SIZE) tinc = PAGE_SIZE - toff; VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object); - if ((tbp->b_pages[j]->valid & - vm_page_bits(toff, tinc)) != 0) - break; if (vm_page_trysbusy(tbp->b_pages[j]) == 0) break; + if ((tbp->b_pages[j]->valid & + vm_page_bits(toff, tinc)) != 0) { + vm_page_sunbusy(tbp->b_pages[j]); + break; + } vm_object_pip_add(tbp->b_bufobj->bo_object, 1); off += tinc; tsize -= tinc; @@ -524,7 +526,7 @@ clean_sbusy: bp->b_pages[bp->b_npages] = m; bp->b_npages++; } - if (m->valid == VM_PAGE_BITS_ALL) + if (vm_page_all_valid(m)) tbp->b_pages[j] = bogus_page; } VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); @@ -548,7 +550,7 @@ clean_sbusy: VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (j = 0; j < bp->b_npages; j++) { VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object); - if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL) + if (vm_page_all_valid(bp->b_pages[j])) bp->b_pages[j] = bogus_page; } VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); Modified: head/sys/vm/device_pager.c ============================================================================== --- head/sys/vm/device_pager.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/device_pager.c Tue Oct 15 03:45:41 2019 (r353539) @@ -395,7 +395,7 @@ old_dev_pager_fault(vm_object_t object, vm_ooffset_t o vm_page_free(*mres); *mres = page; } - page->valid = VM_PAGE_BITS_ALL; + vm_page_valid(page); return (VM_PAGER_OK); } Modified: head/sys/vm/phys_pager.c ============================================================================== --- head/sys/vm/phys_pager.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/phys_pager.c Tue Oct 15 03:45:41 2019 (r353539) @@ -145,12 +145,12 @@ phys_pager_getpages(vm_object_t object, vm_page_t *m, VM_OBJECT_ASSERT_WLOCKED(object); for (i = 0; i < count; i++) { - if (m[i]->valid == 0) { + if (vm_page_none_valid(m[i])) { if ((m[i]->flags & PG_ZERO) == 0) pmap_zero_page(m[i]); - m[i]->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m[i]); } - KASSERT(m[i]->valid == VM_PAGE_BITS_ALL, + KASSERT(vm_page_all_valid(m[i]), ("phys_pager_getpages: partially valid page %p", m[i])); KASSERT(m[i]->dirty == 0, ("phys_pager_getpages: dirty page %p", m[i])); @@ -209,10 +209,8 @@ phys_pager_populate(vm_object_t object, vm_pindex_t pi ahead = MIN(end - i, PHYSALLOC); m = vm_page_grab(object, i, VM_ALLOC_NORMAL | VM_ALLOC_COUNT(ahead)); - if (m->valid != VM_PAGE_BITS_ALL) { + if (!vm_page_all_valid(m)) vm_page_zero_invalid(m, TRUE); - m->valid = VM_PAGE_BITS_ALL; - } KASSERT(m->dirty == 0, ("phys_pager_populate: dirty page %p", m)); } Modified: head/sys/vm/sg_pager.c ============================================================================== --- head/sys/vm/sg_pager.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/sg_pager.c Tue Oct 15 03:45:41 2019 (r353539) @@ -198,7 +198,7 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m, in vm_page_free(m[0]); vm_page_unlock(m[0]); m[0] = page; - page->valid = VM_PAGE_BITS_ALL; + vm_page_valid(page); if (rbehind) *rbehind = 0; Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/swap_pager.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1554,7 +1554,7 @@ swp_pager_async_iodone(struct buf *bp) * be overridden by the original caller of * getpages so don't play cute tricks here. */ - m->valid = 0; + vm_page_invalid(m); } else { /* * If a write error occurs, reactivate page @@ -1582,7 +1582,7 @@ swp_pager_async_iodone(struct buf *bp) KASSERT(m->dirty == 0, ("swp_pager_async_iodone: page %p is dirty", m)); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); if (i < bp->b_pgbefore || i >= bp->b_npages - bp->b_pgafter) vm_page_readahead_finish(m); Modified: head/sys/vm/vm_fault.c ============================================================================== --- head/sys/vm/vm_fault.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/vm_fault.c Tue Oct 15 03:45:41 2019 (r353539) @@ -211,6 +211,7 @@ vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_p return; VM_OBJECT_ASSERT_LOCKED(m->object); + VM_PAGE_OBJECT_BUSY_ASSERT(m); need_dirty = ((fault_type & VM_PROT_WRITE) != 0 && (fault_flags & VM_FAULT_WIRE) == 0) || @@ -285,7 +286,7 @@ vm_fault_soft_fast(struct faultstate *fs, vm_offset_t m = vm_page_lookup(fs->first_object, fs->first_pindex); /* A busy page can be mapped for read|execute access. */ if (m == NULL || ((prot & VM_PROT_WRITE) != 0 && - vm_page_busied(m)) || m->valid != VM_PAGE_BITS_ALL) { + vm_page_busied(m)) || !vm_page_all_valid(m)) { rv = KERN_FAILURE; goto out; } @@ -368,7 +369,7 @@ vm_fault_populate_check_page(vm_page_t m) * valid, and exclusively busied. */ MPASS(m != NULL); - MPASS(m->valid == VM_PAGE_BITS_ALL); + MPASS(vm_page_all_valid(m)); MPASS(vm_page_xbusied(m)); } @@ -830,7 +831,7 @@ RetryFault_oom: * (readable), jump to readrest, else break-out ( we * found the page ). */ - if (fs.m->valid != VM_PAGE_BITS_ALL) + if (!vm_page_all_valid(fs.m)) goto readrest; break; /* break to PAGE HAS BEEN FOUND */ } @@ -1154,7 +1155,7 @@ readrest: VM_CNT_INC(v_ozfod); } VM_CNT_INC(v_zfod); - fs.m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(fs.m); /* Don't try to prefault neighboring pages. */ faultcount = 1; break; /* break to PAGE HAS BEEN FOUND */ @@ -1245,7 +1246,7 @@ readrest: * Oh, well, lets copy it. */ pmap_copy_page(fs.m, fs.first_m); - fs.first_m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(fs.first_m); if (wired && (fault_flags & VM_FAULT_WIRE) == 0) { vm_page_wire(fs.first_m); @@ -1364,7 +1365,7 @@ readrest: * Page must be completely valid or it is not fit to * map into user space. vm_pager_get_pages() ensures this. */ - KASSERT(fs.m->valid == VM_PAGE_BITS_ALL, + KASSERT(vm_page_all_valid(fs.m), ("vm_fault: page %p partially invalid", fs.m)); VM_OBJECT_WUNLOCK(fs.object); @@ -1480,7 +1481,7 @@ vm_fault_dontneed(const struct faultstate *fs, vm_offs entry->start); while ((m = m_next) != NULL && m->pindex < pend) { m_next = TAILQ_NEXT(m, listq); - if (m->valid != VM_PAGE_BITS_ALL || + if (!vm_page_all_valid(m) || vm_page_busied(m)) continue; @@ -1577,7 +1578,7 @@ vm_fault_prefault(const struct faultstate *fs, vm_offs VM_OBJECT_RUNLOCK(lobject); break; } - if (m->valid == VM_PAGE_BITS_ALL && + if (vm_page_all_valid(m) && (m->flags & PG_FICTITIOUS) == 0) pmap_enter_quick(pmap, addr, m, entry->protection); if (!obj_locked || lobject != entry->object.vm_object) @@ -1852,7 +1853,7 @@ again: * all copies of the wired map entry have similar * backing pages. */ - if (dst_m->valid == VM_PAGE_BITS_ALL) { + if (vm_page_all_valid(dst_m)) { pmap_enter(dst_map->pmap, vaddr, dst_m, prot, access | (upgrade ? PMAP_ENTER_WIRED : 0), 0); } Modified: head/sys/vm/vm_map.c ============================================================================== --- head/sys/vm/vm_map.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/vm_map.c Tue Oct 15 03:45:41 2019 (r353539) @@ -2358,7 +2358,7 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_p psize = tmpidx; break; } - if (p->valid == VM_PAGE_BITS_ALL) { + if (vm_page_all_valid(p)) { if (p_start == NULL) { start = addr + ptoa(tmpidx); p_start = p; Modified: head/sys/vm/vm_mmap.c ============================================================================== --- head/sys/vm/vm_mmap.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/vm_mmap.c Tue Oct 15 03:45:41 2019 (r353539) @@ -893,7 +893,7 @@ RestartScan: } } else vm_page_unlock(m); - KASSERT(m->valid == VM_PAGE_BITS_ALL, + KASSERT(vm_page_all_valid(m), ("mincore: page %p is mapped but invalid", m)); } else if (mincoreinfo == 0) { @@ -915,7 +915,7 @@ RestartScan: pindex = OFF_TO_IDX(current->offset + (addr - current->start)); m = vm_page_lookup(object, pindex); - if (m != NULL && m->valid == 0) + if (m != NULL && vm_page_none_valid(m)) m = NULL; if (m != NULL) mincoreinfo = MINCORE_INCORE; Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/vm_object.c Tue Oct 15 03:45:41 2019 (r353539) @@ -841,7 +841,7 @@ rescan: if (pi >= tend) break; np = TAILQ_NEXT(p, listq); - if (p->valid == 0) + if (vm_page_none_valid(p)) continue; if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { if (object->generation != curgeneration) { @@ -1161,10 +1161,10 @@ next_page: } /* - * If the page is not in a normal state, skip it. + * If the page is not in a normal state, skip it. The page + * can not be invalidated while the object lock is held. */ - if (tm->valid != VM_PAGE_BITS_ALL || - vm_page_wired(tm)) + if (!vm_page_all_valid(tm) || vm_page_wired(tm)) goto next_pindex; KASSERT((tm->flags & PG_FICTITIOUS) == 0, ("vm_object_madvise: page %p is fictitious", tm)); @@ -1488,7 +1488,11 @@ vm_object_scan_all_shadowed(vm_object_t object) * object and we might as well give up now. */ pp = vm_page_lookup(object, new_pindex); - if ((pp == NULL || pp->valid == 0) && + /* + * The valid check here is stable due to object lock being + * required to clear valid and initiate paging. + */ + if ((pp == NULL || vm_page_none_valid(pp)) && !vm_pager_has_page(object, new_pindex, NULL, NULL)) return (false); } @@ -1567,7 +1571,7 @@ vm_object_collapse_scan(vm_object_t object, int op) continue; } - KASSERT(pp == NULL || pp->valid != 0, + KASSERT(pp == NULL || !vm_page_none_valid(pp), ("unbusy invalid page %p", pp)); if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL, @@ -1894,7 +1898,7 @@ wired: object->ref_count != 0) pmap_remove_all(p); if ((options & OBJPR_CLEANONLY) == 0) { - p->valid = 0; + vm_page_invalid(p); vm_page_undirty(p); } vm_page_xunbusy(p); @@ -1902,7 +1906,8 @@ wired: } KASSERT((p->flags & PG_FICTITIOUS) == 0, ("vm_object_page_remove: page %p is fictitious", p)); - if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) { + if ((options & OBJPR_CLEANONLY) != 0 && + !vm_page_none_valid(p)) { if ((options & OBJPR_NOTMAPPED) == 0 && object->ref_count != 0 && !vm_page_try_remove_write(p)) Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Tue Oct 15 03:41:36 2019 (r353538) +++ head/sys/vm/vm_page.c Tue Oct 15 03:45:41 2019 (r353539) @@ -1330,7 +1330,7 @@ vm_page_readahead_finish(vm_page_t m) { /* We shouldn't put invalid pages on queues. */ - KASSERT(m->valid != 0, ("%s: %p is invalid", __func__, m)); + KASSERT(!vm_page_none_valid(m), ("%s: %p is invalid", __func__, m)); /* * Since the page is not the actually needed one, whether it should @@ -1430,8 +1430,7 @@ vm_page_dirty_KBI(vm_page_t m) { /* Refer to this operation by its public name. */ - KASSERT(m->valid == VM_PAGE_BITS_ALL, - ("vm_page_dirty: page is invalid!")); + KASSERT(vm_page_all_valid(m), ("vm_page_dirty: page is invalid!")); m->dirty = VM_PAGE_BITS_ALL; } @@ -2657,7 +2656,7 @@ retry: VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, ("page %p has unexpected oflags", m)); /* Don't care: VPO_NOSYNC. */ - if (m->valid != 0) { + if (!vm_page_none_valid(m)) { /* * First, try to allocate a new page * that is above "high". Failing @@ -4334,7 +4333,7 @@ retrylookup: * However, we will not end up with an invalid page and a * shared lock. */ - if (m->valid != VM_PAGE_BITS_ALL || + if (!vm_page_all_valid(m) || (allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) == 0) { sleep = !vm_page_tryxbusy(m); xbusy = true; @@ -4354,7 +4353,7 @@ retrylookup: goto retrylookup; } if ((allocflags & VM_ALLOC_NOCREAT) != 0 && - m->valid != VM_PAGE_BITS_ALL) { + !vm_page_all_valid(m)) { if (xbusy) vm_page_xunbusy(m); else @@ -4364,7 +4363,7 @@ retrylookup: } if ((allocflags & VM_ALLOC_WIRED) != 0) vm_page_wire(m); - if (m->valid == VM_PAGE_BITS_ALL) + if (vm_page_all_valid(m)) goto out; } else if ((allocflags & VM_ALLOC_NOCREAT) != 0) { *mp = NULL; @@ -4386,7 +4385,7 @@ retrylookup: *mp = NULL; return (rv); } - MPASS(m->valid == VM_PAGE_BITS_ALL); + MPASS(vm_page_all_valid(m)); } else { vm_page_zero_invalid(m, TRUE); } @@ -4499,10 +4498,11 @@ retrylookup: goto retrylookup; } } - if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) { + if (vm_page_none_valid(m) && + (allocflags & VM_ALLOC_ZERO) != 0) { if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); - m->valid = VM_PAGE_BITS_ALL; + vm_page_valid(m); } if ((allocflags & VM_ALLOC_NOBUSY) != 0) { if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) @@ -4542,6 +4542,72 @@ vm_page_bits(int base, int size) ((vm_page_bits_t)1 << first_bit)); } +static inline void +vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set) +{ + +#if PAGE_SIZE == 32768 + atomic_set_64((uint64_t *)bits, set); +#elif PAGE_SIZE == 16384 + atomic_set_32((uint32_t *)bits, set); +#elif (PAGE_SIZE == 8192) && defined(atomic_set_16) + atomic_set_16((uint16_t *)bits, set); +#elif (PAGE_SIZE == 4096) && defined(atomic_set_8) + atomic_set_8((uint8_t *)bits, set); +#else /* PAGE_SIZE <= 8192 */ + uintptr_t addr; + int shift; + + addr = (uintptr_t)bits; + /* + * Use a trick to perform a 32-bit atomic on the + * containing aligned word, to not depend on the existence + * of atomic_{set, clear}_{8, 16}. + */ + shift = addr & (sizeof(uint32_t) - 1); +#if BYTE_ORDER == BIG_ENDIAN + shift = (sizeof(uint32_t) - sizeof(vm_page_bits_t) - shift) * NBBY; +#else + shift *= NBBY; +#endif + addr &= ~(sizeof(uint32_t) - 1); + atomic_set_32((uint32_t *)addr, set << shift); +#endif /* PAGE_SIZE */ +} + +static inline void +vm_page_bits_clear(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t clear) +{ + +#if PAGE_SIZE == 32768 + atomic_clear_64((uint64_t *)bits, clear); +#elif PAGE_SIZE == 16384 + atomic_clear_32((uint32_t *)bits, clear); +#elif (PAGE_SIZE == 8192) && defined(atomic_clear_16) + atomic_clear_16((uint16_t *)bits, clear); +#elif (PAGE_SIZE == 4096) && defined(atomic_clear_8) + atomic_clear_8((uint8_t *)bits, clear); +#else /* PAGE_SIZE <= 8192 */ + uintptr_t addr; + int shift; + + addr = (uintptr_t)bits; + /* + * Use a trick to perform a 32-bit atomic on the + * containing aligned word, to not depend on the existence + * of atomic_{set, clear}_{8, 16}. + */ + shift = addr & (sizeof(uint32_t) - 1); +#if BYTE_ORDER == BIG_ENDIAN + shift = (sizeof(uint32_t) - sizeof(vm_page_bits_t) - shift) * NBBY; +#else + shift *= NBBY; +#endif + addr &= ~(sizeof(uint32_t) - 1); + atomic_clear_32((uint32_t *)addr, clear << shift); +#endif /* PAGE_SIZE */ +} + /* * vm_page_set_valid_range: * @@ -4556,8 +4622,9 @@ void vm_page_set_valid_range(vm_page_t m, int base, int size) { int endoff, frag; + vm_page_bits_t pagebits; - VM_OBJECT_ASSERT_WLOCKED(m->object); + vm_page_assert_busied(m); if (size == 0) /* handle degenerate case */ return; @@ -4591,7 +4658,11 @@ vm_page_set_valid_range(vm_page_t m, int base, int siz /* * Set valid bits inclusive of any overlap. */ - m->valid |= vm_page_bits(base, size); + pagebits = vm_page_bits(base, size); + if (vm_page_xbusied(m)) + m->valid |= pagebits; + else + vm_page_bits_set(m, &m->valid, pagebits); } /* @@ -4600,52 +4671,20 @@ vm_page_set_valid_range(vm_page_t m, int base, int siz static __inline void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits) { - uintptr_t addr; -#if PAGE_SIZE < 16384 - int shift; -#endif + vm_page_assert_busied(m); + /* - * If the object is locked and the page is neither exclusive busy nor - * write mapped, then the page's dirty field cannot possibly be - * set by a concurrent pmap operation. + * If the page is xbusied and not write mapped we are the + * only thread that can modify dirty bits. Otherwise, The pmap + * layer can call vm_page_dirty() without holding a distinguished + * lock. The combination of page busy and atomic operations + * suffice to guarantee consistency of the page dirty field. */ - VM_OBJECT_ASSERT_WLOCKED(m->object); - if (!vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) + if (vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) m->dirty &= ~pagebits; - else { - /* - * The pmap layer can call vm_page_dirty() without - * holding a distinguished lock. The combination of - * the object's lock and an atomic operation suffice - * to guarantee consistency of the page dirty field. - * - * For PAGE_SIZE == 32768 case, compiler already - * properly aligns the dirty field, so no forcible - * alignment is needed. Only require existence of - * atomic_clear_64 when page size is 32768. - */ - addr = (uintptr_t)&m->dirty; -#if PAGE_SIZE == 32768 - atomic_clear_64((uint64_t *)addr, pagebits); -#elif PAGE_SIZE == 16384 - atomic_clear_32((uint32_t *)addr, pagebits); -#else /* PAGE_SIZE <= 8192 */ - /* - * Use a trick to perform a 32-bit atomic on the *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910150345.x9F3jgYw028767>