From owner-svn-src-head@freebsd.org Mon Nov 11 16:59:50 2019 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 782091B98D4; Mon, 11 Nov 2019 16:59:50 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47BcYf2XWfz4NS9; Mon, 11 Nov 2019 16:59:50 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 3A66E1A5ED; Mon, 11 Nov 2019 16:59:50 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xABGxoQd075507; Mon, 11 Nov 2019 16:59:50 GMT (envelope-from dougm@FreeBSD.org) Received: (from dougm@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xABGxoND075506; Mon, 11 Nov 2019 16:59:50 GMT (envelope-from dougm@FreeBSD.org) Message-Id: <201911111659.xABGxoND075506@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: dougm set sender to dougm@FreeBSD.org using -f From: Doug Moore Date: Mon, 11 Nov 2019 16:59:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r354618 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: dougm X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 354618 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 11 Nov 2019 16:59:50 -0000 Author: dougm Date: Mon Nov 11 16:59:49 2019 New Revision: 354618 URL: https://svnweb.freebsd.org/changeset/base/354618 Log: swap_pager_meta_free() frees allocated blocks in a way that exploits the sparsity of allocated blocks in a range, without issuing an "are you there?" query for every block in the range. swap_pager_copy() is not so smart. Modify the implementation of swap_pager_meta_free() slightly so that swap_pager_copy() can use that smarter implementation too. Based on an observation of: Yoshihiro Ota (ota_j.email.ne.jp) Reviewed by: kib,alc Tested by: pho Differential Revision: https://reviews.freebsd.org/D22280 Modified: head/sys/vm/swap_pager.c Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Mon Nov 11 16:11:15 2019 (r354617) +++ head/sys/vm/swap_pager.c Mon Nov 11 16:59:49 2019 (r354618) @@ -422,6 +422,8 @@ static daddr_t swp_pager_getswapspace(int *npages, int */ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); +static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, + vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); @@ -933,6 +935,33 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t sta return (0); } +static bool +swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject, + vm_pindex_t pindex, daddr_t addr) +{ + daddr_t dstaddr; + + if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) { + /* Caller should destroy the source block. */ + return (false); + } + + /* + * Destination has no swapblk and is not resident, transfer source. + * swp_pager_meta_build() can sleep. + */ + vm_object_pip_add(srcobject, 1); + VM_OBJECT_WUNLOCK(srcobject); + vm_object_pip_add(dstobject, 1); + dstaddr = swp_pager_meta_build(dstobject, pindex, addr); + KASSERT(dstaddr == SWAPBLK_NONE, + ("Unexpected destination swapblk")); + vm_object_pip_wakeup(dstobject); + VM_OBJECT_WLOCK(srcobject); + vm_object_pip_wakeup(srcobject); + return (true); +} + /* * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager * and destroy the source. @@ -956,8 +985,6 @@ void swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t offset, int destroysource) { - vm_pindex_t i; - daddr_t dstaddr, n_free, s_free, srcaddr; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -984,39 +1011,8 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dst /* * Transfer source to destination. */ - swp_pager_init_freerange(&s_free, &n_free); - for (i = 0; i < dstobject->size; ++i) { - srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); - if (srcaddr == SWAPBLK_NONE) - continue; - dstaddr = swp_pager_meta_ctl(dstobject, i, 0); - if (dstaddr != SWAPBLK_NONE) { - /* - * Destination has valid swapblk or it is represented - * by a resident page. We destroy the source block. - */ - swp_pager_update_freerange(&s_free, &n_free, srcaddr); - continue; - } + swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size); - /* - * Destination has no swapblk and is not resident, - * copy source. - * - * swp_pager_meta_build() can sleep. - */ - vm_object_pip_add(srcobject, 1); - VM_OBJECT_WUNLOCK(srcobject); - vm_object_pip_add(dstobject, 1); - dstaddr = swp_pager_meta_build(dstobject, i, srcaddr); - KASSERT(dstaddr == SWAPBLK_NONE, - ("Unexpected destination swapblk")); - vm_object_pip_wakeup(dstobject); - VM_OBJECT_WLOCK(srcobject); - vm_object_pip_wakeup(srcobject); - } - swp_pager_freeswapspace(s_free, n_free); - /* * Free left over swap blocks in source. * @@ -2003,31 +1999,30 @@ allocated: } /* - * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata + * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap + * metadata, or transfer it into dstobject. * - * The requested range of blocks is freed, with any associated swap - * returned to the swap bitmap. - * * This routine will free swap metadata structures as they are cleaned - * out. This routine does *NOT* operate on swap metadata associated - * with resident pages. + * out. */ static void -swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) +swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, + vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; daddr_t n_free, s_free; - vm_pindex_t last; + vm_pindex_t offset, last; int i, limit, start; - VM_OBJECT_ASSERT_WLOCKED(object); - if (object->type != OBJT_SWAP || count == 0) + VM_OBJECT_ASSERT_WLOCKED(srcobject); + if (srcobject->type != OBJT_SWAP || count == 0) return; swp_pager_init_freerange(&s_free, &n_free); + offset = pindex; last = pindex + count; for (;;) { - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); if (sb == NULL || sb->p >= last) break; @@ -2037,18 +2032,39 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pi for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); + if (dstobject == NULL || + !swp_pager_xfer_source(srcobject, dstobject, + sb->p + i - offset, sb->d[i])) { + swp_pager_update_freerange(&s_free, &n_free, + sb->d[i]); + } sb->d[i] = SWAPBLK_NONE; } pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, + SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } } swp_pager_freeswapspace(s_free, n_free); +} + +/* + * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata + * + * The requested range of blocks is freed, with any associated swap + * returned to the swap bitmap. + * + * This routine will free swap metadata structures as they are cleaned + * out. This routine does *NOT* operate on swap metadata associated + * with resident pages. + */ +static void +swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) +{ + swp_pager_meta_transfer(object, NULL, pindex, count); } /*