Date: Fri, 2 Mar 2018 18:12:25 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r330288 - user/markj/vm-playground/sys/vm Message-ID: <201803021812.w22ICPur037409@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Fri Mar 2 18:12:25 2018 New Revision: 330288 URL: https://svnweb.freebsd.org/changeset/base/330288 Log: Add vm_page_alloc_pages_after(). This is a new page allocation which is intended to complement vm_page_grab_pages(). It permits the allocation of multiple pages, contiguous in the pindex space, with a single call. When VM_ALLOC_{NOWAIT,WAITFAIL} are specified, the returned run may be shorter than the one requested. In support of this function, vm_reserv_extend() and vm_reserv_alloc_page() may now optionally return a run of contiguous pages from the same reservation, and the new vm_phys_alloc_npages() function is used to allocate pages from the physical memory allocator. Modified: user/markj/vm-playground/sys/vm/swap_pager.c user/markj/vm-playground/sys/vm/vm_page.c user/markj/vm-playground/sys/vm/vm_page.h user/markj/vm-playground/sys/vm/vm_pagequeue.h user/markj/vm-playground/sys/vm/vm_reserv.c user/markj/vm-playground/sys/vm/vm_reserv.h user/markj/vm-playground/sys/vm/vnode_pager.c Modified: user/markj/vm-playground/sys/vm/swap_pager.c ============================================================================== --- user/markj/vm-playground/sys/vm/swap_pager.c Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/swap_pager.c Fri Mar 2 18:12:25 2018 (r330288) @@ -1097,10 +1097,10 @@ swap_pager_getpages(vm_object_t object, vm_page_t *ma, int *rahead) { struct buf *bp; - vm_page_t mpred, msucc, p; + vm_page_t mpred, msucc; vm_pindex_t pindex; daddr_t blk; - int i, j, maxahead, maxbehind, reqcount, shift; + int i, maxahead, maxbehind, reqcount, shift; reqcount = count; @@ -1136,39 +1136,27 @@ swap_pager_getpages(vm_object_t object, vm_page_t *ma, /* * Allocate readahead and readbehind pages. */ - shift = rbehind != NULL ? *rbehind : 0; - if (shift != 0) { - for (i = 1; i <= shift; i++) { - p = vm_page_alloc(object, ma[0]->pindex - i, - VM_ALLOC_NORMAL); - if (p == NULL) { - /* Shift allocated pages to the left. */ - for (j = 0; j < i - 1; j++) - bp->b_pages[j] = - bp->b_pages[j + shift - i + 1]; - break; - } - bp->b_pages[shift - i] = p; - } - shift = i - 1; - *rbehind = shift; - } + if (rbehind != NULL && *rbehind > 0) { + shift = vm_page_alloc_pages_after(object, + ma[0]->pindex - *rbehind, VM_ALLOC_NORMAL, &bp->b_pages[0], + *rbehind, mpred); + if (shift != *rbehind) { + /* Drop a partially allocated run. */ + for (i = 0; i < shift; i++) + vm_page_free(bp->b_pages[i]); + shift = *rbehind = 0; + } else + count += *rbehind; + } else + shift = 0; for (i = 0; i < reqcount; i++) bp->b_pages[i + shift] = ma[i]; - if (rahead != NULL) { - for (i = 0; i < *rahead; i++) { - p = vm_page_alloc(object, - ma[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL); - if (p == NULL) - break; - bp->b_pages[shift + reqcount + i] = p; - } - *rahead = i; - } - if (rbehind != NULL) - count += *rbehind; - if (rahead != NULL) + if (rahead != NULL && *rahead > 0) { + *rahead = vm_page_alloc_pages_after(object, + ma[reqcount - 1]->pindex + 1, VM_ALLOC_NORMAL, + &bp->b_pages[reqcount + shift], *rahead, ma[reqcount - 1]); count += *rahead; + } vm_object_pip_add(object, count); Modified: user/markj/vm-playground/sys/vm/vm_page.c ============================================================================== --- user/markj/vm-playground/sys/vm/vm_page.c Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vm_page.c Fri Mar 2 18:12:25 2018 (r330288) @@ -1696,9 +1696,10 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pi * for the request class and false otherwise. */ int -vm_domain_allocate(struct vm_domain *vmd, int req, int npages) +vm_domain_allocate(struct vm_domain *vmd, int req, int npages, bool partial) { u_int limit, old, new; + int avail; req = req & VM_ALLOC_CLASS_MASK; @@ -1707,6 +1708,7 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int */ if (curproc == pageproc && req != VM_ALLOC_INTERRUPT) req = VM_ALLOC_SYSTEM; + if (req == VM_ALLOC_INTERRUPT) limit = 0; else if (req == VM_ALLOC_SYSTEM) @@ -1719,9 +1721,12 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int */ do { old = vmd->vmd_free_count; - new = old - npages; - if (new < limit) + if (old <= limit) return (0); + avail = min(old - limit, (u_int)npages); + if (avail != npages && !partial) + return (0); + new = old - avail; } while (atomic_cmpset_int(&vmd->vmd_free_count, old, new) == 0); /* Wake the page daemon if we've crossed the threshold. */ @@ -1733,7 +1738,7 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int (old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe)) vm_domain_set(vmd); - return (1); + return (avail); } vm_page_t @@ -1764,21 +1769,22 @@ again: * Can we allocate the page from a reservation? */ if (vm_object_reserv(object) && - ((m = vm_reserv_extend(req, object, pindex, domain, mpred)) != NULL || - (m = vm_reserv_alloc_page(req, object, pindex, domain, mpred)) != NULL)) { + ((m = vm_reserv_extend(req, object, pindex, domain, mpred, NULL)) != + NULL || + (m = vm_reserv_alloc_page(req, object, pindex, domain, mpred, + NULL)) != NULL)) { domain = vm_phys_domain(m); vmd = VM_DOMAIN(domain); goto found; } #endif vmd = VM_DOMAIN(domain); - if (object != NULL && !vm_object_reserv(object) && - vmd->vmd_pgcache != NULL) { + if (!vm_object_reserv(object) && vmd->vmd_pgcache != NULL) { m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); if (m != NULL) goto found; } - if (vm_domain_allocate(vmd, req, 1)) { + if (vm_domain_allocate(vmd, req, 1, false) == 1) { /* * If not, allocate it from the free page queues. */ @@ -1828,7 +1834,7 @@ found: m->busy_lock = VPB_SINGLE_EXCLUSIVER; if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); - if (req & VM_ALLOC_WIRED) { + if ((req & VM_ALLOC_WIRED) != 0) { /* * The page lock is not required for wiring a page until that * page is inserted into the object. @@ -1869,7 +1875,191 @@ found: return (m); } +int +vm_page_alloc_pages_after(vm_object_t object, vm_pindex_t pindex, int req, + vm_page_t *ma, int nreq, vm_page_t mpred) +{ + struct vm_domainset_iter di; + int domain, n; + + vm_domainset_iter_page_init(&di, object, &domain, &req); + do { + n = vm_page_alloc_pages_domain_after(object, pindex, domain, + req, ma, nreq, mpred); + if (n > 0) + break; + } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + + return (n); +} + /* + * vm_page_alloc_pages_after: + * + * Allocate a range of pages, contiguous in the pindex space. The + * number of pages actually allocated is returned and may be smaller + * than the number requested unless VM_ALLOC_WAITOK is specified. + * This function is otherwise identical to vm_page_alloc(). + */ +int +vm_page_alloc_pages_domain_after(vm_object_t object, vm_pindex_t pindex, + int domain, int req, vm_page_t *ma, int nreq, vm_page_t mpred) +{ + struct vm_domain *vmd; + vm_page_t m; + int avail, i, nalloc, pool; + u_int busy_lock, flags, oflags; + + KASSERT(nreq > 0, ("invalid nreq %d", nreq)); + KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && + (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && + ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != + (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), + ("inconsistent object(%p)/req(%x)", object, req)); + KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, + ("Can't sleep and retry object insertion.")); + KASSERT(mpred == NULL || mpred->pindex < pindex, + ("mpred %p doesn't precede pindex 0x%jx", mpred, + (uintmax_t)pindex)); + if (object != NULL) + VM_OBJECT_ASSERT_WLOCKED(object); + + nalloc = 0; + +#if VM_NRESERVLEVEL > 0 + if (vm_object_reserv(object)) { + avail = nreq; + m = vm_reserv_extend(req, object, pindex, domain, mpred, + &avail); + if (m == NULL) + m = vm_reserv_alloc_page(req, object, pindex, domain, + mpred, &avail); + if (m != NULL) { + domain = vm_phys_domain(m); + while (nalloc < avail) + ma[nalloc++] = m++; + + /* + * We might have gotten a short run back because we + * reached the end of a reservation. If so, declare + * success now rather than trying to fill the rest of + * the array, in the hope that a subsequent allocation + * attempt will allocate a new reservation. + */ + if (nalloc == nreq || (req & VM_ALLOC_WAITOK) == 0) + goto done; + } + } +#endif + +again: + vmd = VM_DOMAIN(domain); + if ((avail = vm_domain_allocate(vmd, req, nreq - nalloc, true)) > 0) { + pool = object != NULL ? VM_FREEPOOL_DEFAULT : + VM_FREEPOOL_DIRECT; + vm_domain_free_lock(vmd); + do { + i = vm_phys_alloc_npages(domain, pool, &m, + avail - nalloc); + if (i == 0) { + vm_domain_freecnt_inc(vmd, avail - nalloc); + break; + } + for (; i > 0; i--) + ma[nalloc++] = m++; + } while (nalloc < avail); + vm_domain_free_unlock(vmd); + } + if (nalloc == 0 || (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) { +#if VM_NRESERVLEVEL > 0 + if (vm_reserv_reclaim_inactive(domain)) + goto again; +#endif + + /* + * We failed to allocate at least one page, or the caller + * requested a blocking allocation and we weren't able to + * scrounge enough pages in the latest attempt. + */ + if (vm_domain_alloc_fail(vmd, object, req)) + goto again; + return (0); + } + +done: + for (i = 0; i < nalloc; i++) + vm_page_alloc_check(ma[i]); + + /* + * Initialize the pages. Only the PG_ZERO flag is inherited. + */ + flags = 0; + if ((req & VM_ALLOC_ZERO) != 0) + flags |= PG_ZERO; + if ((req & VM_ALLOC_NODUMP) != 0) + flags |= PG_NODUMP; + oflags = (object == NULL || (object->flags & OBJ_UNMANAGED) != 0) ? + VPO_UNMANAGED : 0; + busy_lock = VPB_UNBUSIED; + if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) + busy_lock = VPB_SINGLE_EXCLUSIVER; + if ((req & VM_ALLOC_SBUSY) != 0) + busy_lock = VPB_SHARERS_WORD(1); + + for (i = 0; i < nalloc; i++) { + m = ma[i]; + + m->flags = (m->flags | PG_NODUMP) & flags; + m->aflags = 0; + m->oflags = oflags; + m->busy_lock = busy_lock; + if ((req & VM_ALLOC_WIRED) != 0) { + /* + * The page lock is not required for wiring a page + * until that page is inserted into the object. + */ + m->wire_count = 1; + } + m->act_count = 0; + + if (object != NULL) { + if (vm_page_insert_after(m, object, pindex + i, + mpred)) { + avail = i; + for (; i < nalloc; i++) { + m = ma[i]; + m->busy_lock = VPB_UNBUSIED; + m->oflags = VPO_UNMANAGED; + m->wire_count = 0; + KASSERT(m->object == NULL, + ("page %p has object", m)); + /* Don't change PG_ZERO. */ + vm_page_free_toq(m); + } + if ((req & VM_ALLOC_WAITFAIL) != 0) { + VM_OBJECT_WUNLOCK(object); + vm_radix_wait(); + VM_OBJECT_WLOCK(object); + } + nalloc = avail; + break; + } + + /* Ignore device objects; the pager sets "memattr" for them. */ + if (object->memattr != VM_MEMATTR_DEFAULT && + (object->flags & OBJ_FICTITIOUS) == 0) + pmap_page_set_memattr(m, object->memattr); + } else + m->pindex = pindex + i; + mpred = m; + } + if ((req & VM_ALLOC_WIRED) != 0) + VM_CNT_ADD(v_wire_count, nalloc); + + return (nalloc); +} + +/* * vm_page_alloc_contig: * * Allocate a contiguous set of physical pages of the given size "npages" @@ -1981,7 +2171,7 @@ again: #endif m_ret = NULL; vmd = VM_DOMAIN(domain); - if (vm_domain_allocate(vmd, req, npages)) { + if (vm_domain_allocate(vmd, req, npages, false) == npages) { /* * allocate them from the free page queues. */ @@ -2139,7 +2329,7 @@ vm_page_alloc_freelist_domain(int domain, int freelist */ vmd = VM_DOMAIN(domain); again: - if (vm_domain_allocate(vmd, req, 1)) { + if (vm_domain_allocate(vmd, req, 1, false) == 1) { vm_domain_free_lock(vmd); m = vm_phys_alloc_freelist_pages(domain, freelist, VM_FREEPOOL_DIRECT, 0); @@ -2191,7 +2381,7 @@ vm_page_import(void *arg, void **store, int cnt, int d MIN(n, cnt-i)); if (n == 0) break; - if (!vm_domain_allocate(vmd, VM_ALLOC_NORMAL, n)) { + if (vm_domain_allocate(vmd, VM_ALLOC_NORMAL, n, false) == 0) { vm_phys_free_contig(m, n); break; } @@ -3189,14 +3379,14 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked) if ((m->oflags & VPO_UNMANAGED) == 0) { vm_page_lock_assert(m, MA_OWNED); KASSERT(!pmap_page_is_mapped(m), - ("vm_page_free_toq: freeing mapped page %p", m)); + ("vm_page_free_prep: freeing mapped page %p", m)); } else KASSERT(m->queue == PQ_NONE, - ("vm_page_free_toq: unmanaged page %p is queued", m)); + ("vm_page_free_prep: unmanaged page %p is queued", m)); VM_CNT_INC(v_tfree); if (vm_page_sbusied(m)) - panic("vm_page_free: freeing busy page %p", m); + panic("vm_page_free_prep: freeing busy page %p", m); vm_page_remove(m); @@ -3222,11 +3412,11 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked) vm_page_undirty(m); if (m->wire_count != 0) - panic("vm_page_free: freeing wired page %p", m); + panic("vm_page_free_prep: freeing wired page %p", m); if (m->hold_count != 0) { m->flags &= ~PG_ZERO; KASSERT((m->flags & PG_UNHOLDFREE) == 0, - ("vm_page_free: freeing PG_UNHOLDFREE page %p", m)); + ("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m)); m->flags |= PG_UNHOLDFREE; return (false); } @@ -3703,9 +3893,8 @@ int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count) { - vm_page_t m, mpred; - int pflags; - int i; + vm_page_t m, mpred, msucc; + int i, pflags, run; bool sleep; VM_OBJECT_ASSERT_WLOCKED(object); @@ -3717,6 +3906,7 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pin KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || (allocflags & VM_ALLOC_IGN_SBUSY) != 0, ("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch")); + if (count == 0) return (0); pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | @@ -3728,10 +3918,14 @@ retrylookup: m = vm_radix_lookup_le(&object->rtree, pindex + i); if (m == NULL || m->pindex != pindex + i) { mpred = m; + msucc = mpred != NULL ? TAILQ_NEXT(mpred, listq) : + TAILQ_FIRST(&object->memq); m = NULL; - } else + } else { mpred = TAILQ_PREV(m, pglist, listq); - for (; i < count; i++) { + msucc = TAILQ_NEXT(m, listq); + } + while (i < count) { if (m != NULL) { sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ? vm_page_xbusied(m) : vm_page_busied(m); @@ -3761,21 +3955,41 @@ retrylookup: vm_page_xbusy(m); if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); + if (m->valid == 0 && + (allocflags & VM_ALLOC_ZERO) != 0) { + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } + ma[i++] = m; } else { - m = vm_page_alloc_after(object, pindex + i, - pflags | VM_ALLOC_COUNT(count - i), mpred); - if (m == NULL) { + /* + * Try to allocate multiple consecutive pages. Use the + * succeeding page, if any, to bound the length of the + * requested run. + */ + run = msucc == NULL || msucc->pindex >= pindex + count ? + count - i : msucc->pindex - (pindex + i); + run = vm_page_alloc_pages_after(object, pindex + i, + pflags | VM_ALLOC_COUNT(run), ma + i, run, mpred); + if (run == 0) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) break; goto retrylookup; } + if ((allocflags & VM_ALLOC_ZERO) != 0) { + for (; run != 0; run--, i++) { + m = ma[i]; + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } + } else + i += run; + m = ma[i - 1]; } - if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) { - if ((m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - m->valid = VM_PAGE_BITS_ALL; - } - ma[i] = mpred = m; + mpred = m; + msucc = TAILQ_NEXT(m, listq); m = vm_page_next(m); } return (i); Modified: user/markj/vm-playground/sys/vm/vm_page.h ============================================================================== --- user/markj/vm-playground/sys/vm/vm_page.h Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vm_page.h Fri Mar 2 18:12:25 2018 (r330288) @@ -467,6 +467,10 @@ vm_page_t vm_page_alloc_domain(vm_object_t, vm_pindex_ vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t); vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int, vm_page_t); +int vm_page_alloc_pages_after(vm_object_t, vm_pindex_t, int, vm_page_t *, int, + vm_page_t); +int vm_page_alloc_pages_domain_after(vm_object_t, vm_pindex_t, int, int, + vm_page_t *, int, vm_page_t); vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); Modified: user/markj/vm-playground/sys/vm/vm_pagequeue.h ============================================================================== --- user/markj/vm-playground/sys/vm/vm_pagequeue.h Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vm_pagequeue.h Fri Mar 2 18:12:25 2018 (r330288) @@ -183,7 +183,8 @@ vm_pagequeue_cnt_add(struct vm_pagequeue *pq, int adde void vm_domain_set(struct vm_domain *vmd); void vm_domain_clear(struct vm_domain *vmd); -int vm_domain_allocate(struct vm_domain *vmd, int req, int npages); +int vm_domain_allocate(struct vm_domain *vmd, int req, int npages, + bool partial); /* * vm_pagequeue_domain: Modified: user/markj/vm-playground/sys/vm/vm_reserv.c ============================================================================== --- user/markj/vm-playground/sys/vm/vm_reserv.c Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vm_reserv.c Fri Mar 2 18:12:25 2018 (r330288) @@ -643,7 +643,7 @@ vm_reserv_extend_contig(int req, vm_object_t object, v if (popmap_is_set(rv->popmap, index + i)) goto out; } - if (!vm_domain_allocate(vmd, req, npages)) + if (vm_domain_allocate(vmd, req, npages, false) == 0) goto out; for (i = 0; i < npages; i++) vm_reserv_populate(rv, index + i); @@ -792,7 +792,7 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm */ m = NULL; vmd = VM_DOMAIN(domain); - if (vm_domain_allocate(vmd, req, allocpages)) { + if (vm_domain_allocate(vmd, req, allocpages, false) == allocpages) { vm_domain_free_lock(vmd); m = vm_phys_alloc_contig(domain, allocpages, low, high, ulmax(alignment, VM_LEVEL_0_SIZE), @@ -839,8 +839,10 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm } /* - * Attempts to extend an existing reservation and allocate the page to the - * object. + * Attempts to extend an existing reservation and allocate the request page to + * the object. Opportunistically returns up to "*countp" contiguous pages if + * the caller so requests. The number of pages allocated is returned in + * "*countp". * * The page "mpred" must immediately precede the offset "pindex" within the * specified object. @@ -849,12 +851,12 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm */ vm_page_t vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain, - vm_page_t mpred) + vm_page_t mpred, int *countp) { struct vm_domain *vmd; vm_page_t m, msucc; vm_reserv_t rv; - int index; + int avail, index, nalloc; VM_OBJECT_ASSERT_WLOCKED(object); @@ -886,10 +888,30 @@ vm_reserv_extend(int req, vm_object_t object, vm_pinde m = NULL; goto out; } - if (vm_domain_allocate(vmd, req, 1) == 0) - m = NULL; - else + + /* + * If the caller is prepared to accept multiple pages, try to allocate + * them. We are constrained by: + * 1) the number of pages the caller can accept, + * 2) the number of free pages in the reservation succeeding "index", + * 3) the number of available free pages in the domain. + */ + nalloc = countp != NULL ? imin(VM_LEVEL_0_NPAGES - index, *countp) : 1; + if ((avail = vm_domain_allocate(vmd, req, nalloc, true)) > 0) { vm_reserv_populate(rv, index); + if (countp != NULL) { + for (nalloc = 1; nalloc < avail; nalloc++) { + if (popmap_is_set(rv->popmap, ++index)) + break; + vm_reserv_populate(rv, index); + } + if (nalloc < avail) + /* Return leftover pages. */ + vm_domain_freecnt_inc(vmd, avail - nalloc); + *countp = nalloc; + } + } else + m = NULL; out: vm_reserv_unlock(rv); @@ -897,22 +919,25 @@ out: } /* - * Allocates a page from an existing reservation. + * Allocates a new reservation for the object, and returns a page from that + * reservation. Opportunistically returns up to *"countp" contiguous pages if + * the caller so requests. The number of pages allocated is returned in + * "*countp". * * The page "mpred" must immediately precede the offset "pindex" within the * specified object. * - * The object and free page queue must be locked. + * The object and per-domain free page queues must be locked. */ vm_page_t vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, int domain, - vm_page_t mpred) + vm_page_t mpred, int *countp) { struct vm_domain *vmd; vm_page_t m, msucc; vm_pindex_t first, leftcap, rightcap; vm_reserv_t rv; - int index; + int avail, index, nalloc; VM_OBJECT_ASSERT_WLOCKED(object); @@ -981,16 +1006,24 @@ vm_reserv_alloc_page(int req, vm_object_t object, vm_p /* * Allocate and populate the new reservation. + * + * If the caller is prepared to accept multiple pages, try to allocate + * them. We are constrained by: + * 1) the number of pages the caller can accept, + * 2) the number of free pages in the reservation succeeding "index", + * 3) the number of available free pages in the domain. */ + index = VM_RESERV_INDEX(object, pindex); m = NULL; + nalloc = countp != NULL ? imin(VM_LEVEL_0_NPAGES - index, *countp) : 1; vmd = VM_DOMAIN(domain); - if (vm_domain_allocate(vmd, req, 1)) { + if ((avail = vm_domain_allocate(vmd, req, nalloc, true)) > 0) { vm_domain_free_lock(vmd); m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER); vm_domain_free_unlock(vmd); if (m == NULL) { - vm_domain_freecnt_inc(vmd, 1); + vm_domain_freecnt_inc(vmd, avail); return (NULL); } } else @@ -1000,11 +1033,22 @@ vm_reserv_alloc_page(int req, vm_object_t object, vm_p KASSERT(rv->pages == m, ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); vm_reserv_insert(rv, object, first); - index = VM_RESERV_INDEX(object, pindex); vm_reserv_populate(rv, index); + m = &rv->pages[index]; + if (countp != NULL) { + for (nalloc = 1; nalloc < avail; nalloc++) { + if (popmap_is_set(rv->popmap, ++index)) + break; + vm_reserv_populate(rv, index); + } + if (nalloc < avail) + /* Return leftover pages. */ + vm_domain_freecnt_inc(vmd, avail - nalloc); + *countp = nalloc; + } vm_reserv_unlock(rv); - return (&rv->pages[index]); + return (m); } /* @@ -1227,15 +1271,16 @@ vm_reserv_reclaim(vm_reserv_t rv) /* * Breaks the reservation at the head of the partially populated reservation - * queue, releasing its free pages to the physical memory allocator. Returns - * TRUE if a reservation is broken and FALSE otherwise. + * queue, releasing its free pages to the physical memory allocator, and + * returns the number of pages released. * * The free page queue lock must be held. */ -boolean_t +int vm_reserv_reclaim_inactive(int domain) { vm_reserv_t rv; + int freed; while ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) { vm_reserv_lock(rv); @@ -1243,11 +1288,12 @@ vm_reserv_reclaim_inactive(int domain) vm_reserv_unlock(rv); continue; } + freed = VM_LEVEL_0_NPAGES - rv->popcnt; vm_reserv_reclaim(rv); vm_reserv_unlock(rv); - return (TRUE); + return (freed); } - return (FALSE); + return (0); } /* Modified: user/markj/vm-playground/sys/vm/vm_reserv.h ============================================================================== --- user/markj/vm-playground/sys/vm/vm_reserv.h Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vm_reserv.h Fri Mar 2 18:12:25 2018 (r330288) @@ -54,10 +54,12 @@ vm_page_t vm_reserv_extend_contig(int req, vm_object_t vm_pindex_t pindex, int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_page_t mpred); -vm_page_t vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, - int domain, vm_page_t mpred); +vm_page_t vm_reserv_alloc_page(int req, vm_object_t object, + vm_pindex_t pindex, int domain, vm_page_t mpred, + int *countp); vm_page_t vm_reserv_extend(int req, vm_object_t object, - vm_pindex_t pindex, int domain, vm_page_t mpred); + vm_pindex_t pindex, int domain, vm_page_t mpred, + int *countp); void vm_reserv_break_all(vm_object_t object); boolean_t vm_reserv_free_page(vm_page_t m); void vm_reserv_init(void); @@ -67,7 +69,7 @@ int vm_reserv_level_iffullpop(vm_page_t m); boolean_t vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary); -boolean_t vm_reserv_reclaim_inactive(int domain); +int vm_reserv_reclaim_inactive(int domain); void vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, vm_pindex_t old_object_offset); int vm_reserv_size(int level); Modified: user/markj/vm-playground/sys/vm/vnode_pager.c ============================================================================== --- user/markj/vm-playground/sys/vm/vnode_pager.c Fri Mar 2 17:07:08 2018 (r330287) +++ user/markj/vm-playground/sys/vm/vnode_pager.c Fri Mar 2 18:12:25 2018 (r330288) @@ -897,35 +897,27 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page /* * Fill in the bp->b_pages[] array with requested and optional - * read behind or read ahead pages. Read behind pages are looked - * up in a backward direction, down to a first cached page. Same - * for read ahead pages, but there is no need to shift the array - * in case of encountering a cached page. + * read behind or read ahead pages. */ i = bp->b_npages = 0; - if (rbehind) { - vm_pindex_t startpindex, tpindex; - vm_page_t p; + if (rbehind > 0) { + vm_pindex_t startpindex; + vm_page_t mpred; VM_OBJECT_WLOCK(object); startpindex = m[0]->pindex - rbehind; - if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL && - p->pindex >= startpindex) - startpindex = p->pindex + 1; + if ((mpred = TAILQ_PREV(m[0], pglist, listq)) != NULL && + mpred->pindex >= startpindex) + startpindex = mpred->pindex + 1; - /* tpindex is unsigned; beware of numeric underflow. */ - for (tpindex = m[0]->pindex - 1; - tpindex >= startpindex && tpindex < m[0]->pindex; - tpindex--, i++) { - p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); - if (p == NULL) { - /* Shift the array. */ - for (int j = 0; j < i; j++) - bp->b_pages[j] = bp->b_pages[j + - tpindex + 1 - startpindex]; - break; - } - bp->b_pages[tpindex - startpindex] = p; + i = vm_page_alloc_pages_after(object, startpindex, + VM_ALLOC_NORMAL, &bp->b_pages[0], + m[0]->pindex - startpindex, mpred); + if (i < m[0]->pindex - startpindex) { + /* We have to drop the partially allocated run. */ + for (int j = 0; j < i; j++) + vm_page_free(bp->b_pages[j]); + i = 0; } bp->b_pgbefore = i; @@ -939,29 +931,24 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page bp->b_pages[i] = m[j]; bp->b_npages += count; - if (rahead) { - vm_pindex_t endpindex, tpindex; - vm_page_t p; + if (rahead > 0) { + vm_pindex_t endpindex, startpindex; + vm_page_t msucc; if (!VM_OBJECT_WOWNED(object)) VM_OBJECT_WLOCK(object); - endpindex = m[count - 1]->pindex + rahead + 1; - if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL && - p->pindex < endpindex) - endpindex = p->pindex; + startpindex = m[count - 1]->pindex + 1; + endpindex = startpindex + rahead; + if ((msucc = TAILQ_NEXT(m[count - 1], listq)) != NULL && + msucc->pindex < endpindex) + endpindex = msucc->pindex; if (endpindex > object->size) endpindex = object->size; - for (tpindex = m[count - 1]->pindex + 1; - tpindex < endpindex; i++, tpindex++) { - p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); - if (p == NULL) - break; - bp->b_pages[i] = p; - } - - bp->b_pgafter = i - bp->b_npages; - bp->b_npages = i; + bp->b_pgafter = vm_page_alloc_pages_after(object, startpindex, + VM_ALLOC_NORMAL, &bp->b_pages[i], endpindex - startpindex, + m[count - 1]); + bp->b_npages += bp->b_pgafter; } else bp->b_pgafter = 0;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803021812.w22ICPur037409>