Date: Sun, 4 Feb 2018 19:14:09 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r328860 - user/jeff/numa/sys/vm Message-ID: <201802041914.w14JE9u4097652@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Sun Feb 4 19:14:09 2018 New Revision: 328860 URL: https://svnweb.freebsd.org/changeset/base/328860 Log: Prototype vm pagequeue batches. From Isilon, mlaier & markj Use contig allocations for vm_page_import. Modified: user/jeff/numa/sys/vm/vm_object.c user/jeff/numa/sys/vm/vm_page.c user/jeff/numa/sys/vm/vm_pageout.c user/jeff/numa/sys/vm/vm_pagequeue.h Modified: user/jeff/numa/sys/vm/vm_object.c ============================================================================== --- user/jeff/numa/sys/vm/vm_object.c Sun Feb 4 19:12:03 2018 (r328859) +++ user/jeff/numa/sys/vm/vm_object.c Sun Feb 4 19:14:09 2018 (r328860) @@ -723,7 +723,6 @@ vm_object_terminate_pages(vm_object_t object) vm_page_t p, p_next; struct mtx *mtx, *mtx1; struct vm_pagequeue *pq, *pq1; - int dequeued; VM_OBJECT_ASSERT_WLOCKED(object); @@ -748,7 +747,6 @@ vm_object_terminate_pages(vm_object_t object) if (mtx != NULL) mtx_unlock(mtx); if (pq != NULL) { - vm_pagequeue_cnt_add(pq, dequeued); vm_pagequeue_unlock(pq); pq = NULL; } @@ -766,27 +764,19 @@ vm_object_terminate_pages(vm_object_t object) "page %p is not queued", p)); pq1 = vm_page_pagequeue(p); if (pq != pq1) { - if (pq != NULL) { - vm_pagequeue_cnt_add(pq, dequeued); + if (pq != NULL) vm_pagequeue_unlock(pq); - } pq = pq1; vm_pagequeue_lock(pq); - dequeued = 0; } - p->queue = PQ_NONE; - TAILQ_REMOVE(&pq->pq_pl, p, plinks.q); - dequeued--; } if (vm_page_free_prep(p, true)) continue; unlist: TAILQ_REMOVE(&object->memq, p, listq); } - if (pq != NULL) { - vm_pagequeue_cnt_add(pq, dequeued); + if (pq != NULL) vm_pagequeue_unlock(pq); - } if (mtx != NULL) mtx_unlock(mtx); Modified: user/jeff/numa/sys/vm/vm_page.c ============================================================================== --- user/jeff/numa/sys/vm/vm_page.c Sun Feb 4 19:12:03 2018 (r328859) +++ user/jeff/numa/sys/vm/vm_page.c Sun Feb 4 19:14:09 2018 (r328860) @@ -74,6 +74,13 @@ * * The page daemon can acquire and hold any pair of page queue * locks in any order. * + * * Batch queues are used to defer insertions of pages into the + * main paging queues. The aim is to reduce contention at the + * entry point of the queue by inserting multiple pages in an + * O(1) operation. This comes at the expense of strict LRU. + * Only a page lock is required to insert a page into a batch + * queue. + * * - The object lock is required when inserting or removing * pages from an object (vm_page_insert() or vm_page_remove()). * @@ -425,7 +432,7 @@ vm_page_domain_init(int domain) { struct vm_domain *vmd; struct vm_pagequeue *pq; - int i; + int i, j; vmd = VM_DOMAIN(domain); bzero(vmd, sizeof(*vmd)); @@ -447,6 +454,15 @@ vm_page_domain_init(int domain) TAILQ_INIT(&pq->pq_pl); mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue", MTX_DEF | MTX_DUPOK); + + /* + * The batch queue limits are set in vm_pageout_init() once + * we've set the paging targets. + */ + for (j = 0; j < BPQ_COUNT; j++) { + TAILQ_INIT(&pq->pq_bpqs[j].bpq_pl); + pq->pq_bpqs[j].bpq_lim = 1; + } } mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF); } @@ -2170,9 +2186,10 @@ vm_page_import(void *arg, void **store, int cnt, int d vmd = arg; domain = vmd->vmd_domain; + cnt = rounddown2(cnt, 8); vm_domain_free_lock(vmd); - for (i = 0; i < cnt; i++) { - m = vm_phys_alloc_pages(domain, VM_FREELIST_DEFAULT, 0); + for (i = 0; i < cnt; i+=8) { + m = vm_phys_alloc_pages(domain, VM_FREELIST_DEFAULT, 3); if (m == NULL) break; store[i] = m; @@ -2180,6 +2197,13 @@ vm_page_import(void *arg, void **store, int cnt, int d if (i != 0) vm_domain_freecnt_adj(vmd, -i); vm_domain_free_unlock(vmd); + cnt = i; + for (i = 0; i < cnt; i++) { + if ((i % 8) == 0) + m = store[i]; + else + store[i] = ++m; + } return (i); } @@ -2972,6 +2996,30 @@ vm_page_pagequeue(vm_page_t m) } /* + * vm_page_enqueue_batch: + * + * Concatenate the pages in a batch queue to their corresponding paging + * queue. + * + * The pagequeue must be locked. + */ +static void +vm_page_enqueue_batch(struct vm_pagequeue *pq, u_int idx) +{ + struct vm_batchqueue *bpq; + + KASSERT(idx < BPQ_COUNT, ("invalid batch queue index %u", idx)); + vm_pagequeue_assert_locked(pq); + + bpq = &pq->pq_bpqs[idx]; + if (bpq->bpq_cnt != 0) { + TAILQ_CONCAT(&pq->pq_pl, &bpq->bpq_pl, plinks.q); + vm_pagequeue_cnt_add(pq, bpq->bpq_cnt); + bpq->bpq_cnt = 0; + } +} + +/* * vm_page_dequeue: * * Remove the given page from its current page queue. @@ -2989,6 +3037,7 @@ vm_page_dequeue(vm_page_t m) pq = vm_page_pagequeue(m); vm_pagequeue_lock(pq); m->queue = PQ_NONE; + vm_page_enqueue_batch(pq, BPQ_IDX(m)); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_dec(pq); vm_pagequeue_unlock(pq); @@ -3009,6 +3058,7 @@ vm_page_dequeue_locked(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); pq = vm_page_pagequeue(m); vm_pagequeue_assert_locked(pq); + vm_page_enqueue_batch(pq, BPQ_IDX(m)); m->queue = PQ_NONE; TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_dec(pq); @@ -3024,6 +3074,7 @@ vm_page_dequeue_locked(vm_page_t m) static void vm_page_enqueue(uint8_t queue, vm_page_t m) { + struct vm_batchqueue *bpq; struct vm_pagequeue *pq; vm_page_lock_assert(m, MA_OWNED); @@ -3031,11 +3082,14 @@ vm_page_enqueue(uint8_t queue, vm_page_t m) ("vm_page_enqueue: invalid queue %u request for page %p", queue, m)); pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue]; - vm_pagequeue_lock(pq); m->queue = queue; - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); - vm_pagequeue_cnt_inc(pq); - vm_pagequeue_unlock(pq); + bpq = &pq->pq_bpqs[BPQ_IDX(m)]; + TAILQ_INSERT_TAIL(&bpq->bpq_pl, m, plinks.q); + if (bpq->bpq_cnt++ >= bpq->bpq_lim) { + vm_pagequeue_lock(pq); + vm_page_enqueue_batch(pq, BPQ_IDX(m)); + vm_pagequeue_unlock(pq); + } } /* @@ -3055,6 +3109,7 @@ vm_page_requeue(vm_page_t m) ("vm_page_requeue: page %p is not queued", m)); pq = vm_page_pagequeue(m); vm_pagequeue_lock(pq); + vm_page_enqueue_batch(pq, BPQ_IDX(m)); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); vm_pagequeue_unlock(pq); @@ -3072,10 +3127,12 @@ vm_page_requeue_locked(vm_page_t m) { struct vm_pagequeue *pq; + vm_page_lock_assert(m, MA_OWNED); KASSERT(m->queue != PQ_NONE, ("vm_page_requeue_locked: page %p is not queued", m)); pq = vm_page_pagequeue(m); vm_pagequeue_assert_locked(pq); + vm_page_enqueue_batch(pq, BPQ_IDX(m)); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); } @@ -3396,6 +3453,7 @@ vm_page_unwire(vm_page_t m, uint8_t queue) static inline void _vm_page_deactivate(vm_page_t m, boolean_t noreuse) { + struct vm_batchqueue *bpq; struct vm_pagequeue *pq; int queue; @@ -3416,9 +3474,17 @@ _vm_page_deactivate(vm_page_t m, boolean_t noreuse) } else { if (queue != PQ_NONE) vm_page_dequeue(m); + bpq = &pq->pq_bpqs[BPQ_IDX(m)]; + if (bpq->bpq_cnt < bpq->bpq_lim) { + bpq->bpq_cnt++; + m->queue = PQ_INACTIVE; + TAILQ_INSERT_TAIL(&bpq->bpq_pl, m, plinks.q); + return; + } vm_pagequeue_lock(pq); } m->queue = PQ_INACTIVE; + vm_page_enqueue_batch(pq, BPQ_IDX(m)); if (noreuse) TAILQ_INSERT_BEFORE( &vm_pagequeue_domain(m)->vmd_inacthead, m, Modified: user/jeff/numa/sys/vm/vm_pageout.c ============================================================================== --- user/jeff/numa/sys/vm/vm_pageout.c Sun Feb 4 19:12:03 2018 (r328859) +++ user/jeff/numa/sys/vm/vm_pageout.c Sun Feb 4 19:14:09 2018 (r328860) @@ -1922,6 +1922,7 @@ static void vm_pageout_init_domain(int domain) { struct vm_domain *vmd; + int lim, i, j; vmd = VM_DOMAIN(domain); vmd->vmd_interrupt_free_min = 2; @@ -1960,6 +1961,22 @@ vm_pageout_init_domain(int domain) */ vmd->vmd_background_launder_target = (vmd->vmd_free_target - vmd->vmd_free_min) / 10; + + /* + * Set batch queue limits for paging queues. + * + * We want these to be small relative to the amount of system memory. + * Roughly v_page_count / PA_LOCK_COUNT pages are mapped to a given + * batch queue; ensure that no more than 0.1% of them may be queued in + * the batch queue for a particular page queue. Then no more than + * 0.1% * PQ_COUNT can be queued across all page queues. This gives a + * per-page queue batch limit of 1 page per GB of memory on amd64. + */ + + lim = MAX(vmd->vmd_page_count / 1000 / BPQ_COUNT, 8); + for (i = 0; i < PQ_COUNT; i++) + for (j = 0; j < BPQ_COUNT; j++) + vmd->vmd_pagequeues[i].pq_bpqs[j].bpq_lim = lim; } static void Modified: user/jeff/numa/sys/vm/vm_pagequeue.h ============================================================================== --- user/jeff/numa/sys/vm/vm_pagequeue.h Sun Feb 4 19:12:03 2018 (r328859) +++ user/jeff/numa/sys/vm/vm_pagequeue.h Sun Feb 4 19:14:09 2018 (r328860) @@ -66,11 +66,23 @@ #define _VM_PAGEQUEUE_ #ifdef _KERNEL + +#define BPQ_COUNT PA_LOCK_COUNT +#define BPQ_IDX(m) (pa_index(VM_PAGE_TO_PHYS(m)) % BPQ_COUNT) + +struct vm_batchqueue { + struct pglist bpq_pl; + int bpq_cnt; + int bpq_lim; +} __aligned(CACHE_LINE_SIZE); + struct vm_pagequeue { struct mtx pq_mutex; struct pglist pq_pl; int pq_cnt; const char * const pq_name; + char _pq_pad[0] __aligned(CACHE_LINE_SIZE); + struct vm_batchqueue pq_bpqs[BPQ_COUNT]; } __aligned(CACHE_LINE_SIZE); #include <vm/uma.h>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201802041914.w14JE9u4097652>