Date: Fri, 26 Jun 2009 04:59:55 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r195034 - user/kmacy/releng_7_2_fcs/sys/amd64/amd64 Message-ID: <200906260459.n5Q4xtxs075241@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Fri Jun 26 04:59:55 2009 New Revision: 195034 URL: http://svn.freebsd.org/changeset/base/195034 Log: when page lock is held defer freeing pages to another context Modified: user/kmacy/releng_7_2_fcs/sys/amd64/amd64/pmap.c Modified: user/kmacy/releng_7_2_fcs/sys/amd64/amd64/pmap.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/amd64/amd64/pmap.c Fri Jun 26 04:47:43 2009 (r195033) +++ user/kmacy/releng_7_2_fcs/sys/amd64/amd64/pmap.c Fri Jun 26 04:59:55 2009 (r195034) @@ -184,6 +184,7 @@ struct vp_lock { #define PA_LOCK_COUNT 64 +static vm_page_t global_free __aligned(128); struct mtx pv_lock __aligned(128); struct vp_lock pa_lock[PA_LOCK_COUNT] __aligned(128); @@ -1242,19 +1243,7 @@ pmap_qremove(vm_offset_t sva, int count) /*************************************************** * Page table page management routines..... - ***************************************************/ -static __inline void -pmap_free_zero_pages(vm_page_t free) -{ - vm_page_t m; - - while (free != NULL) { - m = free; - free = m->right; - /* Preserve the page's PG_ZERO setting. */ - vm_page_free_toq(m); - } -} +***************************************************/ /* * Schedule the specified unused page table page to be freed. Specifically, @@ -1273,6 +1262,36 @@ pmap_add_delayed_free_list(vm_page_t m, *free = m; } +static __inline void +pmap_free_zero_pages(vm_page_t free) +{ + vm_page_t m; + + if (free == NULL) + return; + /* + * transfer global free list to local free list + * + */ + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(global_free, &free, FALSE); + global_free = NULL; + mtx_unlock(&pv_lock); + + /* + * Avoid contention from repeated acquisition + * we know we're going to need it for every page in the list + */ + vm_page_lock_queues(); + while (free != NULL) { + m = free; + free = m->right; + /* Preserve the page's PG_ZERO setting. */ + vm_page_free_toq_locked(m); + } + vm_page_unlock_queues(); +} + /* * Inserts the specified page table page into the specified pmap's collection * of idle page table pages. Each of a pmap's page table pages is responsible @@ -1943,7 +1962,7 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_coll * exacerbating the shortage of free pv entries. */ static void -pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq) +pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq, vm_page_t *free) { struct md_page *pvh; pd_entry_t *pde; @@ -1951,7 +1970,7 @@ pmap_collect(pmap_t locked_pmap, struct pt_entry_t *pte, tpte; pv_entry_t next_pv, pv; vm_offset_t va; - vm_page_t m, free; + vm_page_t m; vm_page_lock_queues(); TAILQ_FOREACH(m, &vpq->pl, pageq) { @@ -1982,7 +2001,7 @@ pmap_collect(pmap_t locked_pmap, struct if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); free = NULL; - pmap_unuse_pt(pmap, va, *pde, &free); + pmap_unuse_pt(pmap, va, *pde, free); pmap_invalidate_page(pmap, va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) { @@ -1990,10 +2009,9 @@ pmap_collect(pmap_t locked_pmap, struct if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); } - free_pv_entry(pmap, pv, &free); + free_pv_entry(pmap, pv, free); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); - pmap_free_zero_pages(free); } vm_page_unlock(m); } @@ -2056,7 +2074,7 @@ get_pv_entry(pmap_t pmap, int try) int bit, field; pv_entry_t pv; struct pv_chunk *pc; - vm_page_t m; + vm_page_t m, free = NULL; PMAP_LOCK_ASSERT(pmap, MA_OWNED); mtx_lock(&pv_lock); @@ -2088,7 +2106,7 @@ retry: } PV_STAT(pv_entry_spare--); mtx_unlock(&pv_lock); - return (pv); + goto done; } } /* No free items, allocate another chunk */ @@ -2100,7 +2118,8 @@ retry: pv_entry_count--; PV_STAT(pc_chunk_tryfail++); mtx_unlock(&pv_lock); - return (NULL); + pv = NULL; + goto done; } /* * Reclaim pv entries: At first, destroy mappings to inactive @@ -2115,7 +2134,7 @@ retry: pq = &vm_page_queues[PQ_ACTIVE]; } else panic("get_pv_entry: increase vm.pmap.shpgperproc"); - pmap_collect(pmap, pq); + pmap_collect(pmap, pq, &free); goto retry; } PV_STAT(pc_chunk_count++); @@ -2131,6 +2150,8 @@ retry: TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(pv_entry_spare += _NPCPV - 1); mtx_unlock(&pv_lock); +done: + pmap_free_zero_pages(free); return (pv); } @@ -2735,7 +2756,9 @@ pmap_remove_all(vm_page_t m) PMAP_UNLOCK(pmap); } vm_page_flag_clear(m, PG_WRITEABLE); - pmap_free_zero_pages(free); + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); } /* @@ -3230,7 +3253,9 @@ validate: PA_UNLOCK(pa); PMAP_UNLOCK(pmap); - pmap_free_zero_pages(free); + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); } /* @@ -3623,7 +3648,6 @@ out: if (pa) PA_UNLOCK(pa); PMAP_UNLOCK(pmap); - pmap_free_zero_pages(free); } @@ -4113,6 +4137,7 @@ pmap_is_prefaultable(pmap_t pmap, vm_off return (rv); } + /* * Clear the write and modified bits in each of the given page's mappings. */ @@ -4142,7 +4167,9 @@ pmap_remove_write(vm_page_t m) PMAP_UNLOCK(pmap); } - pmap_free_zero_pages(free); + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4223,7 +4250,9 @@ pmap_ts_referenced(vm_page_t m) } PMAP_UNLOCK(pmap); } - pmap_free_zero_pages(free); + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; do { @@ -4298,7 +4327,10 @@ pmap_clear_modify(vm_page_t m) } PMAP_UNLOCK(pmap); } - pmap_free_zero_pages(free); + + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4356,7 +4388,9 @@ pmap_clear_reference(vm_page_t m) } PMAP_UNLOCK(pmap); } - pmap_free_zero_pages(free); + mtx_lock(&pv_lock); + pmap_add_delayed_free_list(free, &global_free, FALSE); + mtx_unlock(&pv_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906260459.n5Q4xtxs075241>