From owner-svn-src-user@FreeBSD.ORG Mon Apr 19 22:01:59 2010 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id E30B3106564A; Mon, 19 Apr 2010 22:01:59 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id D26B08FC12; Mon, 19 Apr 2010 22:01:59 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o3JM1xx1028464; Mon, 19 Apr 2010 22:01:59 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o3JM1xkJ028462; Mon, 19 Apr 2010 22:01:59 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <201004192201.o3JM1xkJ028462@svn.freebsd.org> From: Kip Macy Date: Mon, 19 Apr 2010 22:01:59 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r206875 - user/kmacy/head_page_lock_2/sys/amd64/amd64 X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 19 Apr 2010 22:02:00 -0000 Author: kmacy Date: Mon Apr 19 22:01:59 2010 New Revision: 206875 URL: http://svn.freebsd.org/changeset/base/206875 Log: ensure that the pmap lock is always held when modifying resident count Modified: user/kmacy/head_page_lock_2/sys/amd64/amd64/pmap.c Modified: user/kmacy/head_page_lock_2/sys/amd64/amd64/pmap.c ============================================================================== --- user/kmacy/head_page_lock_2/sys/amd64/amd64/pmap.c Mon Apr 19 22:00:35 2010 (r206874) +++ user/kmacy/head_page_lock_2/sys/amd64/amd64/pmap.c Mon Apr 19 22:01:59 2010 (r206875) @@ -314,7 +314,6 @@ ls_init(struct lock_stack *ls) ls->ls_top = 0; } - #define ls_push(ls, m) _ls_push((ls), (m), LOCK_FILE, LOCK_LINE) static void @@ -508,6 +507,21 @@ pmap_pte(pmap_t pmap, vm_offset_t va) return (pmap_pde_to_pte(pde, va)); } +static __inline void +pmap_resident_count_inc(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count += count; +} + +static __inline void +pmap_resident_count_dec(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count -= count; +} PMAP_INLINE pt_entry_t * vtopte(vm_offset_t va) @@ -1655,7 +1669,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_of pd = pmap_pde(pmap, va); *pd = 0; } - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); if (m->pindex < NUPDE) { /* We just released a PT, unhold the matching PD */ vm_page_t pdpg; @@ -1889,8 +1903,7 @@ _pmap_allocpte(pmap_t pmap, vm_paddr_t p pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)]; *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; } - - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); return (m); } @@ -2201,7 +2214,7 @@ pmap_collect(pmap_t locked_pmap, struct else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) continue; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, va); KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found" " a 2mpage in page %p's pv list", m)); @@ -2662,13 +2675,14 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t return (FALSE); } if (va < VM_MAXUSER_ADDRESS) - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); } if (TAILQ_EMPTY(pv_list) && ((oldpde & PG_MANAGED) != 0)) { if (pmap_pv_list_try_alloc(pmap, NPTEPG-1, pv_list) == FALSE) { if (mpte == NULL && (va < VM_MAXUSER_ADDRESS)) - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); return (FALSE); + } } mptepa = VM_PAGE_TO_PHYS(mpte); firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); @@ -2734,6 +2748,7 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t " in pmap %p", va, pmap); return (TRUE); } + /* * pmap_remove_pde: do the things to unmap a superpage in a process @@ -2760,7 +2775,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t */ if (oldpde & PG_G) pmap_invalidate_page(kernel_pmap, sva); - pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; + pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); if (oldpde & PG_MANAGED) { pvh = pa_to_pvh(oldpde & PG_PS_FRAME); pmap_pvh_free(pvh, pmap, sva); @@ -2791,7 +2806,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t mpte = pmap_lookup_pt_page(pmap, sva); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pde: pte page wire count error")); mpte->wire_count = 0; @@ -2823,7 +2838,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t */ if (oldpte & PG_G) pmap_invalidate_page(kernel_pmap, va); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); if (oldpte & PG_MANAGED) { m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); vm_page_lock_assert(m, MA_OWNED); @@ -3135,7 +3150,7 @@ pmap_remove_all(vm_page_t m) while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found" " a 2mpage in page %p's pv list", m)); @@ -3605,7 +3620,7 @@ restart: " va: 0x%lx", va)); } } else - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); /* * Enter on the PV list if part of our managed memory. @@ -3751,7 +3766,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t /* * Increment counters. */ - pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; + pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); /* * Map the superpage. @@ -3909,7 +3924,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_ /* * Increment counters */ - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0); if ((prot & VM_PROT_EXECUTE) == 0) @@ -4015,8 +4030,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offs if ((*pde & PG_V) == 0) { pde_store(pde, pa | PG_PS | PG_M | PG_A | PG_U | PG_RW | PG_V); - pmap->pm_stats.resident_count += NBPDR / - PAGE_SIZE; + pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); pmap_pde_mappings++; } else { /* Continue on if the PDE is already valid. */ @@ -4169,8 +4183,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr & PG_PS_FRAME))) { *pde = srcptepaddr & ~PG_W; - dst_pmap->pm_stats.resident_count += - NBPDR / PAGE_SIZE; + pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE); } else dstmpde->wire_count--; PA_UNLOCK(pa); @@ -4219,7 +4232,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm */ *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); - dst_pmap->pm_stats.resident_count++; + pmap_resident_count_inc(dst_pmap, 1); } else { free = NULL; if (pmap_unwire_pte_hold(dst_pmap, @@ -4499,7 +4512,7 @@ restart: atomic_add_int(&pv_entry_count, -1); pc->pc_map[field] |= bitmask; if ((tpte & PG_PS) != 0) { - pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; + pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); if (TAILQ_EMPTY(&pvh->pv_list)) { @@ -4510,7 +4523,7 @@ restart: mpte = pmap_lookup_pt_page(pmap, pv->pv_va); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pages: pte page wire count error")); mpte->wire_count = 0; @@ -4518,7 +4531,7 @@ restart: atomic_subtract_int(&cnt.v_wire_count, 1); } } else { - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));