Date: Mon, 16 Mar 2015 16:29:34 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r280147 - head/sys/powerpc/pseries Message-ID: <201503161629.t2GGTYQX001730@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Mon Mar 16 16:29:33 2015 New Revision: 280147 URL: https://svnweb.freebsd.org/changeset/base/280147 Log: Convert PTE eviction lock from an RW lock to a RM lock. It is held for writing approximately never (< 0.00000001% under heavy VM load, and it can go for months without ever being acquired in normal operation). This provides a 10% (2-minute) improvement in wall clock time for make -j32 buildworld on a 4-core 32-thread POWER8. Modified: head/sys/powerpc/pseries/mmu_phyp.c Modified: head/sys/powerpc/pseries/mmu_phyp.c ============================================================================== --- head/sys/powerpc/pseries/mmu_phyp.c Mon Mar 16 15:56:06 2015 (r280146) +++ head/sys/powerpc/pseries/mmu_phyp.c Mon Mar 16 16:29:33 2015 (r280147) @@ -30,7 +30,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/lock.h> -#include <sys/rwlock.h> +#include <sys/rmlock.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/sysctl.h> @@ -59,7 +59,7 @@ __FBSDID("$FreeBSD$"); extern int n_slbs; -static struct rwlock mphyp_eviction_lock; +static struct rmlock mphyp_eviction_lock; /* * Kernel MMU interface @@ -119,7 +119,7 @@ mphyp_bootstrap(mmu_t mmup, vm_offset_t phandle_t dev, node, root; int idx, len, res; - rw_init(&mphyp_eviction_lock, "pte eviction"); + rm_init(&mphyp_eviction_lock, "pte eviction"); moea64_early_bootstrap(mmup, kernelstart, kernelend); @@ -262,6 +262,7 @@ mphyp_pte_synch(mmu_t mmu, struct pvo_en static int64_t mphyp_pte_clear(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit) { + struct rm_priotracker track; int64_t refchg; uint64_t ptelo, junk; int err; @@ -274,11 +275,11 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en * shared eviction lock. */ PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); - rw_rlock(&mphyp_eviction_lock); + rm_rlock(&mphyp_eviction_lock, &track); refchg = mphyp_pte_synch(mmu, pvo); if (refchg < 0) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg); } @@ -288,7 +289,7 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en * Pessimistically claim that, once modified, it stays so * forever and that it is never referenced. */ - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg & ~LPTE_REF); } @@ -307,7 +308,7 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en refchg |= (ptelo & LPTE_REF); } - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg); } @@ -376,6 +377,7 @@ mphyp_pte_spillable_ident(uintptr_t pteg static int mphyp_pte_insert(mmu_t mmu, struct pvo_entry *pvo) { + struct rm_priotracker track; int64_t result; struct lpte evicted, pte; uint64_t index, junk, lastptelo; @@ -387,7 +389,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e evicted.pte_hi = 0; /* Make sure further insertion is locked out during evictions */ - rw_rlock(&mphyp_eviction_lock); + rm_rlock(&mphyp_eviction_lock, &track); /* * First try primary hash. @@ -396,7 +398,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); if (result == H_SUCCESS) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); pvo->pvo_pte.slot = index; return (0); } @@ -414,7 +416,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); if (result == H_SUCCESS) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); pvo->pvo_pte.slot = index; return (0); } @@ -426,10 +428,8 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e */ /* Lock out all insertions for a bit */ - if (!rw_try_upgrade(&mphyp_eviction_lock)) { - rw_runlock(&mphyp_eviction_lock); - rw_wlock(&mphyp_eviction_lock); - } + rm_runlock(&mphyp_eviction_lock, &track); + rm_wlock(&mphyp_eviction_lock); index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); if (index == -1L) { @@ -442,7 +442,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e if (index == -1L) { /* No freeable slots in either PTEG? We're hosed. */ - rw_wunlock(&mphyp_eviction_lock); + rm_wunlock(&mphyp_eviction_lock); panic("mphyp_pte_insert: overflow"); return (-1); } @@ -462,7 +462,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e */ result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); - rw_wunlock(&mphyp_eviction_lock); /* All clear */ + rm_wunlock(&mphyp_eviction_lock); /* All clear */ pvo->pvo_pte.slot = index; if (result == H_SUCCESS)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201503161629.t2GGTYQX001730>