Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 1 Oct 2015 10:43:41 +0000 (UTC)
From:      Andrew Turner <andrew@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r288445 - head/sys/arm64/arm64
Message-ID:  <201510011043.t91AhfvE032248@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: andrew
Date: Thu Oct  1 10:43:40 2015
New Revision: 288445
URL: https://svnweb.freebsd.org/changeset/base/288445

Log:
  Use pmap_load more consistently. While here try to only load the data once
  when we reuse the same data.
  
  Obtained from:	EuroBSDCon Devsummit
  Sponsored by:	ABT Systems Ltd

Modified:
  head/sys/arm64/arm64/pmap.c

Modified: head/sys/arm64/arm64/pmap.c
==============================================================================
--- head/sys/arm64/arm64/pmap.c	Thu Oct  1 09:53:12 2015	(r288444)
+++ head/sys/arm64/arm64/pmap.c	Thu Oct  1 10:43:40 2015	(r288445)
@@ -243,6 +243,16 @@ static void _pmap_unwire_l3(pmap_t pmap,
     struct spglist *free);
 static int pmap_unuse_l3(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
 
+/*
+ * These load the old table data and store the new value.
+ * They need to be atomic as the System MMU may write to the table at
+ * the same time as the CPU.
+ */
+#define	pmap_load_store(table, entry) atomic_swap_64(table, entry)
+#define	pmap_set(table, mask) atomic_set_64(table, mask)
+#define	pmap_load_clear(table) atomic_swap_64(table, 0)
+#define	pmap_load(table) (*table)
+
 /********************/
 /* Inline functions */
 /********************/
@@ -277,7 +287,7 @@ pmap_l1_to_l2(pd_entry_t *l1, vm_offset_
 {
 	pd_entry_t *l2;
 
-	l2 = (pd_entry_t *)PHYS_TO_DMAP(*l1 & ~ATTR_MASK);
+	l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
 	return (&l2[pmap_l2_index(va)]);
 }
 
@@ -287,7 +297,7 @@ pmap_l2(pmap_t pmap, vm_offset_t va)
 	pd_entry_t *l1;
 
 	l1 = pmap_l1(pmap, va);
-	if ((*l1 & ATTR_DESCR_MASK) != L1_TABLE)
+	if ((pmap_load(l1) & ATTR_DESCR_MASK) != L1_TABLE)
 		return (NULL);
 
 	return (pmap_l1_to_l2(l1, va));
@@ -298,7 +308,7 @@ pmap_l2_to_l3(pd_entry_t *l2, vm_offset_
 {
 	pt_entry_t *l3;
 
-	l3 = (pd_entry_t *)PHYS_TO_DMAP(*l2 & ~ATTR_MASK);
+	l3 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK);
 	return (&l3[pmap_l3_index(va)]);
 }
 
@@ -308,7 +318,7 @@ pmap_l3(pmap_t pmap, vm_offset_t va)
 	pd_entry_t *l2;
 
 	l2 = pmap_l2(pmap, va);
-	if (l2 == NULL || (*l2 & ATTR_DESCR_MASK) != L2_TABLE)
+	if (l2 == NULL || (pmap_load(l2) & ATTR_DESCR_MASK) != L2_TABLE)
 		return (NULL);
 
 	return (pmap_l2_to_l3(l2, va));
@@ -326,19 +336,19 @@ pmap_get_tables(pmap_t pmap, vm_offset_t
 	l1p = pmap_l1(pmap, va);
 	*l1 = l1p;
 
-	if ((*l1p & ATTR_DESCR_MASK) == L1_BLOCK) {
+	if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) {
 		*l2 = NULL;
 		*l3 = NULL;
 		return (true);
 	}
 
-	if ((*l1p & ATTR_DESCR_MASK) != L1_TABLE)
+	if ((pmap_load(l1p) & ATTR_DESCR_MASK) != L1_TABLE)
 		return (false);
 
 	l2p = pmap_l1_to_l2(l1p, va);
 	*l2 = l2p;
 
-	if ((*l2p & ATTR_DESCR_MASK) == L2_BLOCK) {
+	if ((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK) {
 		*l3 = NULL;
 		return (true);
 	}
@@ -348,16 +358,6 @@ pmap_get_tables(pmap_t pmap, vm_offset_t
 	return (true);
 }
 
-/*
- * These load the old table data and store the new value.
- * They need to be atomic as the System MMU may write to the table at
- * the same time as the CPU.
- */
-#define	pmap_load_store(table, entry) atomic_swap_64(table, entry)
-#define	pmap_set(table, mask) atomic_set_64(table, mask)
-#define	pmap_load_clear(table) atomic_swap_64(table, 0)
-#define	pmap_load(table) (*table)
-
 static __inline int
 pmap_is_current(pmap_t pmap)
 {
@@ -799,11 +799,11 @@ pmap_extract(pmap_t pmap, vm_offset_t va
 	 */
 	l2p = pmap_l2(pmap, va);
 	if (l2p != NULL) {
-		l2 = *l2p;
+		l2 = pmap_load(l2p);
 		if ((l2 & ATTR_DESCR_MASK) == L2_TABLE) {
 			l3p = pmap_l2_to_l3(l2p, va);
 			if (l3p != NULL) {
-				l3 = *l3p;
+				l3 = pmap_load(l3p);
 
 				if ((l3 & ATTR_DESCR_MASK) == L3_PAGE)
 					pa = (l3 & ~ATTR_MASK) |
@@ -852,23 +852,25 @@ retry:
 vm_paddr_t
 pmap_kextract(vm_offset_t va)
 {
-	pd_entry_t *l2;
+	pd_entry_t *l2p, l2;
 	pt_entry_t *l3;
 	vm_paddr_t pa;
 
 	if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
 		pa = DMAP_TO_PHYS(va);
 	} else {
-		l2 = pmap_l2(kernel_pmap, va);
-		if (l2 == NULL)
+		l2p = pmap_l2(kernel_pmap, va);
+		if (l2p == NULL)
 			panic("pmap_kextract: No l2");
-		if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK)
-			return ((*l2 & ~ATTR_MASK) | (va & L2_OFFSET));
+		l2 = pmap_load(l2p);
+		if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK)
+			return ((l2 & ~ATTR_MASK) |
+			    (va & L2_OFFSET));
 
-		l3 = pmap_l2_to_l3(l2, va);
+		l3 = pmap_l2_to_l3(l2p, va);
 		if (l3 == NULL)
 			panic("pmap_kextract: No l3...");
-		pa = (*l3 & ~ATTR_MASK) | (va & PAGE_MASK);
+		pa = (pmap_load(l3) & ~ATTR_MASK) | (va & PAGE_MASK);
 	}
 	return (pa);
 }
@@ -1242,11 +1244,11 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t 
 				return (NULL);
 			}
 		} else {
-			pdpg = PHYS_TO_VM_PAGE(*l1 & ~ATTR_MASK);
+			pdpg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK);
 			pdpg->wire_count++;
 		}
 
-		l2 = (pd_entry_t *)PHYS_TO_DMAP(*l1 & ~ATTR_MASK);
+		l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
 		l2 = &l2[ptepindex & Ln_ADDR_MASK];
 		pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE);
 		PTE_SYNC(l2);
@@ -1738,7 +1740,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva
 		if (l2 == NULL)
 			continue;
 
-		l3_paddr = *l2;
+		l3_paddr = pmap_load(l2);
 
 		/*
 		 * Weed out invalid mappings.
@@ -1805,7 +1807,7 @@ pmap_remove_all(vm_page_t m)
 	pv_entry_t pv;
 	pmap_t pmap;
 	pt_entry_t *l3, tl3;
-	pd_entry_t *l2;
+	pd_entry_t *l2, tl2;
 	struct spglist free;
 
 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
@@ -1817,7 +1819,9 @@ pmap_remove_all(vm_page_t m)
 		PMAP_LOCK(pmap);
 		pmap_resident_count_dec(pmap, 1);
 		l2 = pmap_l2(pmap, pv->pv_va);
-		KASSERT((*l2 & ATTR_DESCR_MASK) == L2_TABLE,
+		KASSERT(l2 != NULL, ("pmap_remove_all: no l2 table found"));
+		tl2 = pmap_load(l2);
+		KASSERT((tl2 & ATTR_DESCR_MASK) == L2_TABLE,
 		    ("pmap_remove_all: found a table when expecting "
 		     "a block in %p's pv list", m));
 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
@@ -1837,7 +1841,7 @@ pmap_remove_all(vm_page_t m)
 		 */
 		if (pmap_page_dirty(tl3))
 			vm_page_dirty(m);
-		pmap_unuse_l3(pmap, pv->pv_va, *l2, &free);
+		pmap_unuse_l3(pmap, pv->pv_va, tl2, &free);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 		m->md.pv_gen++;
 		free_pv_entry(pmap, pv);
@@ -1883,7 +1887,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sv
 			va_next = eva;
 
 		l2 = pmap_l1_to_l2(l1, sva);
-		if (l2 == NULL || (*l2 & ATTR_DESCR_MASK) != L2_TABLE)
+		if (l2 == NULL || (pmap_load(l2) & ATTR_DESCR_MASK) != L2_TABLE)
 			continue;
 
 		if (va_next > eva)
@@ -2345,7 +2349,7 @@ pmap_unwire(pmap_t pmap, vm_offset_t sva
 				continue;
 			if ((pmap_load(l3) & ATTR_SW_WIRED) == 0)
 				panic("pmap_unwire: l3 %#jx is missing "
-				    "ATTR_SW_WIRED", (uintmax_t)*l3);
+				    "ATTR_SW_WIRED", (uintmax_t)pmap_load(l3));
 
 			/*
 			 * PG_W must be cleared atomically.  Although the pmap
@@ -2836,7 +2840,7 @@ retry_pv_loop:
 		}
 		l3 = pmap_l3(pmap, pv->pv_va);
 retry:
-		oldl3 = *l3;
+		oldl3 = pmap_load(l3);
 		if ((oldl3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) {
 			if (!atomic_cmpset_long(l3, oldl3,
 			    oldl3 | ATTR_AP(ATTR_AP_RO)))
@@ -2879,7 +2883,7 @@ pmap_ts_referenced(vm_page_t m)
 	pv_entry_t pv, pvf;
 	pmap_t pmap;
 	struct rwlock *lock;
-	pd_entry_t *l2;
+	pd_entry_t *l2p, l2;
 	pt_entry_t *l3;
 	vm_paddr_t pa;
 	int cleared, md_gen, not_cleared;
@@ -2912,12 +2916,14 @@ retry:
 				goto retry;
 			}
 		}
-		l2 = pmap_l2(pmap, pv->pv_va);
-		KASSERT((*l2 & ATTR_DESCR_MASK) == L2_TABLE,
+		l2p = pmap_l2(pmap, pv->pv_va);
+		KASSERT(l2p != NULL, ("pmap_ts_referenced: no l2 table found"));
+		l2 = pmap_load(l2p);
+		KASSERT((l2 & ATTR_DESCR_MASK) == L2_TABLE,
 		    ("pmap_ts_referenced: found an invalid l2 table"));
-		l3 = pmap_l2_to_l3(l2, pv->pv_va);
+		l3 = pmap_l2_to_l3(l2p, pv->pv_va);
 		if ((pmap_load(l3) & ATTR_AF) != 0) {
-			if (safe_to_clear_referenced(pmap, *l3)) {
+			if (safe_to_clear_referenced(pmap, pmap_load(l3))) {
 				/*
 				 * TODO: We don't handle the access flag
 				 * at all. We need to be able to set it in
@@ -2931,8 +2937,8 @@ retry:
 				 * them is wasted effort. We do the
 				 * hard work for unwired pages only.
 				 */
-				pmap_remove_l3(pmap, l3, pv->pv_va,
-				    *l2, &free, &lock);
+				pmap_remove_l3(pmap, l3, pv->pv_va, l2,
+				    &free, &lock);
 				pmap_invalidate_page(pmap, pv->pv_va);
 				cleared++;
 				if (pvf == pv)



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201510011043.t91AhfvE032248>