Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 1 Jul 2017 09:38:52 +0000 (UTC)
From:      Andrew Turner <andrew@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r320529 - head/sys/arm64/arm64
Message-ID:  <201707010938.v619cquC028333@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: andrew
Date: Sat Jul  1 09:38:52 2017
New Revision: 320529
URL: https://svnweb.freebsd.org/changeset/base/320529

Log:
  Remove all calls to cpu_dcache_wb_range from the arm64 pmap code. These
  were unneeded as we tell the tlb the pagetables are in cached memory. This
  gives us a small, but statistically significant improvement over just
  removing the PTE_SYNC cases.
  
  While here remove PTE_SYNC, it's now unneeded.
  
  Sponsored by:	DARPA, AFRL

Modified:
  head/sys/arm64/arm64/pmap.c

Modified: head/sys/arm64/arm64/pmap.c
==============================================================================
--- head/sys/arm64/arm64/pmap.c	Sat Jul  1 05:35:29 2017	(r320528)
+++ head/sys/arm64/arm64/pmap.c	Sat Jul  1 09:38:52 2017	(r320529)
@@ -504,12 +504,6 @@ pmap_l3_valid(pt_entry_t l3)
 
 CTASSERT(L1_BLOCK == L2_BLOCK);
 
-#if 0
-#define	PTE_SYNC(pte)	cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte))
-#else
-#define	PTE_SYNC(pte)	(void)0
-#endif
-
 /*
  * Checks if the page is dirty. We currently lack proper tracking of this on
  * arm64 so for now assume is a page mapped as rw was accessed it is.
@@ -594,8 +588,6 @@ pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t mi
 	dmap_phys_max = pa;
 	dmap_max_addr = va;
 
-	cpu_dcache_wb_range((vm_offset_t)pagetable_dmap,
-	    PAGE_SIZE * DMAP_TABLES);
 	cpu_tlb_flushID();
 }
 
@@ -624,11 +616,7 @@ pmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm
 
 	/* Clean the L2 page table */
 	memset((void *)l2_start, 0, l2pt - l2_start);
-	cpu_dcache_wb_range(l2_start, l2pt - l2_start);
 
-	/* Flush the l1 table to ram */
-	cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE);
-
 	return l2pt;
 }
 
@@ -659,10 +647,7 @@ pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm
 
 	/* Clean the L2 page table */
 	memset((void *)l3_start, 0, l3pt - l3_start);
-	cpu_dcache_wb_range(l3_start, l3pt - l3_start);
 
-	cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE);
-
 	return l3pt;
 }
 
@@ -1131,7 +1116,6 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_
 
 		pte = pmap_l2_to_l3(pde, va);
 		pmap_load_store(pte, (pa & ~L3_OFFSET) | attr);
-		PTE_SYNC(pte);
 
 		va += PAGE_SIZE;
 		pa += PAGE_SIZE;
@@ -1161,7 +1145,6 @@ pmap_kremove(vm_offset_t va)
 	KASSERT(lvl == 3, ("pmap_kremove: Invalid pte level %d", lvl));
 
 	pmap_load_clear(pte);
-	PTE_SYNC(pte);
 	pmap_invalidate_page(kernel_pmap, va);
 }
 
@@ -1184,7 +1167,6 @@ pmap_kremove_device(vm_offset_t sva, vm_size_t size)
 		KASSERT(lvl == 3,
 		    ("Invalid device pagetable level: %d != 3", lvl));
 		pmap_load_clear(pte);
-		PTE_SYNC(pte);
 
 		va += PAGE_SIZE;
 		size -= PAGE_SIZE;
@@ -1244,7 +1226,6 @@ pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 			pa |= ATTR_XN;
 		pte = pmap_l2_to_l3(pde, va);
 		pmap_load_store(pte, pa);
-		PTE_SYNC(pte);
 
 		va += L3_SIZE;
 	}
@@ -1271,7 +1252,6 @@ pmap_qremove(vm_offset_t sva, int count)
 		    ("Invalid device pagetable level: %d != 3", lvl));
 		if (pte != NULL) {
 			pmap_load_clear(pte);
-			PTE_SYNC(pte);
 		}
 
 		va += PAGE_SIZE;
@@ -1343,21 +1323,18 @@ _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t
 
 		l0 = pmap_l0(pmap, va);
 		pmap_load_clear(l0);
-		PTE_SYNC(l0);
 	} else if (m->pindex >= NUL2E) {
 		/* l2 page */
 		pd_entry_t *l1;
 
 		l1 = pmap_l1(pmap, va);
 		pmap_load_clear(l1);
-		PTE_SYNC(l1);
 	} else {
 		/* l3 page */
 		pd_entry_t *l2;
 
 		l2 = pmap_l2(pmap, va);
 		pmap_load_clear(l2);
-		PTE_SYNC(l2);
 	}
 	pmap_resident_count_dec(pmap, 1);
 	if (m->pindex < NUL2E) {
@@ -1498,7 +1475,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str
 		l0index = ptepindex - (NUL2E + NUL1E);
 		l0 = &pmap->pm_l0[l0index];
 		pmap_load_store(l0, VM_PAGE_TO_PHYS(m) | L0_TABLE);
-		PTE_SYNC(l0);
 	} else if (ptepindex >= NUL2E) {
 		vm_pindex_t l0index, l1index;
 		pd_entry_t *l0, *l1;
@@ -1527,7 +1503,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str
 		l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
 		l1 = &l1[ptepindex & Ln_ADDR_MASK];
 		pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE);
-		PTE_SYNC(l1);
 	} else {
 		vm_pindex_t l0index, l1index;
 		pd_entry_t *l0, *l1, *l2;
@@ -1574,7 +1549,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str
 		l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
 		l2 = &l2[ptepindex & Ln_ADDR_MASK];
 		pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE);
-		PTE_SYNC(l2);
 	}
 
 	pmap_resident_count_inc(pmap, 1);
@@ -1727,7 +1701,6 @@ pmap_growkernel(vm_offset_t addr)
 				pmap_zero_page(nkpg);
 			paddr = VM_PAGE_TO_PHYS(nkpg);
 			pmap_load_store(l1, paddr | L1_TABLE);
-			PTE_SYNC(l1);
 			continue; /* try again */
 		}
 		l2 = pmap_l1_to_l2(l1, kernel_vm_end);
@@ -1749,7 +1722,6 @@ pmap_growkernel(vm_offset_t addr)
 			pmap_zero_page(nkpg);
 		paddr = VM_PAGE_TO_PHYS(nkpg);
 		pmap_load_store(l2, paddr | L2_TABLE);
-		PTE_SYNC(l2);
 		pmap_invalidate_page(kernel_pmap, kernel_vm_end);
 
 		kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
@@ -1883,7 +1855,6 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l
 				if ((tpte & ATTR_SW_WIRED) != 0)
 					continue;
 				tpte = pmap_load_clear(pte);
-				PTE_SYNC(pte);
 				pmap_invalidate_page(pmap, va);
 				m = PHYS_TO_VM_PAGE(tpte & ~ATTR_MASK);
 				if (pmap_page_dirty(tpte))
@@ -2272,7 +2243,6 @@ pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	old_l3 = pmap_load_clear(l3);
-	PTE_SYNC(l3);
 	pmap_invalidate_page(pmap, va);
 	if (old_l3 & ATTR_SW_WIRED)
 		pmap->pm_stats.wired_count -= 1;
@@ -2493,7 +2463,6 @@ retry:
 		pte = pmap_l2_to_l3(pde, pv->pv_va);
 		tpte = pmap_load(pte);
 		pmap_load_clear(pte);
-		PTE_SYNC(pte);
 		pmap_invalidate_page(pmap, pv->pv_va);
 		if (tpte & ATTR_SW_WIRED)
 			pmap->pm_stats.wired_count--;
@@ -2595,7 +2564,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t
 				nbits |= ATTR_XN;
 
 			pmap_set(l3p, nbits);
-			PTE_SYNC(l3p);
 			/* XXX: Use pmap_invalidate_range */
 			pmap_invalidate_page(pmap, va);
 		}
@@ -2657,12 +2625,10 @@ pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_ent
 
 	/* Clear the old mapping */
 	pmap_load_clear(pte);
-	PTE_SYNC(pte);
 	pmap_invalidate_range(pmap, va, va + size);
 
 	/* Create the new mapping */
 	pmap_load_store(pte, newpte);
-	PTE_SYNC(pte);
 
 	critical_exit();
 	intr_restore(intr);
@@ -2886,7 +2852,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
 
 				l1_pa = VM_PAGE_TO_PHYS(l1_m);
 				pmap_load_store(pde, l1_pa | L0_TABLE);
-				PTE_SYNC(pde);
 				/* FALLTHROUGH */
 			case 0:
 				/* Get the l1 pde to update */
@@ -2903,7 +2868,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
 
 				l2_pa = VM_PAGE_TO_PHYS(l2_m);
 				pmap_load_store(pde, l2_pa | L1_TABLE);
-				PTE_SYNC(pde);
 				/* FALLTHROUGH */
 			case 1:
 				/* Get the l2 pde to update */
@@ -2919,7 +2883,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
 
 				l3_pa = VM_PAGE_TO_PHYS(l3_m);
 				pmap_load_store(pde, l3_pa | L2_TABLE);
-				PTE_SYNC(pde);
 				break;
 			}
 		}
@@ -3023,7 +2986,6 @@ validate:
 			}
 		} else {
 			pmap_load_store(l3, new_l3);
-			PTE_SYNC(l3);
 			pmap_invalidate_page(pmap, va);
 			if (pmap_page_dirty(orig_l3) &&
 			    (orig_l3 & ATTR_SW_MANAGED) != 0)
@@ -3033,7 +2995,6 @@ validate:
 		pmap_load_store(l3, new_l3);
 	}
 
-	PTE_SYNC(l3);
 	pmap_invalidate_page(pmap, va);
 
 	if (pmap != pmap_kernel()) {
@@ -3231,7 +3192,6 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
 	if ((m->oflags & VPO_UNMANAGED) == 0)
 		pa |= ATTR_SW_MANAGED;
 	pmap_load_store(l3, pa);
-	PTE_SYNC(l3);
 	pmap_invalidate_page(pmap, va);
 	return (mpte);
 }
@@ -3641,7 +3601,6 @@ pmap_remove_pages(pmap_t pmap)
 				    (uintmax_t)tpte));
 
 				pmap_load_clear(pte);
-				PTE_SYNC(pte);
 
 				/*
 				 * Update the vm_page_t clean/reference bits.
@@ -4416,7 +4375,6 @@ pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_
 		l2[i] = newl2 | phys;
 		phys += L2_SIZE;
 	}
-	cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE);
 	KASSERT(l2[0] == ((oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK),
 	    ("Invalid l2 page (%lx != %lx)", l2[0],
 	    (oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK));
@@ -4494,7 +4452,6 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_
 			l3[i] = newl3 | phys;
 			phys += L3_SIZE;
 		}
-		cpu_dcache_wb_range((vm_offset_t)l3, PAGE_SIZE);
 	}
 	KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE),
 	    ("Invalid l3 page (%lx != %lx)", l3[0],



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201707010938.v619cquC028333>