Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 18 Jun 2009 20:42:38 +0000 (UTC)
From:      Andrew Thompson <thompsa@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r194459 - in head/sys: arm/arm arm/include vm
Message-ID:  <200906182042.n5IKgc3t022366@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: thompsa
Date: Thu Jun 18 20:42:37 2009
New Revision: 194459
URL: http://svn.freebsd.org/changeset/base/194459

Log:
  Track the kernel mapping of a physical page by a new entry in vm_page
  structure. When the page is shared, the kernel mapping becomes a special
  type of managed page to force the cache off the page mappings. This is
  needed to avoid stale entries on all ARM VIVT caches, and VIPT caches
  with cache color issue.
  
  Submitted by:	Mark Tinguely
  Reviewed by:	alc
  Tested by:	Grzegorz Bernacki, thompsa

Modified:
  head/sys/arm/arm/pmap.c
  head/sys/arm/include/pmap.h
  head/sys/vm/vm_phys.c

Modified: head/sys/arm/arm/pmap.c
==============================================================================
--- head/sys/arm/arm/pmap.c	Thu Jun 18 19:58:05 2009	(r194458)
+++ head/sys/arm/arm/pmap.c	Thu Jun 18 20:42:37 2009	(r194459)
@@ -200,8 +200,7 @@ static pv_entry_t pmap_get_pv_entry(void
 
 static void		pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t,
     vm_prot_t, boolean_t, int);
-static __inline	void	pmap_fix_cache(struct vm_page *, pmap_t,
-    vm_offset_t);
+static void		pmap_fix_cache(struct vm_page *, pmap_t, vm_offset_t);
 static void		pmap_alloc_l1(pmap_t);
 static void		pmap_free_l1(pmap_t);
 static void		pmap_use_l1(pmap_t);
@@ -406,7 +405,7 @@ int	pmap_needs_pte_sync;
 
 #define pmap_is_current(pm)	((pm) == pmap_kernel() || \
             curproc->p_vmspace->vm_map.pmap == (pm))
-static uma_zone_t pvzone;
+static uma_zone_t pvzone = NULL;
 uma_zone_t l2zone;
 static uma_zone_t l2table_zone;
 static vm_offset_t pmap_kernel_l2dtable_kva;
@@ -1451,6 +1450,7 @@ pmap_fix_cache(struct vm_page *pg, pmap_
 		 * kernel writable or kernel readable with writable user entry
 		 */
 		if ((kwritable && entries) ||
+		    (kwritable > 1) ||
 		    ((kwritable != writable) && kentries &&
 		     (pv->pv_pmap == pmap_kernel() ||
 		      (pv->pv_flags & PVF_WRITE) ||
@@ -1472,7 +1472,8 @@ pmap_fix_cache(struct vm_page *pg, pmap_
 			continue;
 		}
 			/* user is no longer sharable and writable */
-		if (pm != pmap_kernel() && (pv->pv_pmap == pm) &&
+		if (pm != pmap_kernel() &&
+		    (pv->pv_pmap == pm || pv->pv_pmap == pmap_kernel()) &&
 		    !pmwc && (pv->pv_flags & PVF_NC)) {
 
 			pv->pv_flags &= ~(PVF_NC | PVF_MWC);
@@ -1674,7 +1675,29 @@ pmap_enter_pv(struct vm_page *pg, struct
     vm_offset_t va, u_int flags)
 {
 
+	int km;
+
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+
+	if (pg->md.pv_kva) {
+		/* PMAP_ASSERT_LOCKED(pmap_kernel()); */
+		pve->pv_pmap = pmap_kernel();
+		pve->pv_va = pg->md.pv_kva;
+		pve->pv_flags = PVF_WRITE | PVF_UNMAN;
+		pg->md.pv_kva = 0;
+
+		TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list);
+		TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist);
+		if ((km = PMAP_OWNED(pmap_kernel())))
+			PMAP_UNLOCK(pmap_kernel());
+		vm_page_unlock_queues();
+		if ((pve = pmap_get_pv_entry()) == NULL)
+			panic("pmap_kenter_internal: no pv entries");
+		vm_page_lock_queues();
+		if (km)
+			PMAP_LOCK(pmap_kernel());
+	}
+
 	PMAP_ASSERT_LOCKED(pm);
 	pve->pv_pmap = pm;
 	pve->pv_va = va;
@@ -1742,6 +1765,7 @@ static void
 pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve)
 {
 
+	struct pv_entry *pv;
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	PMAP_ASSERT_LOCKED(pm);
 	TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list);
@@ -1766,6 +1790,20 @@ pmap_nuke_pv(struct vm_page *pg, pmap_t 
 			vm_page_flag_clear(pg, PG_WRITEABLE);
 		}
 	}
+	pv = TAILQ_FIRST(&pg->md.pv_list);
+	if (pv != NULL && (pv->pv_flags & PVF_UNMAN) &&
+	    TAILQ_NEXT(pv, pv_list) == NULL) {
+		pg->md.pv_kva = pv->pv_va;
+			/* a recursive pmap_nuke_pv */
+		TAILQ_REMOVE(&pg->md.pv_list, pv, pv_list);
+		TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist);
+		if (pv->pv_flags & PVF_WIRED)
+			--pm->pm_stats.wired_count;
+		pg->md.pvh_attrs &= ~PVF_REF;
+		pg->md.pvh_attrs &= ~PVF_MOD;
+		vm_page_flag_clear(pg, PG_WRITEABLE);
+		pmap_free_pv_entry(pv);
+	}
 }
 
 static struct pv_entry *
@@ -1784,6 +1822,9 @@ pmap_remove_pv(struct vm_page *pg, pmap_
 		pve = TAILQ_NEXT(pve, pv_list);
 	}
 
+	if (pve == NULL && pg->md.pv_kva == va)
+		pg->md.pv_kva = 0;
+
 	return(pve);				/* return removed pve */
 }
 /*
@@ -2711,8 +2752,8 @@ pmap_remove_pages(pmap_t pmap)
 	cpu_idcache_wbinv_all();
 	cpu_l2cache_wbinv_all();
 	for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
-		if (pv->pv_flags & PVF_WIRED) {
-			/* The page is wired, cannot remove it now. */
+		if (pv->pv_flags & PVF_WIRED || pv->pv_flags & PVF_UNMAN) {
+			/* Cannot remove wired or unmanaged pages now. */
 			npv = TAILQ_NEXT(pv, pv_plist);
 			continue;
 		}
@@ -2822,6 +2863,9 @@ pmap_kenter_internal(vm_offset_t va, vm_
 	struct l2_bucket *l2b;
 	pt_entry_t *pte;
 	pt_entry_t opte;
+	struct pv_entry *pve;
+	vm_page_t m;
+
 	PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n",
 	    (uint32_t) va, (uint32_t) pa));
 
@@ -2835,10 +2879,7 @@ pmap_kenter_internal(vm_offset_t va, vm_
 	PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n",
 	    (uint32_t) pte, opte, *pte));
 	if (l2pte_valid(opte)) {
-		cpu_dcache_wbinv_range(va, PAGE_SIZE);
-		cpu_l2cache_wbinv_range(va, PAGE_SIZE);
-		cpu_tlb_flushD_SE(va);
-		cpu_cpwait();
+		pmap_kremove(va);
 	} else {
 		if (opte == 0)
 			l2b->l2b_occupancy++;
@@ -2850,6 +2891,33 @@ pmap_kenter_internal(vm_offset_t va, vm_
 	if (flags & KENTER_USER)
 		*pte |= L2_S_PROT_U;
 	PTE_SYNC(pte);
+
+		/* kernel direct mappings can be shared, so use a pv_entry
+		 * to ensure proper caching.
+		 *
+		 * The pvzone is used to delay the recording of kernel
+		 * mappings until the VM is running.
+		 * 
+		 * This expects the physical memory to have vm_page_array entry.
+		 */
+	if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa))) {
+		vm_page_lock_queues();
+		if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) {
+				/* release vm_page lock for pv_entry UMA */
+			vm_page_unlock_queues();
+			if ((pve = pmap_get_pv_entry()) == NULL)
+				panic("pmap_kenter_internal: no pv entries");	
+			vm_page_lock_queues();
+			PMAP_LOCK(pmap_kernel());
+			pmap_enter_pv(m, pve, pmap_kernel(), va,
+					 PVF_WRITE | PVF_UNMAN);
+			pmap_fix_cache(m, pmap_kernel(), va);
+			PMAP_UNLOCK(pmap_kernel());
+		} else {
+			m->md.pv_kva = va;
+		}
+		vm_page_unlock_queues();
+	}
 }
 
 void
@@ -2886,6 +2954,9 @@ pmap_kremove(vm_offset_t va)
 {
 	struct l2_bucket *l2b;
 	pt_entry_t *pte, opte;
+	struct pv_entry *pve;
+	vm_page_t m;
+	vm_offset_t pa;
 		
 	l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 	if (!l2b)
@@ -2894,6 +2965,25 @@ pmap_kremove(vm_offset_t va)
 	pte = &l2b->l2b_kva[l2pte_index(va)];
 	opte = *pte;
 	if (l2pte_valid(opte)) {
+			/* pa = vtophs(va) taken from pmap_extract() */
+		switch (opte & L2_TYPE_MASK) {
+		case L2_TYPE_L:
+			pa = (opte & L2_L_FRAME) | (va & L2_L_OFFSET);
+			break;
+		default:
+			pa = (opte & L2_S_FRAME) | (va & L2_S_OFFSET);
+			break;
+		}
+			/* note: should never have to remove an allocation
+			 * before the pvzone is initialized.
+			 */
+		vm_page_lock_queues();
+		PMAP_LOCK(pmap_kernel());
+		if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) &&
+		    (pve = pmap_remove_pv(m, pmap_kernel(), va)))
+			pmap_free_pv_entry(pve); 
+		PMAP_UNLOCK(pmap_kernel());
+		vm_page_unlock_queues();
 		cpu_dcache_wbinv_range(va, PAGE_SIZE);
 		cpu_l2cache_wbinv_range(va, PAGE_SIZE);
 		cpu_tlb_flushD_SE(va);
@@ -3137,16 +3227,24 @@ pmap_remove_all(vm_page_t m)
 			cpu_l2cache_inv_range(pv->pv_va, PAGE_SIZE);
 		}
 
-		l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
-		KASSERT(l2b != NULL, ("No l2 bucket"));
-		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
-		*ptep = 0;
-		PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
-		pmap_free_l2_bucket(pv->pv_pmap, l2b, 1);
-		if (pv->pv_flags & PVF_WIRED)
-			pv->pv_pmap->pm_stats.wired_count--;
-		pv->pv_pmap->pm_stats.resident_count--;
-		flags |= pv->pv_flags;
+		if (pv->pv_flags & PVF_UNMAN) {
+			/* remove the pv entry, but do not remove the mapping
+			 * and remember this is a kernel mapped page
+			 */
+			m->md.pv_kva = pv->pv_va;
+		} else {
+			/* remove the mapping and pv entry */
+			l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+			KASSERT(l2b != NULL, ("No l2 bucket"));
+			ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+			*ptep = 0;
+			PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
+			pmap_free_l2_bucket(pv->pv_pmap, l2b, 1);
+			if (pv->pv_flags & PVF_WIRED)
+				pv->pv_pmap->pm_stats.wired_count--;
+			pv->pv_pmap->pm_stats.resident_count--;
+			flags |= pv->pv_flags;
+		}
 		pmap_nuke_pv(m, pv->pv_pmap, pv);
 		PMAP_UNLOCK(pv->pv_pmap);
 		pmap_free_pv_entry(pv);
@@ -3428,25 +3526,19 @@ do_l2b_alloc:
 			 * It is part of our managed memory so we
 			 * must remove it from the PV list
 			 */
-			pve = pmap_remove_pv(opg, pmap, va);
-			if (m && (m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) &&
-			    pve)
-				pmap_free_pv_entry(pve);
-			else if (!pve && 
-			    !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS)))
-				pve = pmap_get_pv_entry();
-			KASSERT(pve != NULL || m->flags & (PG_UNMANAGED | 
-			    PG_FICTITIOUS), ("No pv"));
-			oflags = pve->pv_flags;
-			
+			if ((pve = pmap_remove_pv(opg, pmap, va))) {
+
+			/* note for patch: the oflags/invalidation was moved
+			 * because PG_FICTITIOUS pages could free the pve
+			 */
+			    oflags = pve->pv_flags;
 			/*
 			 * If the old mapping was valid (ref/mod
 			 * emulation creates 'invalid' mappings
 			 * initially) then make sure to frob
 			 * the cache.
 			 */
-			if ((oflags & PVF_NC) == 0 &&
-			    l2pte_valid(opte)) {
+			    if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) {
 				if (PV_BEEN_EXECD(oflags)) {
 					pmap_idcache_wbinv_range(pmap, va,
 					    PAGE_SIZE);
@@ -3456,15 +3548,43 @@ do_l2b_alloc:
 						    PAGE_SIZE, TRUE,
 						    (oflags & PVF_WRITE) == 0);
 					}
-			}
-		} else if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS)))
-			if ((pve = pmap_get_pv_entry()) == NULL) {
-				panic("pmap_enter: no pv entries");	
-			}
-		if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) {
+			    }
+
+			/* free/allocate a pv_entry for UNMANAGED pages if
+			 * this physical page is not/is already mapped.
+			 */
+
+			    if (m && ((m->flags & PG_FICTITIOUS) ||
+				((m->flags & PG_UNMANAGED) &&
+				  !m->md.pv_kva &&
+				 TAILQ_EMPTY(&m->md.pv_list)))) {
+				pmap_free_pv_entry(pve);
+				pve = NULL;
+			    }
+			} else if (m && !(m->flags & PG_FICTITIOUS) &&
+				 (!(m->flags & PG_UNMANAGED) || m->md.pv_kva ||
+				  !TAILQ_EMPTY(&m->md.pv_list)))
+				pve = pmap_get_pv_entry();
+		} else if (m && !(m->flags & PG_FICTITIOUS) &&
+			   (!(m->flags & PG_UNMANAGED) || m->md.pv_kva ||
+			   !TAILQ_EMPTY(&m->md.pv_list)))
+			pve = pmap_get_pv_entry();
+
+		if (m && !(m->flags & PG_FICTITIOUS)) {
 			KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
-			    ("pmap_enter: managed mapping within the clean submap"));
-			pmap_enter_pv(m, pve, pmap, va, nflags);
+		    	("pmap_enter: managed mapping within the clean submap"));
+			if (m->flags & PG_UNMANAGED) {
+				if (!TAILQ_EMPTY(&m->md.pv_list) ||
+				     m->md.pv_kva) {
+					KASSERT(pve != NULL, ("No pv"));
+					nflags |= PVF_UNMAN;
+					pmap_enter_pv(m, pve, pmap, va, nflags);
+				} else
+					m->md.pv_kva = va;
+			} else {
+				KASSERT(pve != NULL, ("No pv"));
+				pmap_enter_pv(m, pve, pmap, va, nflags);
+			}
 		}
 	}
 	/*

Modified: head/sys/arm/include/pmap.h
==============================================================================
--- head/sys/arm/include/pmap.h	Thu Jun 18 19:58:05 2009	(r194458)
+++ head/sys/arm/include/pmap.h	Thu Jun 18 20:42:37 2009	(r194459)
@@ -94,6 +94,7 @@ struct	pv_entry;
 
 struct	md_page {
 	int pvh_attrs;
+	vm_offset_t pv_kva;		/* first kernel VA mapping */
 	TAILQ_HEAD(,pv_entry)	pv_list;
 };
 
@@ -494,6 +495,7 @@ void	pmap_use_minicache(vm_offset_t, vm_
 #define	PVF_EXEC	0x10		/* mapping is executable */
 #define	PVF_NC		0x20		/* mapping is non-cacheable */
 #define	PVF_MWC		0x40		/* mapping is used multiple times in userland */
+#define	PVF_UNMAN	0x80		/* mapping is unmanaged */
 
 void vector_page_setprot(int);
 

Modified: head/sys/vm/vm_phys.c
==============================================================================
--- head/sys/vm/vm_phys.c	Thu Jun 18 19:58:05 2009	(r194458)
+++ head/sys/vm/vm_phys.c	Thu Jun 18 20:42:37 2009	(r194459)
@@ -382,8 +382,7 @@ vm_phys_paddr_to_vm_page(vm_paddr_t pa)
 		if (pa >= seg->start && pa < seg->end)
 			return (&seg->first_page[atop(pa - seg->start)]);
 	}
-	panic("vm_phys_paddr_to_vm_page: paddr %#jx is not in any segment",
-	    (uintmax_t)pa);
+	return (NULL);
 }
 
 /*



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906182042.n5IKgc3t022366>