Date: Mon, 25 Feb 2008 04:17:03 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 136143 for review Message-ID: <200802250417.m1P4H3Hv020277@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=136143 Change 136143 by kmacy@pandemonium:kmacy:xen31 on 2008/02/25 04:16:08 add extensive KTR logging revert to recursive page tables eliminate most uses of writable page tables disable pmap_copy for now (causes SIGBUS) Affected files ... .. //depot/projects/xen31/sys/i386/xen/pmap.c#39 edit Differences ... ==== //depot/projects/xen31/sys/i386/xen/pmap.c#39 (text+ko) ==== @@ -103,6 +103,8 @@ * and to when physical maps must be made correct. */ +#define PMAP_DIAGNOSTIC + #include "opt_cpu.h" #include "opt_pmap.h" #include "opt_msgbuf.h" @@ -112,6 +114,7 @@ #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/ktr.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mman.h> @@ -193,8 +196,6 @@ #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) -#define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \ - atomic_clear_int((u_int *)(pte), PG_W)) #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) struct pmap kernel_pmap_store; @@ -310,7 +311,6 @@ - static __inline void pagezero(void *page) { @@ -331,22 +331,27 @@ pd_set(struct pmap *pmap, int ptepindex, vm_paddr_t val, int type) { vm_paddr_t pdir_ma = vtomach(&pmap->pm_pdir[ptepindex]); - vm_paddr_t shadow_pdir_ma = vtomach(&pmap->pm_pdir_shadow[ptepindex]); switch (type) { case SH_PD_SET_VA: +#if 0 xen_queue_pt_update(shadow_pdir_ma, xpmap_ptom(val & ~(PG_RW))); +#endif xen_queue_pt_update(pdir_ma, xpmap_ptom(val)); break; case SH_PD_SET_VA_MA: +#if 0 xen_queue_pt_update(shadow_pdir_ma, val & ~(PG_RW)); +#endif xen_queue_pt_update(pdir_ma, val); break; case SH_PD_SET_VA_CLEAR: +#if 0 xen_queue_pt_update(shadow_pdir_ma, 0); +#endif xen_queue_pt_update(pdir_ma, 0); break; } @@ -409,7 +414,6 @@ #ifdef PAE kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT); #endif - kernel_pmap->pm_pdir_shadow = PCPU_GET(pdir_shadow); kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); LIST_INIT(&allpmaps); @@ -441,7 +445,7 @@ } SYSMAP(caddr_t, CMAP1, CADDR1, 1) SYSMAP(caddr_t, CMAP3, CADDR3, 1) - *CMAP3 = 0; + PT_SET_MA(CADDR3, 0); /* * Crashdump maps. @@ -467,9 +471,8 @@ mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF); virtual_avail = va; + PT_SET_MA(CADDR1, 0); - *CMAP1 = 0; - /* * Leave in place an identity mapping (virt == phys) for the low 1 MB * physical memory region that is used by the ACPI wakeup code. This @@ -809,6 +812,9 @@ u_int cpumask; u_int other_cpus; + CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x", + pmap, va); + sched_pin(); if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { invlpg(va); @@ -832,6 +838,9 @@ u_int other_cpus; vm_offset_t addr; + CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x", + pmap, sva, eva); + sched_pin(); if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { for (addr = sva; addr < eva; addr += PAGE_SIZE) @@ -857,6 +866,8 @@ u_int cpumask; u_int other_cpus; + CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap); + sched_pin(); if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { invltlb(); @@ -889,6 +900,8 @@ PMAP_INLINE void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { + CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x", + pmap, va); if (pmap == kernel_pmap || pmap->pm_active) invlpg(va); @@ -900,6 +913,10 @@ { vm_offset_t addr; + if (eva - sva > PAGE_SIZE) + CTR3(KTR_PMAP, "pmap_invalidate_range: pmap=%p sva=0x%x eva=0x%x", + pmap, sva, eva); + if (pmap == kernel_pmap || pmap->pm_active) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); @@ -910,6 +927,8 @@ pmap_invalidate_all(pmap_t pmap) { + CTR1(KTR_PMAP, "pmap_invalidate_all: pmap=%p", pmap); + if (pmap == kernel_pmap || pmap->pm_active) invltlb(); } @@ -933,8 +952,8 @@ { return (pmap == kernel_pmap || - (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) && - (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME))); + (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) && + (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME))); } /* @@ -958,9 +977,11 @@ newpf = *pde & PG_FRAME; if ((*PMAP2 & PG_FRAME) != newpf) { PT_UPDATES_FLUSH(); - *PMAP2 = newpf | PG_V | PG_A | PG_M; - pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2); + PT_SET_MA(PADDR2, newpf | PG_V | PG_A | PG_M); + CTR3(KTR_PMAP, "pmap_pte: pmap=%p va=0x%x newpte=0x%08x", + pmap, va, (*PMAP2 & 0xffffffff)); } + return (PADDR2 + (i386_btop(va) & (NPTEPG - 1))); } return (0); @@ -975,7 +996,9 @@ { if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2) { - *PMAP2 = 0; + CTR1(KTR_PMAP, "pmap_pte_release: pte=0x%jx", + *PMAP2); + PT_SET_VA(PMAP2, 0, TRUE); mtx_unlock(&PMAP2mutex); } } @@ -1016,11 +1039,13 @@ newpf = *pde & PG_FRAME; if ((*PMAP1 & PG_FRAME) != newpf) { PT_UPDATES_FLUSH(); - *PMAP1 = newpf | PG_V | PG_A | PG_M; + PT_SET_MA(PADDR1, newpf | PG_V | PG_A | PG_M); + CTR3(KTR_PMAP, "pmap_pte_quick: pmap=%p va=0x%x newpte=0x%jx", + pmap, va, *PMAP1); + #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif - invlcaddr(PADDR1); PMAP1changed++; } else #ifdef SMP @@ -1048,7 +1073,8 @@ vm_paddr_t rtval; pt_entry_t *pte; pd_entry_t pde; - + pt_entry_t pteval; + rtval = 0; PMAP_LOCK(pmap); pde = pmap->pm_pdir[va >> PDRSHIFT]; @@ -1059,7 +1085,8 @@ return rtval; } pte = pmap_pte(pmap, va); - rtval = (PT_GET(pte) & PG_FRAME) | (va & PAGE_MASK); + pteval = *pte ? xpmap_mtop(*pte) : 0; + rtval = (pteval & PG_FRAME) | (va & PAGE_MASK); pmap_pte_release(pte); } PMAP_UNLOCK(pmap); @@ -1124,7 +1151,7 @@ sched_pin(); pte = PT_GET(pmap_pte_quick(pmap, va)); if (*PMAP1) - *PMAP1 = 0; + PT_SET_MA(PADDR1, 0); if ((pte & PG_V) && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { m = PHYS_TO_VM_PAGE(pte & PG_FRAME); @@ -1149,10 +1176,7 @@ PMAP_INLINE void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { - pt_entry_t *pte; - - pte = vtopte(va); - pte_store(pte, pa | PG_RW | PG_V | pgeflag); + PT_SET_MA(va, xpmap_ptom(pa)| PG_RW | PG_V | pgeflag); } PMAP_INLINE void @@ -1205,6 +1229,8 @@ vm_offset_t va, sva; va = sva = *virt; + CTR4(KTR_PMAP, "pmap_map: va=0x%x start=0x%jx end=0x%jx prot=0x%x", + va, start, end, prot); while (start < end) { pmap_kenter(va, start); va += PAGE_SIZE; @@ -1235,7 +1261,8 @@ multicall_entry_t mcl[16]; multicall_entry_t *mclp = mcl; int error; - + + CTR2(KTR_PMAP, "pmap_qenter:sva=0x%x count=%d", va, count); pte = vtopte(sva); endpte = pte + count; while (pte < endpte) { @@ -1266,7 +1293,7 @@ #ifdef INVARIANTS for (pte = vtopte(sva), mclcount = 0; mclcount < count; mclcount++, pte++) - KASSERT(*pte, ("pte not set for va=0x%x\n", sva + mclcount*PAGE_SIZE)); + KASSERT(*pte, ("pte not set for va=0x%x", sva + mclcount*PAGE_SIZE)); #endif } @@ -1281,12 +1308,17 @@ { vm_offset_t va; + CTR2(KTR_PMAP, "pmap_qremove: sva=0x%x count=%d", sva, count); va = sva; + vm_page_lock_queues(); + critical_enter(); while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); + critical_exit(); + vm_page_unlock_queues(); } /*************************************************** @@ -1324,6 +1356,7 @@ { vm_offset_t pteva; + PT_UPDATES_FLUSH(); /* * unmap the page table page */ @@ -1401,8 +1434,8 @@ int pmap_pinit(pmap_t pmap) { - vm_page_t m, ptdpg[NPGPTD*2 + 1]; - int npgptd = NPGPTD*2 + 1; + vm_page_t m, ptdpg[NPGPTD + 1]; + int npgptd = NPGPTD + 1; static int color; int i; @@ -1419,17 +1452,9 @@ PMAP_LOCK_DESTROY(pmap); return (0); } -#ifdef XEN - pmap->pm_pdir_shadow = (pd_entry_t *)kmem_alloc_nofault(kernel_map, - NBPTD); - if (pmap->pm_pdir == NULL) { - PMAP_LOCK_DESTROY(pmap); - return (0); - } -#ifdef PAE +#if defined(XEN) && defined(PAE) pmap->pm_pdpt = (pd_entry_t *)kmem_alloc_nofault(kernel_map, 1); #endif -#endif #if defined(PAE) && !defined(XEN) pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO); @@ -1454,12 +1479,10 @@ ptdpg[i++] = m; } } - pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD); - for (i = 0; i < NPGPTD; i++) { if ((ptdpg[i]->flags & PG_ZERO) == 0) - bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE); + pagezero(&pmap->pm_pdir[i*NPTEPG]); } mtx_lock_spin(&allpmaps_lock); @@ -1470,8 +1493,8 @@ bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t)); #ifdef PAE #ifdef XEN - pmap_qenter((vm_offset_t)pmap->pm_pdpt, &ptdpg[NPGPTD*2], 1); - if ((ptdpg[NPGPTD*2]->flags & PG_ZERO) == 0) + pmap_qenter((vm_offset_t)pmap->pm_pdpt, &ptdpg[NPGPTD], 1); + if ((ptdpg[NPGPTD]->flags & PG_ZERO) == 0) bzero(pmap->pm_pdpt, PAGE_SIZE); #endif for (i = 0; i < NPGPTD; i++) { @@ -1483,33 +1506,31 @@ } #endif #ifdef XEN - pmap_qenter((vm_offset_t)pmap->pm_pdir_shadow, &ptdpg[NPGPTD], NPGPTD); - - for (i = 0; i < NPGPTD; i++) - if ((ptdpg[i + NPGPTD]->flags & PG_ZERO) == 0) - bzero(pmap->pm_pdir_shadow + (i * NPDEPG), PAGE_SIZE); - for (i = 0; i < NPGPTD; i++) - pmap->pm_pdir_shadow[PTDPTDI + i] = - xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i])) | PG_V; - for (i = 0; i < NPGPTD; i++) - pmap->pm_pdir[PTDPTDI + i] = - xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i + NPGPTD])) | PG_V; - for (i = KPTDI; i < KPTDI + nkpt; i++) - pmap->pm_pdir_shadow[i] = PTD[i] & ~(PG_RW); for (i = 0; i < NPGPTD; i++) { pt_entry_t *pd; - + vm_paddr_t ma; + + ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i])); pd = pmap->pm_pdir + (i * NPDEPG); PT_SET_MA(pd, *vtopte((vm_offset_t)pd) & ~(PG_M|PG_A|PG_U|PG_RW)); - pd = pmap->pm_pdir_shadow + (i * NPDEPG); - PT_SET_MA(pd, *vtopte((vm_offset_t)pd) & ~(PG_M|PG_A|PG_U|PG_RW)); +#if 0 + xen_pgd_pin(ma); +#endif } + #ifdef PAE PT_SET_MA(pmap->pm_pdpt, *vtopte((vm_offset_t)pmap->pm_pdpt) & ~PG_RW); #endif + vm_page_lock_queues(); xen_flush_queue(); + xen_pgdpt_pin(xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD]))); + for (i = 0; i < NPGPTD; i++) { + vm_paddr_t ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i])); + PT_SET_VA_MA(&pmap->pm_pdir[PTDPTDI + i], ma | PG_V | PG_A, FALSE); + } + xen_flush_queue(); + vm_page_unlock_queues(); #endif - pmap->pm_active = 0; TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1522,7 +1543,7 @@ * mapped correctly. */ static vm_page_t -_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) +_pmap_allocpte(pmap_t pmap, unsigned int ptepindex, int flags) { vm_paddr_t ptema; vm_page_t m; @@ -1557,15 +1578,16 @@ * Map the pagetable page into the process address space, if * it isn't already there. */ - pmap->pm_stats.resident_count++; ptema = xpmap_ptom(VM_PAGE_TO_PHYS(m)); xen_pt_pin(ptema); - PD_SET_VA_MA(pmap, ptepindex, + PT_SET_VA_MA(&pmap->pm_pdir[ptepindex], (ptema | PG_U | PG_RW | PG_V | PG_A | PG_M), TRUE); - - return m; + + KASSERT(pmap->pm_pdir[ptepindex], + ("_pmap_allocpte: ptepindex=%d did not get mapped", ptepindex)); + return (m); } static vm_page_t @@ -1615,9 +1637,13 @@ * Here if the pte page isn't mapped, or if it has * been deallocated. */ + CTR3(KTR_PMAP, "pmap_allocpte: pmap=%p va=0x%08x flags=0x%x", + pmap, va, flags); m = _pmap_allocpte(pmap, ptepindex, flags); if (m == NULL && (flags & M_WAITOK)) goto retry; + + KASSERT(pmap->pm_pdir[ptepindex], ("ptepindex=%d did not get mapped", ptepindex)); } return (m); } @@ -1733,9 +1759,9 @@ int i; #ifdef XEN #ifdef PAE - int npgptd = 2*NPGPTD + 1; + int npgptd = NPGPTD + 1; #else - int npgptd = 2*NPGPTD; + int npgptd = NPGPTD; #endif #else int npgptd = NPGPTD; @@ -1750,16 +1776,11 @@ LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); -#ifdef XEN for (i = 0; i < NPGPTD; i++) - ptdpg[i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir_shadow + (i*NPDEPG)) & PG_FRAME); - pmap_qremove((vm_offset_t)pmap->pm_pdir_shadow, NPGPTD); -#endif - for (i = 0; i < NPGPTD; i++) - ptdpg[NPGPTD + i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir + (i*NPDEPG)) & PG_FRAME); + ptdpg[i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir + (i*NPDEPG)) & PG_FRAME); pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD); #if defined(PAE) && defined(XEN) - ptdpg[2*NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdpt)); + ptdpg[NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdpt)); #endif for (i = 0; i < npgptd; i++) { @@ -1852,9 +1873,8 @@ mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) - PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, FALSE); + PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE); - PT_UPDATES_FLUSH(); mtx_unlock_spin(&allpmaps_lock); kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); if (kernel_vm_end - 1 >= kernel_map->max_offset) { @@ -2176,9 +2196,13 @@ pt_entry_t oldpte; vm_page_t m; + CTR3(KTR_PMAP, "pmap_remove_pte: pmap=%p *ptq=0x%jx va=0x%x", + pmap, *ptq, va); + mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); - oldpte = pte_load_clear(ptq); + oldpte = *ptq; + PT_SET_VA_MA(ptq, 0, TRUE); if (oldpte & PG_W) pmap->pm_stats.wired_count -= 1; /* @@ -2194,10 +2218,7 @@ * why, but I think the most likely explanation is that xen's writable * page table implementation doesn't respect the unused bits. */ - if ((oldpte & PG_MANAGED) -#ifdef PTEBROKEN - || ((oldpte & PG_V) && (va < VM_MAXUSER_ADDRESS)) -#endif + if ((oldpte & PG_MANAGED) || ((oldpte & PG_V) && (va < VM_MAXUSER_ADDRESS)) ) { m = PHYS_TO_VM_PAGE(xpmap_mtop(oldpte) & PG_FRAME); @@ -2227,6 +2248,9 @@ { pt_entry_t *pte; + CTR2(KTR_PMAP, "pmap_remove_page: pmap=%p va=0x%x", + pmap, va); + mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); PMAP_LOCK_ASSERT(pmap, MA_OWNED); @@ -2235,7 +2259,7 @@ pmap_remove_pte(pmap, pte, va, free); pmap_invalidate_page(pmap, va); if (*PMAP1) - *PMAP1 = 0; + PT_SET_MA(PADDR1, 0); } @@ -2253,7 +2277,10 @@ pt_entry_t *pte; vm_page_t free = NULL; int anyvalid; - + + CTR3(KTR_PMAP, "pmap_remove: pmap=%p sva=0x%x eva=0x%x", + pmap, sva, eva); + /* * Perform an unsynchronized read. This is, however, safe. */ @@ -2330,12 +2357,13 @@ break; } } + PT_UPDATES_FLUSH(); if (*PMAP1) - *PMAP1 = 0; + PT_SET_VA_MA(PMAP1, 0, TRUE); out: - sched_unpin(); if (anyvalid) pmap_invalidate_all(pmap); + sched_unpin(); vm_page_unlock_queues(); PMAP_UNLOCK(pmap); pmap_free_zero_pages(free); @@ -2367,8 +2395,8 @@ * XXX This makes pmap_remove_all() illegal for non-managed pages! */ if (m->flags & PG_FICTITIOUS) { - panic("pmap_remove_all: illegal for unmanaged page, va: 0x%x", - VM_PAGE_TO_PHYS(m)); + panic("pmap_remove_all: illegal for unmanaged page, va: 0x%jx", + VM_PAGE_TO_PHYS(m) & 0xffffffff); } #endif mtx_assert(&vm_page_queue_mtx, MA_OWNED); @@ -2378,7 +2406,9 @@ PMAP_LOCK(pmap); pmap->pm_stats.resident_count--; pte = pmap_pte_quick(pmap, pv->pv_va); - tpte = pte_load_clear(pte); + + tpte = *pte; + PT_SET_VA_MA(pte, 0, TRUE); if (tpte & PG_W) pmap->pm_stats.wired_count--; if (tpte & PG_A) @@ -2402,8 +2432,9 @@ PMAP_UNLOCK(pmap); } vm_page_flag_clear(m, PG_WRITEABLE); + PT_UPDATES_FLUSH(); if (*PMAP1) - *PMAP1 = 0; + PT_SET_MA(PADDR1, 0); sched_unpin(); } @@ -2419,6 +2450,9 @@ pt_entry_t *pte; int anychanged; + CTR4(KTR_PMAP, "pmap_protect: pmap=%p sva=0x%x eva=0x%x prot=0x%x", + pmap, sva, eva, prot); + if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; @@ -2506,6 +2540,12 @@ #endif if (pbits != obits) { +#ifdef XEN + obits = *pte; + PT_SET_VA_MA(pte, pbits, TRUE); + if (*pte != pbits) + goto retry; +#else #ifdef PAE if (!atomic_cmpset_64(pte, obits, pbits)) goto retry; @@ -2514,6 +2554,7 @@ pbits)) goto retry; #endif +#endif if (obits & PG_G) pmap_invalidate_page(pmap, sva); else @@ -2521,11 +2562,12 @@ } } } + PT_UPDATES_FLUSH(); if (*PMAP1) - *PMAP1 = 0; - sched_unpin(); + PT_SET_VA_MA(PMAP1, 0, TRUE); if (anychanged) pmap_invalidate_all(pmap); + sched_unpin(); vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -2554,6 +2596,8 @@ vm_page_t mpte, om; boolean_t invlva; + CTR6(KTR_PMAP, "pmap_enter: pmap=%08p va=0x%08x access=0x%x m=%p prot=0x%x wired=%d", + pmap, va, access, m, prot, wired); va = trunc_page(va); #ifdef PMAP_DIAGNOSTIC if (va > VM_MAX_KERNEL_ADDRESS) @@ -2596,7 +2640,7 @@ */ if (pte == NULL) { panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x\n", - (uintmax_t)pmap->pm_pdir[PTDPTDI], va); + (uintmax_t)pmap->pm_pdir[va >> PDRSHIFT], va); } pa = VM_PAGE_TO_PHYS(m); @@ -2606,11 +2650,12 @@ #if 0 KASSERT((*pte & PG_V) || (*pte == 0), ("address set but not valid pte=%p *pte=0x%016jx", pte, *pte)); -#endif - origpte = PT_GET(pte); +#endif + origpte = *pte; + if (origpte) + origpte = xpmap_mtop(origpte); opa = origpte & PG_FRAME; - /* * Mapping has not changed, must be protection or wiring change. */ @@ -2700,6 +2745,7 @@ if (pmap == kernel_pmap) newpte |= pgeflag; + critical_enter(); /* * if the mapping or permission bits are different, we need * to update the pte. @@ -2707,7 +2753,8 @@ if ((origpte & ~(PG_M|PG_A)) != newpte) { if (origpte) { invlva = FALSE; - origpte = pte_load_store(pte, newpte | PG_A); + origpte = *pte; + PT_SET_VA(pte, newpte | PG_A, FALSE); if (origpte & PG_A) { if (origpte & PG_MANAGED) vm_page_flag_set(om, PG_REFERENCED); @@ -2730,14 +2777,17 @@ } if (invlva) pmap_invalidate_page(pmap, va); - } else - pte_store(pte, newpte | PG_A); + } else{ + PT_SET_VA(pte, newpte | PG_A, FALSE); + } + } + PT_UPDATES_FLUSH(); + critical_exit(); if (*PMAP1) - *PMAP1 = 0; + PT_SET_VA_MA(PMAP1, 0, TRUE); sched_unpin(); vm_page_unlock_queues(); - KASSERT(*vtopte(va), ("pte not set for va=0x%x\n", va)); PMAP_UNLOCK(pmap); } @@ -2804,6 +2854,9 @@ int count = 0; mclp = &mcl; + CTR4(KTR_PMAP, "pmap_enter_quick: pmap=%p va=0x%x m=%p prot=0x%x", + pmap, va, m, prot); + PMAP_LOCK(pmap); (void) pmap_enter_quick_locked(&mclp, &count, pmap, va, m, prot, NULL); if (count) @@ -2900,6 +2953,7 @@ * quick entry into any pmap, one would likely use pmap_pte_quick. * But that isn't as quick as vtopte. */ + KASSERT(pmap_is_current(pmap), ("entering pages in non-current pmap")); pte = vtopte(va); if (*pte & PG_V) { if (mpte != NULL) { @@ -3040,6 +3094,7 @@ PMAP_LOCK(pmap); pmap->pm_stats.resident_count += size >> PAGE_SHIFT; npdes = size >> PDRSHIFT; + critical_enter(); for(i = 0; i < npdes; i++) { PD_SET_VA(pmap, ptepindex, ptepa | PG_U | PG_M | PG_RW | PG_V | PG_PS, FALSE); @@ -3047,6 +3102,7 @@ ptepindex += 1; } pmap_invalidate_all(pmap); + critical_exit(); out: PMAP_UNLOCK(pmap); } @@ -3064,21 +3120,25 @@ { pt_entry_t *pte; + vm_page_lock_queues(); PMAP_LOCK(pmap); pte = pmap_pte(pmap, va); - if (wired && !pmap_pte_w(pte)) + if (wired && !pmap_pte_w(pte)) { + PT_SET_VA_MA((pte), *(pte) | PG_W, TRUE); pmap->pm_stats.wired_count++; - else if (!wired && pmap_pte_w(pte)) + } else if (!wired && pmap_pte_w(pte)) { + PT_SET_VA_MA((pte), *(pte) & ~PG_W, TRUE); pmap->pm_stats.wired_count--; - + } + /* * Wiring is not a hardware characteristic so there is no need to * invalidate TLB. */ - pmap_pte_set_w(pte, wired); pmap_pte_release(pte); PMAP_UNLOCK(pmap); + vm_page_unlock_queues(); } @@ -3100,12 +3160,21 @@ vm_offset_t end_addr = src_addr + len; vm_offset_t pdnxt; + return; + if (dst_addr != src_addr) return; - if (!pmap_is_current(src_pmap)) + if (!pmap_is_current(src_pmap)) { + CTR2(KTR_PMAP, + "pmap_copy, skipping: pdir[PTDPTDI]=0x%jx PTDpde[0]=0x%jx", + (src_pmap->pm_pdir[PTDPTDI] & PG_FRAME), (PTDpde[0] & PG_FRAME)); + return; - + } + CTR5(KTR_PMAP, "pmap_copy: dst_pmap=%p src_pmap=%p dst_addr=0x%x len=%d src_addr=0x%x", + dst_pmap, src_pmap, dst_addr, len, src_addr); + vm_page_lock_queues(); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); @@ -3168,11 +3237,7 @@ * accessed (referenced) bits * during the copy. */ -#if 0 PT_SET_VA_MA(dst_pte, ptetemp & ~(PG_W | PG_M | PG_A), FALSE); -#else - *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); -#endif dst_pmap->pm_stats.resident_count++; } else { free = NULL; @@ -3190,9 +3255,9 @@ src_pte++; } } + PT_UPDATES_FLUSH(); if (*PMAP1) - *PMAP1 = 0; - PT_UPDATES_FLUSH(); + PT_SET_VA_MA(PMAP1, 0, TRUE); sched_unpin(); vm_page_unlock_queues(); PMAP_UNLOCK(src_pmap); @@ -3213,10 +3278,9 @@ if (*sysmaps->CMAP2) panic("pmap_zero_page: CMAP2 busy"); sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M; - invlcaddr(sysmaps->CADDR2); + PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M); pagezero(sysmaps->CADDR2); - *sysmaps->CMAP2 = 0; + PT_SET_MA(sysmaps->CADDR2, 0); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -3237,14 +3301,13 @@ if (*sysmaps->CMAP2) panic("pmap_zero_page: CMAP2 busy"); sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M; + PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M); - invlcaddr(sysmaps->CADDR2); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else bzero((char *)sysmaps->CADDR2 + off, size); - *sysmaps->CMAP2 = 0; + PT_SET_MA(sysmaps->CADDR2, 0); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -3262,10 +3325,9 @@ if (*CMAP3) panic("pmap_zero_page: CMAP3 busy"); sched_pin(); - *CMAP3 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M; - invlcaddr(CADDR3); + PT_SET_MA(CADDR3, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M); pagezero(CADDR3); - *CMAP3 = 0; + PT_SET_MA(CADDR3, 0); sched_unpin(); } @@ -3287,13 +3349,11 @@ if (*sysmaps->CMAP2) panic("pmap_copy_page: CMAP2 busy"); sched_pin(); - invlpg((u_int)sysmaps->CADDR1); - invlpg((u_int)sysmaps->CADDR2); - *sysmaps->CMAP1 = PG_V | xpmap_ptom(VM_PAGE_TO_PHYS(src)) | PG_A; - *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(dst)) | PG_A | PG_M; + PT_SET_MA(sysmaps->CADDR1, PG_V | xpmap_ptom(VM_PAGE_TO_PHYS(src)) | PG_A); + PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(dst)) | PG_A | PG_M); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); - *sysmaps->CMAP1 = 0; - *sysmaps->CMAP2 = 0; + PT_SET_MA(sysmaps->CADDR1, 0); + PT_SET_MA(sysmaps->CADDR2, 0); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -3312,7 +3372,7 @@ int loops = 0; if (m->flags & PG_FICTITIOUS) - return FALSE; + return (FALSE); mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { @@ -3377,11 +3437,14 @@ uint32_t inuse, bitmask; int allfree; + CTR1(KTR_PMAP, "pmap_remove_pages: pmap=%p", pmap); + if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } vm_page_lock_queues(); + KASSERT(pmap_is_current(pmap), ("removing pages from non-current pmap")); PMAP_LOCK(pmap); sched_pin(); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { @@ -3395,19 +3458,8 @@ pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; - if (pmap->pm_pdir_shadow[pv->pv_va >> PDRSHIFT] == 0) { - printf("PDIR IS ZERO @ VA %08x\n", pv->pv_va); - /* workaround insufficient wired count - * on page directory - this only buys - * us a little bit of time as the list - * on one of the pages is now corrupt - */ - allfree = 0; - goto skip; - } - pte = vtopte(pv->pv_va); - tpte = PT_GET(pte); + tpte = *pte ? xpmap_mtop(*pte) : 0; if (tpte == 0) { printf( @@ -3448,7 +3500,6 @@ vm_page_flag_clear(m, PG_WRITEABLE); pmap_unuse_pt(pmap, pv->pv_va, &free); - skip: /* Mark free */ PV_STAT(pv_entry_frees++); @@ -3471,8 +3522,10 @@ pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); } } - if (*PMAP1) - *PMAP1 = 0; + PT_UPDATES_FLUSH(); + if (*PMAP1) + PT_SET_MA(PADDR1, 0); + sched_unpin(); pmap_invalidate_all(pmap); vm_page_unlock_queues(); @@ -3510,7 +3563,7 @@ break; } if (*PMAP1) - *PMAP1 = 0; + PT_SET_MA(PADDR1, 0); sched_unpin(); return (rv); } @@ -3529,9 +3582,9 @@ return (rv); - if (*pmap_pde(pmap, addr)) { + if (pmap_is_current(pmap) && *pmap_pde(pmap, addr)) { pte = vtopte(addr); - rv = ((*pte & PG_V) == 0); + rv = (*pte == 0); } return (rv); } @@ -3610,8 +3663,9 @@ PMAP_UNLOCK(pmap); } vm_page_flag_clear(m, PG_WRITEABLE); + PT_UPDATES_FLUSH(); if (*PMAP1) - *PMAP1 = 0; + PT_SET_MA(PADDR1, 0); sched_unpin(); } @@ -3658,8 +3712,10 @@ >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200802250417.m1P4H3Hv020277>