Date: Tue, 13 Nov 2007 07:54:15 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 129014 for review Message-ID: <200711130754.lAD7sFG4049115@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=129014 Change 129014 by kmacy@entropy_kmacy_xen31 on 2007/11/13 07:54:05 first cut at integrating xen support into latest pmap.c Affected files ... .. //depot/projects/xen31/sys/i386/xen/pmap.c#2 edit Differences ... ==== //depot/projects/xen31/sys/i386/xen/pmap.c#2 (text+ko) ==== @@ -150,6 +150,9 @@ #include <machine/xbox.h> #endif +#include <machine/xen/xenvar.h> +#include <machine/xen/xenfunc.h> + #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU) #define CPU_ENABLE_SSE #endif @@ -187,8 +190,12 @@ #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) -#define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \ - atomic_clear_int((u_int *)(pte), PG_W)) +#define pmap_pte_set_w(pte, v) { \ + if (v) \ + PT_SET_VA_MA(pte, *pte | PG_W, TRUE); \ + else \ + PT_SET_VA_MA(pte, *pte & ~PG_W, TRUE); \ +} #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) struct pmap kernel_pmap_store; @@ -248,6 +255,7 @@ static pt_entry_t *PMAP1 = 0, *PMAP2; static pt_entry_t *PADDR1 = 0, *PADDR2; #ifdef SMP +static int PMAP1_inuse = 0, PMAP2_inuse = 0; static int PMAP1cpu; static int PMAP1changedcpu; SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, @@ -287,6 +295,8 @@ static void pmap_pte_release(pt_entry_t *pte); static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *); static vm_offset_t pmap_kmem_choose(vm_offset_t addr); +static void pmap_copy_ma_range(vm_paddr_t src, vm_paddr_t dst, uint32_t soff, uint32_t eoff); + #ifdef PAE static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); #endif @@ -294,6 +304,31 @@ CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t)); CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t)); +void +pd_set(struct pmap *pmap, vm_paddr_t *ptr, vm_paddr_t val, int type) +{ + vm_paddr_t shadow_pdir_ma = pmap->pm_pdir[PTDPTDI] & ~0xFFF; + vm_paddr_t shadow_offset = (vm_paddr_t)(ptr - pmap->pm_pdir)*sizeof(vm_paddr_t); + + switch (type) { + case SH_PD_SET_VA: + xen_queue_pt_update(shadow_pdir_ma + shadow_offset, + xpmap_ptom(val & ~(PG_RW|PG_M))); + xen_queue_pt_update(vtomach(ptr), + xpmap_ptom(val)); + break; + case SH_PD_SET_VA_MA: + xen_queue_pt_update(shadow_pdir_ma + shadow_offset, + val & ~(PG_RW|PG_M)); + xen_queue_pt_update(vtomach(ptr), val); + break; + case SH_PD_SET_VA_CLEAR: + xen_queue_pt_update(shadow_pdir_ma + shadow_offset, 0); + xen_queue_pt_update(vtomach(ptr), 0); + break; + } +} + /* * Move the kernel virtual free pointer to the next * 4MB. This is used to help improve performance @@ -382,7 +417,7 @@ } SYSMAP(caddr_t, CMAP1, CADDR1, 1) SYSMAP(caddr_t, CMAP3, CADDR3, 1) - *CMAP3 = 0; + PT_CLEAR_VA(CMAP3, TRUE); /* * Crashdump maps. @@ -408,14 +443,17 @@ mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF); virtual_avail = va; - - *CMAP1 = 0; - + PT_CLEAR_VA(CMAP1, TRUE); + /* * Leave in place an identity mapping (virt == phys) for the low 1 MB * physical memory region that is used by the ACPI wakeup code. This * mapping must not have PG_G set. */ +#ifndef XEN + /* + * leave here deliberately to show that this is not supported + */ #ifdef XBOX /* FIXME: This is gross, but needed for the XBOX. Since we are in such * an early stadium, we cannot yet neatly map video memory ... :-( @@ -430,6 +468,7 @@ /* Turn on PG_G on kernel page(s) */ pmap_set_pg(); +#endif } /* @@ -555,6 +594,11 @@ pt_entry_t *pte; vm_offset_t va; + panic("IMPLEMENT XXX"); + + + + va = *head; if (va == 0) return (va); /* Out of memory */ @@ -754,6 +798,7 @@ smp_masked_invlpg(pmap->pm_active & other_cpus, va); } sched_unpin(); + PT_UPDATES_FLUSH(); } void @@ -779,6 +824,7 @@ sva, eva); } sched_unpin(); + PT_UPDATES_FLUSH(); } void @@ -822,6 +868,7 @@ if (pmap == kernel_pmap || pmap->pm_active) invlpg(va); + PT_UPDATES_FLUSH(); } PMAP_INLINE void @@ -832,6 +879,7 @@ if (pmap == kernel_pmap || pmap->pm_active) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); + PT_UPDATES_FLUSH(); } PMAP_INLINE void @@ -874,18 +922,22 @@ { pd_entry_t newpf; pd_entry_t *pde; - + pd_entry_t tmppf; + pde = pmap_pde(pmap, va); - if (*pde & PG_PS) + if (PT_GET(pde) & PG_PS) return (pde); - if (*pde != 0) { + if (PT_GET(pde) != 0) { /* are we current address space or kernel? */ if (pmap_is_current(pmap)) return (vtopte(va)); mtx_lock(&PMAP2mutex); - newpf = *pde & PG_FRAME; - if ((*PMAP2 & PG_FRAME) != newpf) { - *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M; + newpf = PT_GET(pde) & PG_FRAME; + tmppf = PT_GET(PMAP2) & PG_FRAME; + + if (tmppf != newpf) { + PT_SET_VA(PMAP2, newpf | PG_V | PG_A | PG_M/* XXX does PG_M cause problems? */, TRUE); + pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2); } return (PADDR2 + (i386_btop(va) & (NPTEPG - 1))); @@ -910,6 +962,7 @@ { invlpg((u_int)caddr); + PT_UPDATES_FLUSH(); } /* @@ -927,7 +980,8 @@ { pd_entry_t newpf; pd_entry_t *pde; - + pd_entry_t tmppf; + pde = pmap_pde(pmap, va); if (*pde & PG_PS) return (pde); @@ -937,9 +991,11 @@ return (vtopte(va)); mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); - newpf = *pde & PG_FRAME; - if ((*PMAP1 & PG_FRAME) != newpf) { - *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M; + + newpf = PT_GET(pde) & PG_FRAME; + tmppf = PT_GET(PMAP1) & PG_FRAME; + if (tmppf != newpf) { + PT_SET_VA(PMAP1, newpf | PG_V | PG_A | PG_M/* ??? */, TRUE); #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif @@ -974,16 +1030,55 @@ rtval = 0; PMAP_LOCK(pmap); + pde = PT_GET(&pmap->pm_pdir[va >> PDRSHIFT]); + if (pde != 0) { + if ((pde & PG_PS) != 0) { + rtval = (pde & PG_PS_FRAME) | (va & PDRMASK); + PMAP_UNLOCK(pmap); + return rtval; + } + pte = pmap_pte(pmap, va); + rtval = (PT_GET(pte) & PG_FRAME) | (va & PAGE_MASK); + pmap_pte_release(pte); + if (PMAP2_inuse) { + PT_CLEAR_VA(PMAP2, TRUE); + PMAP2_inuse = 0; + } + } + PMAP_UNLOCK(pmap); + return (rtval); +} + +/* + * Routine: pmap_extract_ma + * Function: + * Like pmap_extract, but returns machine address + */ +vm_paddr_t +pmap_extract_ma(pmap_t pmap, vm_offset_t va) +{ + vm_paddr_t rtval; + pt_entry_t *pte; + pd_entry_t pde; + + rtval = 0; + PMAP_LOCK(pmap); pde = pmap->pm_pdir[va >> PDRSHIFT]; if (pde != 0) { if ((pde & PG_PS) != 0) { - rtval = (pde & PG_PS_FRAME) | (va & PDRMASK); + rtval = (pde & ~PDRMASK) | (va & PDRMASK); PMAP_UNLOCK(pmap); return rtval; } pte = pmap_pte(pmap, va); rtval = (*pte & PG_FRAME) | (va & PAGE_MASK); pmap_pte_release(pte); +#ifdef XEN + if (PMAP2_inuse) { + PT_CLEAR_VA(PMAP2, TRUE); + PMAP2_inuse = 0; + } +#endif } PMAP_UNLOCK(pmap); return (rtval); @@ -1006,7 +1101,7 @@ m = NULL; vm_page_lock_queues(); PMAP_LOCK(pmap); - pde = *pmap_pde(pmap, va); + pde = PT_GET(pmap_pde(pmap, va)); if (pde != 0) { if (pde & PG_PS) { if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { @@ -1016,7 +1111,11 @@ } } else { sched_pin(); - pte = *pmap_pte_quick(pmap, va); + pte = PT_GET(pmap_pte_quick(pmap, va)); + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } if (pte != 0 && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { m = PHYS_TO_VM_PAGE(pte & PG_FRAME); @@ -1048,6 +1147,16 @@ } PMAP_INLINE void +pmap_kenter_ma(vm_offset_t va, vm_paddr_t ma) +{ + pt_entry_t *pte; + + pte = vtopte(va); + PT_SET_VA_MA(pte, ma | PG_RW | PG_V | pgeflag, TRUE); +} + + +PMAP_INLINE void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) { pt_entry_t *pte; @@ -1182,7 +1291,8 @@ /* * unmap the page table page */ - pmap->pm_pdir[m->pindex] = 0; + xen_pt_unpin(pmap->pm_pdir[m->pindex]); + PD_CLEAR_VA(pmap, &pmap->pm_pdir[m->pindex], TRUE); --pmap->pm_stats.resident_count; /* @@ -1221,7 +1331,7 @@ if (va >= VM_MAXUSER_ADDRESS) return 0; - ptepde = *pmap_pde(pmap, va); + ptepde = PT_GET(pmap_pde(pmap, va)); mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); return pmap_unwire_pte_hold(pmap, mpte, free); } @@ -1251,8 +1361,9 @@ int pmap_pinit(pmap_t pmap) { - vm_page_t m, ptdpg[NPGPTD]; - vm_paddr_t pa; + vm_page_t m, ptdpg[NPGPTD*2]; + vm_paddr_t ma, ma_shadow; + int npgptd = NPGPTD*2; static int color; int i; @@ -1295,30 +1406,53 @@ } pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD); - +#ifndef XEN for (i = 0; i < NPGPTD; i++) { if ((ptdpg[i]->flags & PG_ZERO) == 0) bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE); } - +#endif mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); /* Wire in kernel global address entries. */ /* XXX copies current process, does not fill in MPPTDI */ - bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t)); + bcopy(kernel_pmap->pm_pdir + KPTDI, pmap->pm_pdir + KPTDI, + nkpt * sizeof(pd_entry_t)); + /* XXX need to copy global address entries to page directory's L1 shadow */ + ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD])); + /* L1 pin shadow page director{y,ies} */ + for (i = 0; i < NPGPTD; i++) { + ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD + i])); + pmap_copy_ma_range(kernel_pmap->pm_pdir[PTDPTDI + i] & ~(PG_RW|PG_M), ma, + KPTDI*sizeof(pd_entry_t), nkpt*sizeof(pd_entry_t)); + xen_pt_pin(ma); + } + #ifdef SMP - pmap->pm_pdir[MPPTDI] = PTD[MPPTDI]; + pmap->pm_pdir[MPPTDI] = kernel_pmap->pm_pdir[MPPTDI]; #endif - /* install self-referential address mapping entry(s) */ + /* pin and install L1 shadow */ for (i = 0; i < NPGPTD; i++) { - pa = VM_PAGE_TO_PHYS(ptdpg[i]); - pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M; + ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i])); + ma_shadow = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD+i])); + /* re-map page directory read-only and pin */ + PT_SET_MA(pmap->pm_pdir + i*PAGE_SIZE, + (ma | PG_V | PG_A)); + xen_pgd_pin(ma); + /* add L1 shadow of L2 */ + xen_queue_pt_update(vtomach(&pmap->pm_pdir[PTDPTDI + i]), + ma_shadow | PG_V | PG_A); + xen_queue_pt_update(ma_shadow + PTDPTDI*sizeof(vm_paddr_t), + vtomach(pmap->pm_pdir) | PG_V | PG_A); + #ifdef PAE - pmap->pm_pdpt[i] = pa | PG_V; + #error "unsupported currently" + pmap->pm_pdpt[i] = ma | PG_V; #endif } + xen_flush_queue(); pmap->pm_active = 0; TAILQ_INIT(&pmap->pm_pvchunk); @@ -1371,8 +1505,9 @@ pmap->pm_stats.resident_count++; ptepa = VM_PAGE_TO_PHYS(m); - pmap->pm_pdir[ptepindex] = - (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M); + xen_pt_pin(xpmap_ptom(ptepa)); + PD_SET_VA(pmap, &pmap->pm_pdir[ptepindex], + (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M), TRUE); return m; } @@ -1396,7 +1531,7 @@ /* * Get the page directory entry */ - ptepa = pmap->pm_pdir[ptepindex]; + ptepa = PT_GET(&pmap->pm_pdir[ptepindex]); /* * This supports switching from a 4MB page to a @@ -1534,8 +1669,9 @@ void pmap_release(pmap_t pmap) { - vm_page_t m, ptdpg[NPGPTD]; - int i; + vm_page_t m, ptdpg[NPGPTD+1]; + vm_paddr_t ma; + int i, npgptd = NPGPTD + 1; KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", @@ -1547,19 +1683,26 @@ mtx_unlock_spin(&allpmaps_lock); for (i = 0; i < NPGPTD; i++) - ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i] & - PG_FRAME); + ptdpg[i] = PHYS_TO_VM_PAGE(PT_GET(&pmap->pm_pdir[PTDPTDI + i]) & PG_FRAME); + + ptdpg[NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir)); + for (i = 0; i < nkpt + NPGPTD; i++) + PD_CLEAR_VA(pmap, &pmap->pm_pdir[PTDPTDI + i], FALSE); + bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) * sizeof(*pmap->pm_pdir)); #ifdef SMP - pmap->pm_pdir[MPPTDI] = 0; + PD_CLEAR_VA(pmap, &pmap->pm_pdir[MPPTDI], FALSE); #endif pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD); - for (i = 0; i < NPGPTD; i++) { + for (i = 0; i < npgptd; i++) { m = ptdpg[i]; + ma = xpmap_ptom(VM_PAGE_TO_PHYS(m)); + /* unpinning L1 and L2 treated the same */ + xen_pgd_unpin(ma); #ifdef PAE KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME), ("pmap_release: got wrong ptd page")); @@ -1642,13 +1785,14 @@ pmap_zero_page(nkpg); ptppaddr = VM_PAGE_TO_PHYS(nkpg); newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M); - pdir_pde(PTD, kernel_vm_end) = newpdir; + PD_SET_VA(kernel_pmap, &pdir_pde(kernel_pmap->pm_pdir, kernel_vm_end), newpdir, TRUE); mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) { pde = pmap_pde(pmap, kernel_vm_end); - pde_store(pde, newpdir); + PD_SET_VA(pmap, pde, newpdir, FALSE); } + PT_UPDATES_FLUSH(); mtx_unlock_spin(&allpmaps_lock); kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); if (kernel_vm_end - 1 >= kernel_map->max_offset) { @@ -2024,6 +2168,10 @@ return; pmap_remove_pte(pmap, pte, va, free); pmap_invalidate_page(pmap, va); + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } } /* @@ -2088,7 +2236,7 @@ * Check for large page. */ if ((ptpaddr & PG_PS) != 0) { - pmap->pm_pdir[pdirindex] = 0; + PD_CLEAR_VA(pmap, &pmap->pm_pdir[pdirindex], TRUE); pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; anyvalid = 1; continue; @@ -2117,6 +2265,10 @@ break; } } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } out: sched_unpin(); if (anyvalid) @@ -2188,6 +2340,10 @@ PMAP_UNLOCK(pmap); } vm_page_flag_clear(m, PG_WRITEABLE); + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); } @@ -2259,7 +2415,9 @@ sva += PAGE_SIZE) { vm_page_t m; +#ifndef XEN retry: +#endif /* * Regardless of whether a pte is 32 or 64 bits in * size, PG_RW, PG_A, and PG_M are among the least @@ -2290,6 +2448,7 @@ #endif if (pbits != obits) { +#ifndef XEN #ifdef PAE if (!atomic_cmpset_64(pte, obits, pbits)) goto retry; @@ -2298,6 +2457,9 @@ pbits)) goto retry; #endif +#else + PT_SET_VA(pte, pbits, FALSE); +#endif if (obits & PG_G) pmap_invalidate_page(pmap, sva); else @@ -2305,6 +2467,10 @@ } } } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); if (anychanged) pmap_invalidate_all(pmap); @@ -2360,7 +2526,7 @@ #if 0 && defined(PMAP_DIAGNOSTIC) else { pd_entry_t *pdeaddr = pmap_pde(pmap, va); - origpte = *pdeaddr; + origpte = PT_GET(pdeaddr); if ((origpte & PG_V) == 0) { panic("pmap_enter: invalid kernel page table page, pdir=%p, pde=%p, va=%p\n", pmap->pm_pdir[PTDPTDI], origpte, va); @@ -2369,7 +2535,7 @@ #endif pde = pmap_pde(pmap, va); - if ((*pde & PG_PS) != 0) + if ((PT_GET(pde) & PG_PS) != 0) panic("pmap_enter: attempted pmap_enter on 4MB page"); pte = pmap_pte_quick(pmap, va); @@ -2383,7 +2549,7 @@ pa = VM_PAGE_TO_PHYS(m); om = NULL; - origpte = *pte; + origpte = PT_GET(pte); opa = origpte & PG_FRAME; /* @@ -2506,6 +2672,10 @@ } else pte_store(pte, newpte | PG_A); } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); vm_page_unlock_queues(); PMAP_UNLOCK(pmap); @@ -2622,7 +2792,7 @@ * But that isn't as quick as vtopte. */ pte = vtopte(va); - if (*pte) { + if (PT_GET(pte)) { if (mpte != NULL) { mpte->wire_count--; mpte = NULL; @@ -2744,8 +2914,8 @@ pmap->pm_stats.resident_count += size >> PAGE_SHIFT; npdes = size >> PDRSHIFT; for(i = 0; i < npdes; i++) { - pde_store(&pmap->pm_pdir[ptepindex], - ptepa | PG_U | PG_RW | PG_V | PG_PS); + PD_SET_VA(pmap, &pmap->pm_pdir[ptepindex], + ptepa | PG_U | PG_RW | PG_V | PG_PS, FALSE); ptepa += NBPDR; ptepindex += 1; } @@ -2781,6 +2951,10 @@ */ pmap_pte_set_w(pte, wired); pmap_pte_release(pte); + if (PMAP2_inuse) { + PT_CLEAR_VA(PMAP2, TRUE); + PMAP2_inuse = 0; + } PMAP_UNLOCK(pmap); } @@ -2830,14 +3004,13 @@ pdnxt = (addr + NBPDR) & ~PDRMASK; ptepindex = addr >> PDRSHIFT; - srcptepaddr = src_pmap->pm_pdir[ptepindex]; + srcptepaddr = PT_GET(&src_pmap->pm_pdir[ptepindex]); if (srcptepaddr == 0) continue; if (srcptepaddr & PG_PS) { if (dst_pmap->pm_pdir[ptepindex] == 0) { - dst_pmap->pm_pdir[ptepindex] = srcptepaddr & - ~PG_W; + PD_SET_VA(dst_pmap, &dst_pmap->pm_pdir[ptepindex], srcptepaddr & ~PG_W, TRUE); dst_pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; } @@ -2872,8 +3045,8 @@ * accessed (referenced) bits * during the copy. */ - *dst_pte = ptetemp & ~(PG_W | PG_M | - PG_A); + PT_SET_VA(dst_pte, ptetemp & ~(PG_W | PG_M | PG_A), FALSE); + dst_pmap->pm_stats.resident_count++; } else { free = NULL; @@ -2891,8 +3064,13 @@ src_pte++; } } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, FALSE); + PMAP1_inuse = 0; + } sched_unpin(); vm_page_unlock_queues(); + PT_UPDATES_FLUSH(); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } @@ -2928,9 +3106,11 @@ panic("pmap_zero_page: CMAP2 busy"); sched_pin(); *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; + PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE); + invlcaddr(sysmaps->CADDR2); pagezero(sysmaps->CADDR2); - *sysmaps->CMAP2 = 0; + PT_CLEAR_VA(sysmaps->CMAP2, TRUE); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -2951,13 +3131,15 @@ if (*sysmaps->CMAP2) panic("pmap_zero_page: CMAP2 busy"); sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; + PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE); + invlcaddr(sysmaps->CADDR2); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else bzero((char *)sysmaps->CADDR2 + off, size); - *sysmaps->CMAP2 = 0; + + PT_CLEAR_VA(sysmaps->CMAP2, TRUE); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -2975,10 +3157,10 @@ if (*CMAP3) panic("pmap_zero_page: CMAP3 busy"); sched_pin(); - *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; + PT_SET_VA(CMAP3, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE); invlcaddr(CADDR3); pagezero(CADDR3); - *CMAP3 = 0; + PT_CLEAR_VA(CMAP3, TRUE); sched_unpin(); } @@ -3002,15 +3184,45 @@ sched_pin(); invlpg((u_int)sysmaps->CADDR1); invlpg((u_int)sysmaps->CADDR2); - *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A; - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M; + PT_SET_VA(sysmaps->CMAP1, PG_V | VM_PAGE_TO_PHYS(src) | PG_A, FALSE); + PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M, TRUE); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); - *sysmaps->CMAP1 = 0; - *sysmaps->CMAP2 = 0; + PT_CLEAR_VA(sysmaps->CMAP1, FALSE); + PT_CLEAR_VA(sysmaps->CMAP2, TRUE); + sched_unpin(); + mtx_unlock(&sysmaps->lock); +} + + + +void +pmap_copy_ma_range(vm_paddr_t src, vm_paddr_t dst, uint32_t soff, uint32_t len) +{ + struct sysmaps *sysmaps; + + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + + mtx_lock(&sysmaps->lock); + if (*sysmaps->CMAP1) + panic("pmap_copy_ma: CMAP1 busy"); + if (*sysmaps->CMAP2) + panic("pmap_copy_ma: CMAP2 busy"); + sched_pin(); + + invlpg((u_int)sysmaps->CADDR1); + invlpg((u_int)sysmaps->CADDR2); + + PT_SET_VA_MA(sysmaps->CMAP1, PG_V | src | PG_A, FALSE); + PT_SET_VA_MA(sysmaps->CMAP2, PG_V | PG_RW | dst | PG_A | PG_M, TRUE); + + bcopy(sysmaps->CADDR1 + soff, sysmaps->CADDR2 + soff, len); + PT_CLEAR_VA(sysmaps->CMAP1, FALSE); + PT_CLEAR_VA(sysmaps->CMAP2, TRUE); sched_unpin(); mtx_unlock(&sysmaps->lock); } + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may @@ -3078,7 +3290,7 @@ inuse &= ~bitmask; pte = vtopte(pv->pv_va); - tpte = *pte; + tpte = PT_GET(pte); if (tpte == 0) { printf( @@ -3140,6 +3352,10 @@ pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); } } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); pmap_invalidate_all(pmap); vm_page_unlock_queues(); @@ -3176,6 +3392,10 @@ if (rv) break; } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); return (rv); } @@ -3193,6 +3413,13 @@ boolean_t rv; rv = FALSE; + /* XXX + * in order for writable pagetables to help, + * this has to work - check if we aren't doing + * an invlpg on the page tables linear mappings + */ + return (rv); + PMAP_LOCK(pmap); if (*pmap_pde(pmap, addr)) { pte = vtopte(addr); @@ -3202,6 +3429,43 @@ return (rv); } + +void +pmap_map_readonly(pmap_t pmap, vm_offset_t va, int len) +{ + int i, npages = round_page(len) >> PAGE_SHIFT; + for (i = 0; i < npages; i++) { + pt_entry_t *pte; + pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE)); + pte_store(pte, xpmap_mtop(*pte & ~(PG_RW|PG_M))); + PMAP_MARK_PRIV(xpmap_mtop(*pte)); + pmap_pte_release(pte); + } + PT_UPDATES_FLUSH(); + if (PMAP2_inuse) { + PT_CLEAR_VA(PMAP2, TRUE); + PMAP2_inuse = 0; + } +} + +void +pmap_map_readwrite(pmap_t pmap, vm_offset_t va, int len) +{ + int i, npages = round_page(len) >> PAGE_SHIFT; + for (i = 0; i < npages; i++) { + pt_entry_t *pte; + pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE)); + PMAP_MARK_UNPRIV(xpmap_mtop(*pte)); + pte_store(pte, xpmap_mtop(*pte) | (PG_RW|PG_M)); + pmap_pte_release(pte); + } + PT_UPDATES_FLUSH(); + if (PMAP2_inuse) { + PT_CLEAR_VA(PMAP2, TRUE); + PMAP2_inuse = 0; + } +} + /* * Clear the write and modified bits in each of the given page's mappings. */ @@ -3221,17 +3485,23 @@ pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); +#ifndef XEN retry: - oldpte = *pte; +#endif + oldpte = PT_GET(pte); if ((oldpte & PG_RW) != 0) { /* * Regardless of whether a pte is 32 or 64 bits * in size, PG_RW and PG_M are among the least * significant 32 bits. */ +#ifndef XEN if (!atomic_cmpset_int((u_int *)pte, oldpte, oldpte & ~(PG_RW | PG_M))) goto retry; +#else + PT_SET_VA(pte, oldpte & ~(PG_RW|PG_M), TRUE); +#endif if ((oldpte & PG_M) != 0) vm_page_dirty(m); pmap_invalidate_page(pmap, pv->pv_va); @@ -3239,6 +3509,10 @@ PMAP_UNLOCK(pmap); } vm_page_flag_clear(m, PG_WRITEABLE); + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); } @@ -3260,6 +3534,8 @@ pv_entry_t pv, pvf, pvn; pmap_t pmap; pt_entry_t *pte; + vm_paddr_t v; + int rtval = 0; if (m->flags & PG_FICTITIOUS) @@ -3275,8 +3551,13 @@ pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); - if ((*pte & PG_A) != 0) { + if (pte && ((v = PT_GET(pte)) & PG_A) != 0) { +#ifndef XEN atomic_clear_int((u_int *)pte, PG_A); +#else + PT_SET_VA(pte, v & ~PG_A, TRUE); +#endif + pmap_invalidate_page(pmap, pv->pv_va); rtval++; if (rtval > 4) @@ -3285,6 +3566,10 @@ PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } + if (PMAP1_inuse) { + PT_CLEAR_VA(PMAP1, TRUE); + PMAP1_inuse = 0; + } sched_unpin(); return (rtval); } @@ -3298,7 +3583,8 @@ pv_entry_t pv; pmap_t pmap; pt_entry_t *pte; - + vm_paddr_t val; + mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & PG_FICTITIOUS) != 0) return; @@ -3307,14 +3593,20 @@ pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); - if ((*pte & PG_M) != 0) { + val = PT_GET(pte); + + if ((val & PG_M) != 0) { /* * Regardless of whether a pte is 32 or 64 bits * in size, PG_M is among the least significant * 32 bits. */ +#ifndef XEN atomic_clear_int((u_int *)pte, PG_M); +#else + PT_SET_VA(pv->pv_va, val & ~PG_M, FALSE); pmap_invalidate_page(pmap, pv->pv_va); +#endif } PMAP_UNLOCK(pmap); } @@ -3332,7 +3624,8 @@ pv_entry_t pv; pmap_t pmap; pt_entry_t *pte; - + vm_paddr_t val; + mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & PG_FICTITIOUS) != 0) return; @@ -3341,13 +3634,18 @@ pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); - if ((*pte & PG_A) != 0) { + val = PT_GET(pte); + if ((val & PG_A) != 0) { /* * Regardless of whether a pte is 32 or 64 bits * in size, PG_A is among the least significant * 32 bits. */ >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200711130754.lAD7sFG4049115>