Date: Thu, 17 Apr 2014 10:53:11 +0000 (UTC) From: "Cherry G. Mathew" <cherry@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r264590 - projects/amd64_xen_pv/sys/amd64/xen Message-ID: <201404171053.s3HArBiT089061@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cherry Date: Thu Apr 17 10:53:10 2014 New Revision: 264590 URL: http://svnweb.freebsd.org/changeset/base/264590 Log: This commit brings us closer to native. Desegregate pmap_pv.c and pmap.c Bring in backing table functions from native and reduce dependency on mmu_map.[ch] Boot to single usermode breaks with this commit, however, this is a good checkpoint to commit. Next stop - direct map using 4K pages. Approved by: gibbs (implicit) Modified: projects/amd64_xen_pv/sys/amd64/xen/pmap.c projects/amd64_xen_pv/sys/amd64/xen/pmap_pv.c Modified: projects/amd64_xen_pv/sys/amd64/xen/pmap.c ============================================================================== --- projects/amd64_xen_pv/sys/amd64/xen/pmap.c Thu Apr 17 10:43:38 2014 (r264589) +++ projects/amd64_xen_pv/sys/amd64/xen/pmap.c Thu Apr 17 10:53:10 2014 (r264590) @@ -83,6 +83,9 @@ * SUCH DAMAGE. */ + +#define AMD64_NPT_AWARE + #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); @@ -138,13 +141,16 @@ __FBSDID("$FreeBSD$"); #include <sys/sched.h> #include <vm/vm.h> -#include <vm/vm_extern.h> -#include <vm/vm_object.h> -#include <vm/vm_page.h> #include <vm/vm_param.h> -#include <vm/vm_map.h> #include <vm/vm_kern.h> -#include <vm/pmap.h> +#include <vm/vm_page.h> +#include <vm/vm_map.h> +#include <vm/vm_object.h> +#include <vm/vm_extern.h> +#include <vm/vm_pageout.h> +#include <vm/vm_pager.h> +#include <vm/vm_radix.h> +#include <vm/vm_reserv.h> #include <vm/uma.h> #include <machine/md_var.h> @@ -156,6 +162,162 @@ __FBSDID("$FreeBSD$"); #include <amd64/xen/mmu_map.h> #include <amd64/xen/pmap_pv.h> +static __inline boolean_t +pmap_emulate_ad_bits(pmap_t pmap) +{ + + return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0); +} + +static __inline pt_entry_t +pmap_valid_bit(pmap_t pmap) +{ + pt_entry_t mask; + + switch (pmap->pm_type) { + case PT_X86: + mask = X86_PG_V; + break; + case PT_EPT: + if (pmap_emulate_ad_bits(pmap)) + mask = EPT_PG_EMUL_V; + else + mask = EPT_PG_READ; + break; + default: + panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type); + } + + return (mask); +} + +static __inline pt_entry_t +pmap_rw_bit(pmap_t pmap) +{ + pt_entry_t mask; + + switch (pmap->pm_type) { + case PT_X86: + mask = X86_PG_RW; + break; + case PT_EPT: + if (pmap_emulate_ad_bits(pmap)) + mask = EPT_PG_EMUL_RW; + else + mask = EPT_PG_WRITE; + break; + default: + panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type); + } + + return (mask); +} + +static __inline pt_entry_t +pmap_global_bit(pmap_t pmap) +{ + pt_entry_t mask; + + switch (pmap->pm_type) { + case PT_X86: + mask = X86_PG_G; + break; + case PT_EPT: + mask = 0; + break; + default: + panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type); + } + + return (mask); +} + +static __inline pt_entry_t +pmap_accessed_bit(pmap_t pmap) +{ + pt_entry_t mask; + + switch (pmap->pm_type) { + case PT_X86: + mask = X86_PG_A; + break; + case PT_EPT: + if (pmap_emulate_ad_bits(pmap)) + mask = EPT_PG_READ; + else + mask = EPT_PG_A; + break; + default: + panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type); + } + + return (mask); +} + +static __inline pt_entry_t +pmap_modified_bit(pmap_t pmap) +{ + pt_entry_t mask; + + switch (pmap->pm_type) { + case PT_X86: + mask = X86_PG_M; + break; + case PT_EPT: + if (pmap_emulate_ad_bits(pmap)) + mask = EPT_PG_WRITE; + else + mask = EPT_PG_M; + break; + default: + panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type); + } + + return (mask); +} + +#ifdef PV_STATS +#define PV_STAT(x) do { x ; } while (0) +#else +#define PV_STAT(x) do { } while (0) +#endif + +#define pa_index(pa) ((pa) >> PDRSHIFT) +#define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) + +#define NPV_LIST_LOCKS MAXCPU + +#define PHYS_TO_PV_LIST_LOCK(pa) \ + (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) + +#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ + struct rwlock **_lockp = (lockp); \ + struct rwlock *_new_lock; \ + \ + _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ + if (_new_lock != *_lockp) { \ + if (*_lockp != NULL) \ + rw_wunlock(*_lockp); \ + *_lockp = _new_lock; \ + rw_wlock(*_lockp); \ + } \ +} while (0) + +#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ + CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) + +#define RELEASE_PV_LIST_LOCK(lockp) do { \ + struct rwlock **_lockp = (lockp); \ + \ + if (*_lockp != NULL) { \ + rw_wunlock(*_lockp); \ + *_lockp = NULL; \ + } \ +} while (0) + +#define VM_PAGE_TO_PV_LIST_LOCK(m) \ + PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) + extern vm_offset_t pa_index; /* from machdep.c */ extern unsigned long physfree; /* from machdep.c */ @@ -201,6 +363,23 @@ static u_int64_t DMPTphys; /* phys addr static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ #endif /* DMAPSUPPORT || DMAP4KSUPPORT */ +static struct rwlock_padalign pvh_global_lock; + +/* + * Data for the pv entry allocation mechanism + */ +TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); +struct mtx pv_chunks_mutex; +struct rwlock pv_list_locks[NPV_LIST_LOCKS]; +static struct md_page *pv_table; + +static int pmap_flags = 0; // XXX: PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */ + +static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); +static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); +static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); +static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, + struct spglist *free); static vm_paddr_t boot_ptphys; /* phys addr of start of * kernel bootstrap tables @@ -222,6 +401,133 @@ extern struct xenstore_domain_interface extern vm_map_t pv_map; +/********************/ +/* Inline functions */ +/********************/ + +/* XXX: */ + +#define MACH_TO_DMAP(_m) PHYS_TO_DMAP(xpmap_mtop(_m)) + +/* Return a non-clipped PD index for a given VA */ +static __inline vm_pindex_t +pmap_pde_pindex(vm_offset_t va) +{ + return (va >> PDRSHIFT); +} + + +/* Return various clipped indexes for a given VA */ +static __inline vm_pindex_t +pmap_pte_index(vm_offset_t va) +{ + + return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1)); +} + +static __inline vm_pindex_t +pmap_pde_index(vm_offset_t va) +{ + + return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1)); +} + +static __inline vm_pindex_t +pmap_pdpe_index(vm_offset_t va) +{ + + return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1)); +} + +static __inline vm_pindex_t +pmap_pml4e_index(vm_offset_t va) +{ + + return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1)); +} + +/* Return a pointer to the PML4 slot that corresponds to a VA */ +static __inline pml4_entry_t * +pmap_pml4e(pmap_t pmap, vm_offset_t va) +{ + + return (&pmap->pm_pml4[pmap_pml4e_index(va)]); +} + +/* Return a pointer to the PDP slot that corresponds to a VA */ +static __inline pdp_entry_t * +pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va) +{ + pdp_entry_t *pdpe; + + pdpe = (pdp_entry_t *)MACH_TO_DMAP(*pml4e & PG_FRAME); + return (&pdpe[pmap_pdpe_index(va)]); +} + +/* Return a pointer to the PDP slot that corresponds to a VA */ +static __inline pdp_entry_t * +pmap_pdpe(pmap_t pmap, vm_offset_t va) +{ + pml4_entry_t *pml4e; + pt_entry_t PG_V; + + PG_V = pmap_valid_bit(pmap); + pml4e = pmap_pml4e(pmap, va); + if ((*pml4e & PG_V) == 0) + return (NULL); + return (pmap_pml4e_to_pdpe(pml4e, va)); +} + +/* Return a pointer to the PD slot that corresponds to a VA */ +static __inline pd_entry_t * +pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va) +{ + pd_entry_t *pde; + + pde = (pd_entry_t *)MACH_TO_DMAP(*pdpe & PG_FRAME); + return (&pde[pmap_pde_index(va)]); +} + +/* Return a pointer to the PD slot that corresponds to a VA */ +static __inline pd_entry_t * +pmap_pde(pmap_t pmap, vm_offset_t va) +{ + pdp_entry_t *pdpe; + pt_entry_t PG_V; + + PG_V = pmap_valid_bit(pmap); + pdpe = pmap_pdpe(pmap, va); + if (pdpe == NULL || (*pdpe & PG_V) == 0) + return (NULL); + return (pmap_pdpe_to_pde(pdpe, va)); +} + +/* Return a pointer to the PT slot that corresponds to a VA */ +static __inline pt_entry_t * +pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) +{ + pt_entry_t *pte; + + pte = (pt_entry_t *)MACH_TO_DMAP(*pde & PG_FRAME); + return (&pte[pmap_pte_index(va)]); +} + +/* Return a pointer to the PT slot that corresponds to a VA */ +static __inline pt_entry_t * +pmap_pte(pmap_t pmap, vm_offset_t va) +{ + pd_entry_t *pde; + pt_entry_t PG_V; + + PG_V = pmap_valid_bit(pmap); + pde = pmap_pde(pmap, va); + if (pde == NULL || (*pde & PG_V) == 0) + return (NULL); + if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */ + return ((pt_entry_t *)pde); + return (pmap_pde_to_pte(pde, va)); +} + /* Index offset into a pagetable, for a given va */ static int pt_index(uintptr_t va) @@ -229,10 +535,119 @@ pt_index(uintptr_t va) return ((va & PDRMASK) >> PAGE_SHIFT); } +static __inline void +pmap_resident_count_inc(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count += count; +} + +static __inline void +pmap_resident_count_dec(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count -= count; +} + +/* XXX: Remove after merge */ +static pt_entry_t * +pmap_vtopte_inspect(pmap_t pmap, uintptr_t va, void *addr) +{ + KASSERT(addr != NULL, ("addr == NULL")); + + mmu_map_t tptr = *(mmu_map_t *)addr; + + pd_entry_t *pte; /* PTE address to return */ + + struct mmu_map_mbackend mb = { + ptmb_mappedalloc, + ptmb_mappedfree, + ptmb_ptov, + ptmb_vtop + }; + + mmu_map_t_init(tptr, &mb); + + if (!mmu_map_inspect_va(pmap, tptr, va)) { + return NULL; /* XXX: fix api, return some kind of #define */ + } + + pte = mmu_map_pt(tptr); /* Read out PT from mmu state */ + + /* add VA offset */ + pte += pt_index(va); + + return pte; +} + +static pt_entry_t * +vtopte_inspect(uintptr_t va, void *addr) +{ + return pmap_vtopte_inspect(kernel_pmap, va, addr); +} + +static pt_entry_t * +pmap_vtopte_hold(pmap_t pmap, uintptr_t va, void *addr) +{ + KASSERT(addr != NULL, ("addr == NULL")); + + mmu_map_t tptr = *(mmu_map_t *)addr; + + pd_entry_t *pte; /* PTE address to return */ + + struct mmu_map_mbackend mb = { + ptmb_mappedalloc, + ptmb_mappedfree, + ptmb_ptov, + ptmb_vtop + }; + + mmu_map_t_init(tptr, &mb); + + if (!mmu_map_inspect_va(pmap, tptr, va)) { + mmu_map_hold_va(pmap, tptr, va); /* PT hierarchy */ + xen_flush_queue(); + } + + pte = mmu_map_pt(tptr); /* Read out PT from mmu state */ + + /* add VA offset */ + pte += pt_index(va); + + return pte; +} + +pt_entry_t * +vtopte_hold(uintptr_t va, void *addr) +{ + return pmap_vtopte_hold(kernel_pmap, va, addr); +} + +static void +pmap_vtopte_release(pmap_t pmap, uintptr_t va, void *addr) +{ + mmu_map_t tptr = *(mmu_map_t *)addr; + + mmu_map_release_va(pmap, tptr, va); + mmu_map_t_fini(tptr); + +} + +void +vtopte_release(uintptr_t va, void *addr) +{ + pmap_vtopte_release(kernel_pmap, va, addr); +} + +/* -> XXX: */ + + /* return kernel virtual address of 'n' claimed physical pages at boot. */ static uintptr_t -vallocpages(vm_paddr_t *firstaddr, int n) +allocpages(vm_paddr_t *firstaddr, int n) { uintptr_t ret = *firstaddr + KERNBASE; bzero((void *)ret, n * PAGE_SIZE); @@ -258,6 +673,10 @@ static void pmap_xen_setpages_ro(uintptr_t va, vm_size_t npages) { vm_size_t i; + pt_entry_t PG_V; + + PG_V = pmap_valid_bit(kernel_pmap); + for (i = 0; i < npages; i++) { PT_SET_MA(va + PAGE_SIZE * i, phystomach(ptmb_vtop(va + PAGE_SIZE * i)) | PG_U | PG_V); @@ -269,6 +688,12 @@ static void pmap_xen_setpages_rw(uintptr_t va, vm_size_t npages) { vm_size_t i; + + pt_entry_t PG_V, PG_RW; + + PG_V = pmap_valid_bit(kernel_pmap); + PG_RW = pmap_rw_bit(kernel_pmap); + for (i = 0; i < npages; i++) { PT_SET_MA(va + PAGE_SIZE * i, phystomach(ptmb_vtop(va + PAGE_SIZE * i)) | PG_U | PG_V | PG_RW); @@ -281,6 +706,9 @@ extern int end; /* End of kernel binary static pt_entry_t pmap_xen_kernel_vaflags(uintptr_t va) { + pt_entry_t PG_RW; + PG_RW = pmap_rw_bit(kernel_pmap); + if ((va > (uintptr_t) &etext && /* .data, .bss et. al */ (va < (uintptr_t) &end)) || @@ -297,7 +725,7 @@ pmap_xen_kernel_vaflags(uintptr_t va) uintptr_t tmpva; static void -create_boot_pagetables(vm_paddr_t *firstaddr) +create_pagetables(vm_paddr_t *firstaddr) { int i; int nkpdpe; @@ -310,23 +738,23 @@ create_boot_pagetables(vm_paddr_t *first /* Allocate pseudo-physical pages for kernel page tables. */ nkpt = howmany(nkmapped, NPTEPG); nkpdpe = howmany(nkpt, NPDEPG); - KPML4phys = vallocpages(firstaddr, 1); - KPDPphys = vallocpages(firstaddr, NKPML4E); - KPDphys = vallocpages(firstaddr, nkpdpe); - KPTphys = vallocpages(firstaddr, nkpt); + KPML4phys = allocpages(firstaddr, 1); + KPDPphys = allocpages(firstaddr, NKPML4E); + KPDphys = allocpages(firstaddr, nkpdpe); + KPTphys = allocpages(firstaddr, nkpt); #ifdef DMAPSUPPORT int ndm1g; ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT; if (ndmpdp < 4) /* Minimum 4GB of dirmap */ ndmpdp = 4; - DMPDPphys = vallocpages(firstaddr, NDMPML4E); + DMPDPphys = allocpages(firstaddr, NDMPML4E); ndm1g = 0; if ((amd_feature & AMDID_PAGE1GB) != 0) ndm1g = ptoa(Maxmem) >> PDPSHIFT; if (ndm1g < ndmpdp) - DMPDphys = vallocpages(firstaddr, ndmpdp - ndm1g); + DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g); dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; #endif /* DMAPSUPPORT */ #ifdef DMAP4KSUPPORT @@ -335,13 +763,13 @@ create_boot_pagetables(vm_paddr_t *first int ndmpdpe = howmany(ndmpt, NPDEPG); tmpva = ptoa(ndmapped); - DMPDPphys = vallocpages(firstaddr, NDMPML4E); - DMPDphys = vallocpages(firstaddr, ndmpdpe); - DMPTphys = vallocpages(firstaddr, ndmpt); + DMPDPphys = allocpages(firstaddr, NDMPML4E); + DMPDphys = allocpages(firstaddr, ndmpdpe); + DMPTphys = allocpages(firstaddr, ndmpt); for (i = 0;ptoa(i) < ptoa(ndmapped); i++) { ((pt_entry_t *)DMPTphys)[i] = phystomach(i << PAGE_SHIFT); - ((pt_entry_t *)DMPTphys)[i] |= PG_V | PG_U; + ((pt_entry_t *)DMPTphys)[i] |= X86_PG_V | X86_PG_U; } pmap_xen_setpages_ro(DMPTphys, (i - 1) / NPTEPG + 1); @@ -349,7 +777,7 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < ndmpt; i ++) { ((pd_entry_t *)DMPDphys)[i] = phystomach(VTOP(DMPTphys) + (i << PAGE_SHIFT)); - ((pd_entry_t *)DMPDphys)[i] |= PG_V | PG_U; + ((pd_entry_t *)DMPDphys)[i] |= X86_PG_V | X86_PG_U | X86_PG_RW; } @@ -358,7 +786,7 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < ndmpdpe; i++) { ((pdp_entry_t *)DMPDPphys)[i] = phystomach(VTOP(DMPDphys) + (i << PAGE_SHIFT)); - ((pdp_entry_t *)DMPDPphys)[i] |= PG_V | PG_U; + ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_V | X86_PG_U | X86_PG_RW; } pmap_xen_setpages_ro(DMPDPphys, (ndmpdpe - 1) / NPDPEPG + 1); @@ -367,7 +795,7 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < NDMPML4E - 1; i++) { ((pdp_entry_t *)KPML4phys)[DMPML4I + i] = phystomach(VTOP(DMPDPphys) + (i << PAGE_SHIFT)); - ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= PG_V | PG_U; + ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= X86_PG_V | X86_PG_U | X86_PG_RW; } #endif /* DMAP4KSUPPORT */ @@ -380,7 +808,7 @@ create_boot_pagetables(vm_paddr_t *first /* Fill in the underlying page table pages */ for (i = 0; ptoa(i) < ptoa(nkmapped); i++) { ((pt_entry_t *)KPTphys)[i] = phystomach(i << PAGE_SHIFT); - ((pt_entry_t *)KPTphys)[i] |= PG_V | PG_U; + ((pt_entry_t *)KPTphys)[i] |= X86_PG_V | X86_PG_U; ((pt_entry_t *)KPTphys)[i] |= pmap_xen_kernel_vaflags(PTOV(i << PAGE_SHIFT)); } @@ -391,7 +819,7 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < nkpt; i++) { ((pd_entry_t *)KPDphys)[i] = phystomach(VTOP(KPTphys) + (i << PAGE_SHIFT)); - ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_U; + ((pd_entry_t *)KPDphys)[i] |= X86_PG_RW | X86_PG_V | X86_PG_U; } @@ -400,7 +828,7 @@ create_boot_pagetables(vm_paddr_t *first /* This replaces some of the KPTphys entries above */ for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) { ((pd_entry_t *)KPDphys)[i] = phystomach(i << PDRSHIFT); - ((pd_entry_t *)KPDphys)[i] |= PG_U | PG_RW | PG_V | PG_PS | PG_G; + ((pd_entry_t *)KPDphys)[i] |= X86_PG_U | X86_PG_RW | X86_PG_V | X86_PG_PS | X86_PG_G; } #endif @@ -410,7 +838,7 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < nkpdpe; i++) { ((pdp_entry_t *)KPDPphys)[i + KPDPI] = phystomach(VTOP(KPDphys) + (i << PAGE_SHIFT)); - ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U; + ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= X86_PG_RW | X86_PG_V | X86_PG_U; } pmap_xen_setpages_ro(KPDPphys, (nkpdpe - 1) / NPDPEPG + 1); @@ -431,7 +859,7 @@ create_boot_pagetables(vm_paddr_t *first if ((i << PDRSHIFT) > ptoa(Maxmem)) { /* * Since the page is zeroed out at - * vallocpages(), the remaining ptes will be + * allocpages(), the remaining ptes will be * invalid. */ @@ -439,8 +867,8 @@ create_boot_pagetables(vm_paddr_t *first } ((pd_entry_t *)DMPDphys)[j] = (vm_paddr_t)(phystomach(i << PDRSHIFT)); /* Preset PG_M and PG_A because demotion expects it. */ - ((pd_entry_t *)DMPDphys)[j] |= PG_U | PG_V | PG_PS /* | PG_G */ | - PG_M | PG_A; + ((pd_entry_t *)DMPDphys)[j] |= X86_PG_U | X86_PG_V | X86_PG_PS /* | X86_PG_G */ | + X86_PG_M | X86_PG_A; } /* Mark pages R/O */ pmap_xen_setpages_ro(DMPDphys, ndmpdp - ndm1g); @@ -450,7 +878,7 @@ create_boot_pagetables(vm_paddr_t *first if ((i << PDPSHIFT) > ptoa(Maxmem)) { /* * Since the page is zeroed out at - * vallocpages(), the remaining ptes will be + * allocpages(), the remaining ptes will be * invalid. */ @@ -459,13 +887,13 @@ create_boot_pagetables(vm_paddr_t *first ((pdp_entry_t *)DMPDPphys)[i] = (vm_paddr_t)phystomach(i << PDPSHIFT); /* Preset PG_M and PG_A because demotion expects it. */ - ((pdp_entry_t *)DMPDPphys)[i] |= PG_U | PG_V | PG_PS | PG_G | - PG_M | PG_A; + ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_U | X86_PG_V | X86_PG_PS | X86_PG_G | + X86_PG_M | X86_PG_A; } for (j = 0; i < ndmpdp; i++, j++) { ((pdp_entry_t *)DMPDPphys)[i] = phystomach(VTOP(DMPDphys) + (j << PAGE_SHIFT)); - ((pdp_entry_t *)DMPDPphys)[i] |= PG_V | PG_U; + ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_V | X86_PG_U; } pmap_xen_setpages_ro(DMPDPphys, NDMPML4E); @@ -474,17 +902,17 @@ create_boot_pagetables(vm_paddr_t *first for (i = 0; i < NDMPML4E; i++) { ((pdp_entry_t *)KPML4phys)[DMPML4I + i] = phystomach(VTOP(DMPDPphys) + (i << PAGE_SHIFT)); - ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= PG_V | PG_U; + ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= X86_PG_V | X86_PG_U; } #endif /* DMAPSUPPORT */ /* And recursively map PML4 to itself in order to get PTmap */ ((pdp_entry_t *)KPML4phys)[PML4PML4I] = phystomach(VTOP(KPML4phys)); - ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_V | PG_U; + ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= X86_PG_V | X86_PG_U; /* Connect the KVA slot up to the PML4 */ ((pdp_entry_t *)KPML4phys)[KPML4I] = phystomach(VTOP(KPDPphys)); - ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U; + ((pdp_entry_t *)KPML4phys)[KPML4I] |= X86_PG_RW | X86_PG_V | X86_PG_U; pmap_xen_setpages_ro(KPML4phys, 1); @@ -504,6 +932,11 @@ pmap_xen_bootpages(vm_paddr_t *firstaddr uintptr_t va; vm_paddr_t ma; + pt_entry_t PG_V, PG_RW; + + PG_V = pmap_valid_bit(kernel_pmap); + PG_RW = pmap_rw_bit(kernel_pmap); + /* i) Share info */ ma = xen_start_info->shared_info; @@ -523,14 +956,14 @@ pmap_xen_bootpages(vm_paddr_t *firstaddr * bootup). */ - va = vallocpages(firstaddr, 1); + va = allocpages(firstaddr, 1); PT_SET_MA(va, ma | PG_RW | PG_V | PG_U); HYPERVISOR_shared_info = (void *) va; #if 0 /* ii) Userland page table base */ - va = vallocpages(firstaddr, 1); + va = allocpages(firstaddr, 1); bzero((void *)va, PAGE_SIZE); /* @@ -572,7 +1005,7 @@ mmu_alloc(void) KASSERT(physfree != 0, ("physfree must have been set before using mmu_alloc")); - va = vallocpages(&physfree, atop(PAGE_SIZE)); + va = allocpages(&physfree, atop(PAGE_SIZE)); /* * Xen requires the page table hierarchy to be R/O. @@ -593,7 +1026,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) ptmb_ptov = boot_ptov; ptmb_vtop = boot_vtop; - create_boot_pagetables(firstaddr); + create_pagetables(firstaddr); /* Switch to the new kernel tables */ xen_pt_switch(xpmap_ptom(VTOP(KPML4phys))); @@ -623,7 +1056,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) /* * Xen guarantees mapped virtual addresses at boot time upto - * xenstack + 512KB. We want to use these for vallocpages() + * xenstack + 512KB. We want to use these for allocpages() * and therefore don't want to touch these mappings since * they're scarce resources. Move along to the end of * guaranteed mapping. @@ -640,13 +1073,25 @@ pmap_bootstrap(vm_paddr_t *firstaddr) /* * Initialize the kernel pmap (which is statically allocated). */ + PMAP_LOCK_INIT(kernel_pmap); + kernel_pmap->pm_pml4 = (pdp_entry_t *)KPML4phys; - kernel_pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t) KPML4phys); + kernel_pmap->pm_cr3 = xpmap_ptom((vm_offset_t) VTOP(KPML4phys)); kernel_pmap->pm_root.rt_root = 0; + CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ + + CPU_ZERO(&kernel_pmap->pm_save); + TAILQ_INIT(&kernel_pmap->pm_pvchunk); + kernel_pmap->pm_flags = pmap_flags; + + /* + * Initialize the global pv list lock. + */ + rw_init(&pvh_global_lock, "pmap pv global"); + pmap_pv_init(); - pmap_pv_pmap_init(kernel_pmap); tsz = mmu_map_t_size(); @@ -669,66 +1114,24 @@ pmap_page_init(vm_page_t m) pmap_pv_vm_page_init(m); } -/* - * Map in backing memory from kernel_vm_end to addr, - * and update kernel_vm_end. +/* + * Initialize the pmap module. + * Called by vm_init, to initialize any structures that the pmap + * system needs to map virtual memory. */ + void -pmap_growkernel(uintptr_t addr) +pmap_init(void) { - KASSERT(kernel_vm_end < addr, ("trying to shrink kernel VA!")); - - addr = trunc_page(addr); + uintptr_t va; - char tbuf[tsz]; /* Safe to do this on the stack since tsz is - * effectively const. - */ + /* XXX: review the use of gdtset for the purpose below */ - mmu_map_t tptr = tbuf; + /* Get a va for console and map the console mfn into it */ + vm_paddr_t ma = xen_start_info->console.domU.mfn << PAGE_SHIFT; - struct mmu_map_mbackend mb = { - ptmb_mappedalloc, - ptmb_mappedfree, - ptmb_ptov, - ptmb_vtop - }; - - mmu_map_t_init(tptr, &mb); - - for (;addr <= kernel_vm_end;addr += PAGE_SIZE) { - - if (mmu_map_inspect_va(kernel_pmap, tptr, addr)) { - continue; - } - int pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; - vm_page_t m = vm_page_alloc(NULL, 0, pflags); - KASSERT(m != NULL, ("Backing page alloc failed!")); - vm_paddr_t pa = VM_PAGE_TO_PHYS(m); - - pmap_kenter(addr, pa); - } - - mmu_map_t_fini(tptr); -} - -/* - * Initialize the pmap module. - * Called by vm_init, to initialize any structures that the pmap - * system needs to map virtual memory. - */ - -void -pmap_init(void) -{ - uintptr_t va; - - /* XXX: review the use of gdtset for the purpose below */ - - /* Get a va for console and map the console mfn into it */ - vm_paddr_t ma = xen_start_info->console.domU.mfn << PAGE_SHIFT; - - va = kva_alloc(PAGE_SIZE); - KASSERT(va != 0, ("Could not allocate KVA for console page!\n")); + va = kva_alloc(PAGE_SIZE); + KASSERT(va != 0, ("Could not allocate KVA for console page!\n")); pmap_kenter_ma(va, ma); console_page = (void *)va; @@ -750,142 +1153,169 @@ pmap_init(void) } void -pmap_pinit0(pmap_t pmap) +pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - PMAP_LOCK_INIT(pmap); - pmap->pm_pml4 = (void *) KPML4phys; - pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t) KPML4phys); - pmap->pm_root.rt_root = 0; - CPU_ZERO(&pmap->pm_active); - CPU_ZERO(&pmap->pm_save); - PCPU_SET(curpmap, pmap); - pmap_pv_pmap_init(pmap); - bzero(&pmap->pm_stats, sizeof pmap->pm_stats); - pmap->pm_pcid = -1; + pmap_invalidate_range(pmap, va, va + PAGE_SIZE); } -int -pmap_pinit(pmap_t pmap) +void +pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - KASSERT(pmap != kernel_pmap, - ("%s: kernel map re-initialised!", __func__)); + vm_offset_t addr; + /* XXX: TODO SMP */ + sched_pin(); - /* - * allocate the page directory page - */ - pmap->pm_pml4 = (void *) kmem_malloc(kernel_arena, PAGE_SIZE, M_ZERO); - if (pmap->pm_pml4 == NULL) return 0; + for (addr = sva; addr < eva; addr += PAGE_SIZE) + invlpg(addr); - pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t)pmap->pm_pml4); + sched_unpin(); +} - /* - * We do not wire in kernel space, or the self-referencial - * entry in userspace pmaps becase both kernel and userland - * share ring3 privilege. The user/kernel context switch is - * arbitrated by the hypervisor by means of pre-loaded values - * for kernel and user %cr3. The userland parts of kernel VA - * may be conditionally overlaid with the VA of curthread, - * since the kernel occasionally needs to access userland - * process VA space. - */ +void +pmap_invalidate_all(pmap_t pmap) +{ - pmap_xen_setpages_ro((uintptr_t)pmap->pm_pml4, 1); + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) + invltlb(); +} - xen_pgdir_pin(pmap->pm_cr3); +/* + * Routine: pmap_extract + * Function: + * Extract the physical page address associated + * with the given map/virtual_address pair. + */ - pmap->pm_root.rt_root = 0; - CPU_ZERO(&pmap->pm_active); - pmap_pv_pmap_init(pmap); - bzero(&pmap->pm_stats, sizeof pmap->pm_stats); - pmap->pm_pcid = -1; /* No pcid for now */ - CPU_ZERO(&pmap->pm_save); +vm_paddr_t +pmap_extract(pmap_t pmap, vm_offset_t va) +{ - return 1; -} + pt_entry_t PG_V; -void pmap_xen_userload(pmap_t pmap) -{ - KASSERT(pmap != kernel_pmap, - ("Kernel pmap requested on user load.\n")); + PG_V = pmap_valid_bit(pmap); - int i; - for (i = 0; i < NUPML4E; i++) { - pml4_entry_t pml4e; - pml4e = (pmap->pm_pml4[i]); - PT_SET_VA_MA((pml4_entry_t *)KPML4phys + i, pml4e, false); + if (pmap == kernel_pmap) { + return pmap_kextract(va); } - PT_UPDATES_FLUSH(); - invltlb(); - /* Tell xen about user pmap switch */ - xen_pt_user_switch(pmap->pm_cr3); -} + pt_entry_t *pte; + vm_paddr_t ma = 0; -void -pmap_release(pmap_t pmap) -{ - KASSERT(pmap != kernel_pmap, - ("%s: kernel pmap released", __func__)); + + /* Walk the PT hierarchy to get the ma */ + char tbuf[tsz]; /* Safe to do this on the stack since tsz is + * effectively const. + */ - KASSERT(pmap->pm_stats.resident_count == 0, - ("pmap_release: pmap resident count %ld != 0", - pmap->pm_stats.resident_count)); + mmu_map_t tptr = tbuf; - KASSERT(vm_radix_is_empty(&pmap->pm_root), - ("pmap_release: pmap has reserved page table page(s)")); + pte = pmap_vtopte_inspect(pmap, va, &tptr); - xen_pgdir_unpin(pmap->pm_cr3); - pmap_xen_setpages_rw((uintptr_t)pmap->pm_pml4, 1); + if (pte != NULL && (*pte & PG_V)) { + ma = (*pte & PG_FRAME) | (va & PAGE_MASK); + } - bzero(pmap->pm_pml4, PAGE_SIZE); - kmem_free(kernel_arena, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE); + pmap_vtopte_release(pmap, va, &tptr); + + return xpmap_mtop(ma); } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201404171053.s3HArBiT089061>