Date: Mon, 13 Oct 2008 18:59:59 +0000 (UTC) From: Rafal Jaworowski <raj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r183838 - head/sys/arm/arm Message-ID: <200810131859.m9DIxxgw083830@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: raj Date: Mon Oct 13 18:59:59 2008 New Revision: 183838 URL: http://svn.freebsd.org/changeset/base/183838 Log: Provide L2 cache synchronization (write back + invalidation) on ARM. Note the cpu_l2cache_wbinv_* routines are no-ops on systems not populated with L2 caches. Obtained from: Marvell, Semihalf Modified: head/sys/arm/arm/busdma_machdep.c head/sys/arm/arm/elf_machdep.c head/sys/arm/arm/genassym.c head/sys/arm/arm/pmap.c head/sys/arm/arm/swtch.S Modified: head/sys/arm/arm/busdma_machdep.c ============================================================================== --- head/sys/arm/arm/busdma_machdep.c Mon Oct 13 18:44:59 2008 (r183837) +++ head/sys/arm/arm/busdma_machdep.c Mon Oct 13 18:59:59 2008 (r183838) @@ -629,6 +629,8 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi newmap->allocbuffer = tmpaddr; cpu_idcache_wbinv_range((vm_offset_t)*vaddr, dmat->maxsize); + cpu_l2cache_wbinv_range((vm_offset_t)*vaddr, + dmat->maxsize); *vaddr = tmpaddr; } else newmap->origbuffer = newmap->allocbuffer = NULL; Modified: head/sys/arm/arm/elf_machdep.c ============================================================================== --- head/sys/arm/arm/elf_machdep.c Mon Oct 13 18:44:59 2008 (r183837) +++ head/sys/arm/arm/elf_machdep.c Mon Oct 13 18:59:59 2008 (r183838) @@ -215,6 +215,7 @@ elf_cpu_load_file(linker_file_t lf __unu { cpu_idcache_wbinv_all(); + cpu_l2cache_wbinv_all(); cpu_tlb_flushID(); return (0); } Modified: head/sys/arm/arm/genassym.c ============================================================================== --- head/sys/arm/arm/genassym.c Mon Oct 13 18:44:59 2008 (r183837) +++ head/sys/arm/arm/genassym.c Mon Oct 13 18:59:59 2008 (r183838) @@ -79,7 +79,9 @@ ASSYM(CF_SETTTB, offsetof(struct cpu_fun ASSYM(CF_CONTROL, offsetof(struct cpu_functions, cf_control)); ASSYM(CF_CONTEXT_SWITCH, offsetof(struct cpu_functions, cf_context_switch)); ASSYM(CF_DCACHE_WB_RANGE, offsetof(struct cpu_functions, cf_dcache_wb_range)); +ASSYM(CF_L2CACHE_WB_RANGE, offsetof(struct cpu_functions, cf_l2cache_wb_range)); ASSYM(CF_IDCACHE_WBINV_ALL, offsetof(struct cpu_functions, cf_idcache_wbinv_all)); +ASSYM(CF_L2CACHE_WBINV_ALL, offsetof(struct cpu_functions, cf_l2cache_wbinv_all)); ASSYM(CF_TLB_FLUSHID_SE, offsetof(struct cpu_functions, cf_tlb_flushID_SE)); ASSYM(CF_ICACHE_SYNC, offsetof(struct cpu_functions, cf_icache_sync_all)); Modified: head/sys/arm/arm/pmap.c ============================================================================== --- head/sys/arm/arm/pmap.c Mon Oct 13 18:44:59 2008 (r183837) +++ head/sys/arm/arm/pmap.c Mon Oct 13 18:59:59 2008 (r183838) @@ -151,6 +151,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/ktr.h> #include <sys/proc.h> #include <sys/malloc.h> #include <sys/msgbuf.h> @@ -1196,27 +1197,104 @@ pmap_tlb_flushD(pmap_t pm) } static PMAP_INLINE void +pmap_l2cache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) +{ + vm_size_t rest; + pd_entry_t *pde; + pt_entry_t *ptep; + + rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); + + while (len > 0) { + CTR4(KTR_PMAP, "pmap_l2cache_wbinv_range: pmap %p is_kernel %d " + "va 0x%08x len 0x%x ", pm, pm == pmap_kernel(), va, rest); + if (pmap_get_pde_pte(pm, va, &pde, &ptep) && l2pte_valid(*ptep)) + cpu_l2cache_wb_range(va, rest); + + len -= rest; + va += rest; + + rest = MIN(PAGE_SIZE, len); + } +} + +static PMAP_INLINE void pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) { - if (pmap_is_current(pm)) + if (pmap_is_current(pm)) { cpu_idcache_wbinv_range(va, len); + pmap_l2cache_wbinv_range(pm, va, len); + } +} + +static PMAP_INLINE void +pmap_l2cache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len) +{ + vm_size_t rest; + pd_entry_t *pde; + pt_entry_t *ptep; + + rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); + + while (len > 0) { + CTR4(KTR_PMAP, "pmap_l2cache_wb_range: pmap %p is_kernel %d " + "va 0x%08x len 0x%x ", pm, pm == pmap_kernel(), va, rest); + if (pmap_get_pde_pte(pm, va, &pde, &ptep) && l2pte_valid(*ptep)) + cpu_l2cache_wb_range(va, rest); + + len -= rest; + va += rest; + + rest = MIN(PAGE_SIZE, len); + } } static PMAP_INLINE void -pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, - boolean_t do_inv, boolean_t rd_only) +pmap_l2cache_inv_range(pmap_t pm, vm_offset_t va, vm_size_t len) { + vm_size_t rest; + pd_entry_t *pde; + pt_entry_t *ptep; + + rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); + + while (len > 0) { + CTR4(KTR_PMAP, "pmap_l2cache_wb_range: pmap %p is_kernel %d " + "va 0x%08x len 0x%x ", pm, pm == pmap_kernel(), va, rest); + if (pmap_get_pde_pte(pm, va, &pde, &ptep) && l2pte_valid(*ptep)) + cpu_l2cache_inv_range(va, rest); + + len -= rest; + va += rest; + + rest = MIN(PAGE_SIZE, len); + } +} + +static PMAP_INLINE void +pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, boolean_t do_inv, + boolean_t rd_only) +{ + CTR4(KTR_PMAP, "pmap_dcache_wb_range: pmap %p is_kernel %d va 0x%08x " + "len 0x%x ", pm, pm == pmap_kernel(), va, len); + CTR2(KTR_PMAP, " do_inv %d rd_only %d", do_inv, rd_only); if (pmap_is_current(pm)) { if (do_inv) { - if (rd_only) + if (rd_only) { cpu_dcache_inv_range(va, len); - else + pmap_l2cache_inv_range(pm, va, len); + } + else { cpu_dcache_wbinv_range(va, len); + pmap_l2cache_wbinv_range(pm, va, len); + } } else - if (!rd_only) + if (!rd_only) { cpu_dcache_wb_range(va, len); + pmap_l2cache_wb_range(pm, va, len); + } } } @@ -1224,16 +1302,20 @@ static PMAP_INLINE void pmap_idcache_wbinv_all(pmap_t pm) { - if (pmap_is_current(pm)) + if (pmap_is_current(pm)) { cpu_idcache_wbinv_all(); + cpu_l2cache_wbinv_all(); + } } static PMAP_INLINE void pmap_dcache_wbinv_all(pmap_t pm) { - if (pmap_is_current(pm)) + if (pmap_is_current(pm)) { cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); + } } /* @@ -2169,6 +2251,8 @@ pmap_set_pt_cache_mode(pd_entry_t *kl1, PTE_SYNC(pdep); cpu_dcache_wbinv_range((vm_offset_t)pdep, sizeof(*pdep)); + cpu_l2cache_wbinv_range((vm_offset_t)pdep, + sizeof(*pdep)); rv = 1; } } else { @@ -2185,6 +2269,8 @@ pmap_set_pt_cache_mode(pd_entry_t *kl1, PTE_SYNC(ptep); cpu_dcache_wbinv_range((vm_offset_t)ptep, sizeof(*ptep)); + cpu_l2cache_wbinv_range((vm_offset_t)ptep, + sizeof(*ptep)); rv = 1; } } @@ -2337,6 +2423,7 @@ pmap_bootstrap(vm_offset_t firstaddr, vm } cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); cpu_tlb_flushID(); cpu_cpwait(); @@ -2373,6 +2460,7 @@ pmap_bootstrap(vm_offset_t firstaddr, vm mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); pmap_init_l1(l1, kernel_l1pt); cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); virtual_avail = round_page(virtual_avail); virtual_end = lastaddr; @@ -2402,6 +2490,7 @@ pmap_release(pmap_t pmap) struct pcb *pcb; pmap_idcache_wbinv_all(pmap); + cpu_l2cache_wbinv_all(); pmap_tlb_flushID(pmap); cpu_cpwait(); if (vector_page < KERNBASE) { @@ -2589,6 +2678,7 @@ pmap_growkernel(vm_offset_t addr) * rarely */ cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); cpu_tlb_flushD(); cpu_cpwait(); kernel_vm_end = pmap_curmaxkvaddr; @@ -2614,6 +2704,7 @@ pmap_remove_pages(pmap_t pmap) vm_page_lock_queues(); PMAP_LOCK(pmap); cpu_idcache_wbinv_all(); + cpu_l2cache_wbinv_all(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_flags & PVF_WIRED) { /* The page is wired, cannot remove it now. */ @@ -2726,6 +2817,7 @@ pmap_kenter_internal(vm_offset_t va, vm_ (uint32_t) pte, opte, *pte)); if (l2pte_valid(opte)) { cpu_dcache_wbinv_range(va, PAGE_SIZE); + cpu_l2cache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); cpu_cpwait(); } else { @@ -2784,6 +2876,7 @@ pmap_kremove(vm_offset_t va) opte = *pte; if (l2pte_valid(opte)) { cpu_dcache_wbinv_range(va, PAGE_SIZE); + cpu_l2cache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); cpu_cpwait(); *pte = 0; @@ -3052,6 +3145,9 @@ pmap_protect(pmap_t pm, vm_offset_t sva, u_int flags; int flush; + CTR4(KTR_PMAP, "pmap_protect: pmap %p sva 0x%08x eva 0x%08x prot %x", + pm, sva, eva, prot); + if ((prot & VM_PROT_READ) == 0) { pmap_remove(pm, sva, eva); return; @@ -3286,9 +3382,11 @@ do_l2b_alloc: */ if (pmap_is_current(pmap) && (oflags & PVF_NC) == 0 && - (opte & L2_S_PROT_W) != 0 && - (prot & VM_PROT_WRITE) == 0) + (opte & L2_S_PROT_W) != 0 && + (prot & VM_PROT_WRITE) == 0) { cpu_dcache_wb_range(va, PAGE_SIZE); + pmap_l2cache_wb_range(pmap, va, PAGE_SIZE); + } } else { /* * New mapping, or changing the backing page @@ -3777,11 +3875,15 @@ pmap_remove(pmap_t pm, vm_offset_t sva, total++; if (is_exec) { cpu_idcache_wbinv_range(sva, - PAGE_SIZE); + PAGE_SIZE); + cpu_l2cache_wbinv_range(sva, + PAGE_SIZE); cpu_tlb_flushID_SE(sva); } else if (is_refd) { cpu_dcache_wbinv_range(sva, - PAGE_SIZE); + PAGE_SIZE); + cpu_l2cache_wbinv_range(sva, + PAGE_SIZE); cpu_tlb_flushD_SE(sva); } } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) { @@ -3789,6 +3891,7 @@ pmap_remove(pmap_t pm, vm_offset_t sva, * for a current pmap */ cpu_idcache_wbinv_all(); + cpu_l2cache_wbinv_all(); flushall = 1; total++; } @@ -3842,9 +3945,11 @@ pmap_zero_page_generic(vm_paddr_t phys, if (off || size != PAGE_SIZE) { bzero(dstpg + off, size); cpu_dcache_wbinv_range((vm_offset_t)(dstpg + off), size); + cpu_l2cache_wbinv_range((vm_offset_t)(dstpg + off), size); } else { bzero_page((vm_offset_t)dstpg); cpu_dcache_wbinv_range((vm_offset_t)dstpg, PAGE_SIZE); + cpu_l2cache_wbinv_range((vm_offset_t)dstpg, PAGE_SIZE); } #else @@ -4139,6 +4244,8 @@ pmap_copy_page_generic(vm_paddr_t src, v mtx_unlock(&cmtx); cpu_dcache_inv_range(csrcp, PAGE_SIZE); cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); + cpu_l2cache_inv_range(csrcp, PAGE_SIZE); + cpu_l2cache_wbinv_range(cdstp, PAGE_SIZE); } #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ @@ -4201,6 +4308,7 @@ pmap_copy_page(vm_page_t src, vm_page_t #endif cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); if (_arm_memcpy && PAGE_SIZE >= _min_memcpy_size && _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) @@ -4210,6 +4318,7 @@ pmap_copy_page(vm_page_t src, vm_page_t dstpg = arm_ptovirt(VM_PAGE_TO_PHYS(dst)); bcopy_page(srcpg, dstpg); cpu_dcache_wbinv_range(dstpg, PAGE_SIZE); + cpu_l2cache_wbinv_range(dstpg, PAGE_SIZE); #else pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); #endif Modified: head/sys/arm/arm/swtch.S ============================================================================== --- head/sys/arm/arm/swtch.S Mon Oct 13 18:44:59 2008 (r183837) +++ head/sys/arm/arm/swtch.S Mon Oct 13 18:59:59 2008 (r183838) @@ -143,6 +143,8 @@ ENTRY(cpu_throw) ldr r9, .Lcpufuncs mov lr, pc ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] + mov lr, pc + ldr pc, [r9, #CF_L2CACHE_WBINV_ALL] ldr r0, [r7, #(PCB_PL1VEC)] ldr r1, [r7, #(PCB_DACR)] /* @@ -172,6 +174,8 @@ ENTRY(cpu_throw) movne r1, #4 movne lr, pc ldrne pc, [r9, #CF_DCACHE_WB_RANGE] + movne lr, pc + ldrne pc, [r9, #CF_L2CACHE_WB_RANGE] #endif /* PMAP_INCLUDE_PTE_SYNC */ /* @@ -328,6 +332,8 @@ ENTRY(cpu_switch) ldr r1, .Lcpufuncs mov lr, pc ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] + mov lr, pc + ldr pc, [r1, #CF_L2CACHE_WBINV_ALL] .Lcs_cache_purge_skipped: /* rem: r6 = lock */ /* rem: r9 = new PCB */ @@ -360,6 +366,8 @@ ENTRY(cpu_switch) mov r1, #4 mov lr, pc ldr pc, [r2, #CF_DCACHE_WB_RANGE] + mov lr, pc + ldr pc, [r2, #CF_L2CACHE_WB_RANGE] .Lcs_same_vector: #endif /* PMAP_INCLUDE_PTE_SYNC */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200810131859.m9DIxxgw083830>