Date: Sat, 2 Jul 2011 11:14:54 +0000 (UTC) From: Marius Strobl <marius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r223719 - in head/sys: boot/sparc64/loader sparc64/include sparc64/sparc64 Message-ID: <201107021114.p62BEssr037439@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: marius Date: Sat Jul 2 11:14:54 2011 New Revision: 223719 URL: http://svn.freebsd.org/changeset/base/223719 Log: - For Cheetah- and Zeus-class CPUs don't flush all unlocked entries from the TLBs in order to get rid of the user mappings but instead traverse them an flush only the latter like we also do for the Spitfire-class. Also flushing the unlocked kernel entries can cause instant faults which when called from within cpu_switch() are handled with the scheduler lock held which in turn can cause timeouts on the acquisition of the lock by other CPUs. This was easily seen with a 16-core V890 but occasionally also happened with 2-way machines. While at it, move the SPARC64-V support code entirely to zeus.c. This causes a little bit of duplication but is less confusing than partially using Cheetah-class bits for these. - For SPARC64-V ensure that 4-Mbyte page entries are stored in the 1024- entry, 2-way set associative TLB. - In {d,i}tlb_get_data_sun4u() turn off the interrupts in order to ensure that ASI_{D,I}TLB_DATA_ACCESS_REG actually are read twice back-to-back. Tested by: Peter Jeremy (16-core US-IV), Michael Moll (2-way SPARC64-V) Modified: head/sys/boot/sparc64/loader/main.c head/sys/sparc64/include/cache.h head/sys/sparc64/include/cpu.h head/sys/sparc64/include/tlb.h head/sys/sparc64/sparc64/cache.c head/sys/sparc64/sparc64/cheetah.c head/sys/sparc64/sparc64/machdep.c head/sys/sparc64/sparc64/mp_machdep.c head/sys/sparc64/sparc64/pmap.c head/sys/sparc64/sparc64/spitfire.c head/sys/sparc64/sparc64/zeus.c Modified: head/sys/boot/sparc64/loader/main.c ============================================================================== --- head/sys/boot/sparc64/loader/main.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/boot/sparc64/loader/main.c Sat Jul 2 11:14:54 2011 (r223719) @@ -96,10 +96,10 @@ static struct mmu_ops { typedef void kernel_entry_t(vm_offset_t mdp, u_long o1, u_long o2, u_long o3, void *openfirmware); -static inline u_long dtlb_get_data_sun4u(u_int); +static inline u_long dtlb_get_data_sun4u(u_int, u_int); static int dtlb_enter_sun4u(u_int, u_long data, vm_offset_t); static vm_offset_t dtlb_va_to_pa_sun4u(vm_offset_t); -static inline u_long itlb_get_data_sun4u(u_int); +static inline u_long itlb_get_data_sun4u(u_int, u_int); static int itlb_enter_sun4u(u_int, u_long data, vm_offset_t); static vm_offset_t itlb_va_to_pa_sun4u(vm_offset_t); static void itlb_relocate_locked0_sun4u(void); @@ -136,6 +136,7 @@ u_int itlb_slot; static int cpu_impl; static u_int dtlb_slot_max; static u_int itlb_slot_max; +static u_int tlb_locked; static vm_offset_t curkva = 0; static vm_offset_t heapva; @@ -355,42 +356,55 @@ __elfN(exec)(struct preloaded_file *fp) } static inline u_long -dtlb_get_data_sun4u(u_int slot) +dtlb_get_data_sun4u(u_int tlb, u_int slot) { + u_long data, pstate; + slot = TLB_DAR_SLOT(tlb, slot); /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work - * around errata of USIII and beyond. + * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to + * work around errata of USIII and beyond. */ - (void)ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG); - return (ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG)); + pstate = rdpr(pstate); + wrpr(pstate, pstate & ~PSTATE_IE, 0); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + wrpr(pstate, pstate, 0); + return (data); } static inline u_long -itlb_get_data_sun4u(u_int slot) +itlb_get_data_sun4u(u_int tlb, u_int slot) { + u_long data, pstate; + slot = TLB_DAR_SLOT(tlb, slot); /* - * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work - * around errata of USIII and beyond. + * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to + * work around errata of USIII and beyond. */ - (void)ldxa(TLB_DAR_SLOT(slot), ASI_ITLB_DATA_ACCESS_REG); - return (ldxa(TLB_DAR_SLOT(slot), ASI_ITLB_DATA_ACCESS_REG)); + pstate = rdpr(pstate); + wrpr(pstate, pstate & ~PSTATE_IE, 0); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + wrpr(pstate, pstate, 0); + return (data); } static vm_offset_t dtlb_va_to_pa_sun4u(vm_offset_t va) { u_long pstate, reg; - int i; + u_int i, tlb; pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); for (i = 0; i < dtlb_slot_max; i++) { - reg = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); + reg = ldxa(TLB_DAR_SLOT(tlb_locked, i), + ASI_DTLB_TAG_READ_REG); if (TLB_TAR_VA(reg) != va) continue; - reg = dtlb_get_data_sun4u(i); + reg = dtlb_get_data_sun4u(tlb_locked, i); wrpr(pstate, pstate, 0); reg >>= TD_PA_SHIFT; if (cpu_impl == CPU_IMPL_SPARC64V || @@ -411,10 +425,11 @@ itlb_va_to_pa_sun4u(vm_offset_t va) pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); for (i = 0; i < itlb_slot_max; i++) { - reg = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); + reg = ldxa(TLB_DAR_SLOT(tlb_locked, i), + ASI_ITLB_TAG_READ_REG); if (TLB_TAR_VA(reg) != va) continue; - reg = itlb_get_data_sun4u(i); + reg = itlb_get_data_sun4u(tlb_locked, i); wrpr(pstate, pstate, 0); reg >>= TD_PA_SHIFT; if (cpu_impl == CPU_IMPL_SPARC64V || @@ -458,14 +473,14 @@ itlb_relocate_locked0_sun4u(void) pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); - data = itlb_get_data_sun4u(0); + data = itlb_get_data_sun4u(tlb_locked, 0); if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) { wrpr(pstate, pstate, 0); return; } /* Flush the mapping of slot 0. */ - tag = ldxa(TLB_DAR_SLOT(0), ASI_ITLB_TAG_READ_REG); + tag = ldxa(TLB_DAR_SLOT(tlb_locked, 0), ASI_ITLB_TAG_READ_REG); stxa(TLB_DEMAP_VA(TLB_TAR_VA(tag)) | TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, ASI_IMMU_DEMAP, 0); flush(0); /* The USIII-family ignores the address. */ @@ -475,11 +490,12 @@ itlb_relocate_locked0_sun4u(void) * that formerly were in slot 0. */ for (i = 1; i < itlb_slot_max; i++) { - if ((itlb_get_data_sun4u(i) & TD_V) != 0) + if ((itlb_get_data_sun4u(tlb_locked, i) & TD_V) != 0) continue; stxa(AA_IMMU_TAR, ASI_IMMU, tag); - stxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, data); + stxa(TLB_DAR_SLOT(tlb_locked, i), ASI_ITLB_DATA_ACCESS_REG, + data); flush(0); /* The USIII-family ignores the address. */ break; } @@ -651,6 +667,26 @@ tlb_init_sun4u(void) phandle_t bsp; cpu_impl = VER_IMPL(rdpr(ver)); + switch (cpu_impl) { + case CPU_IMPL_SPARC64: + case CPU_IMPL_ULTRASPARCI: + case CPU_IMPL_ULTRASPARCII: + case CPU_IMPL_ULTRASPARCIIi: + case CPU_IMPL_ULTRASPARCIIe: + tlb_locked = TLB_DAR_T32; + break; + case CPU_IMPL_ULTRASPARCIII: + case CPU_IMPL_ULTRASPARCIIIp: + case CPU_IMPL_ULTRASPARCIIIi: + case CPU_IMPL_ULTRASPARCIIIip: + case CPU_IMPL_ULTRASPARCIV: + case CPU_IMPL_ULTRASPARCIVp: + tlb_locked = TLB_DAR_T16; + break; + case CPU_IMPL_SPARC64V: + tlb_locked = TLB_DAR_FTLB; + break; + } bsp = find_bsp_sun4u(OF_child(root), cpu_get_mid_sun4u()); if (bsp == 0) panic("%s: no node for bootcpu?!?!", __func__); @@ -821,21 +857,23 @@ pmap_print_tlb_sun4u(void) pstate = rdpr(pstate); for (i = 0; i < itlb_slot_max; i++) { wrpr(pstate, pstate & ~PSTATE_IE, 0); - tte = itlb_get_data_sun4u(i); + tte = itlb_get_data_sun4u(tlb_locked, i); wrpr(pstate, pstate, 0); if (!(tte & TD_V)) continue; - tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); + tag = ldxa(TLB_DAR_SLOT(tlb_locked, i), + ASI_ITLB_TAG_READ_REG); printf("iTLB-%2u: ", i); pmap_print_tte_sun4u(tag, tte); } for (i = 0; i < dtlb_slot_max; i++) { wrpr(pstate, pstate & ~PSTATE_IE, 0); - tte = dtlb_get_data_sun4u(i); + tte = dtlb_get_data_sun4u(tlb_locked, i); wrpr(pstate, pstate, 0); if (!(tte & TD_V)) continue; - tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); + tag = ldxa(TLB_DAR_SLOT(tlb_locked, i), + ASI_DTLB_TAG_READ_REG); printf("dTLB-%2u: ", i); pmap_print_tte_sun4u(tag, tte); } Modified: head/sys/sparc64/include/cache.h ============================================================================== --- head/sys/sparc64/include/cache.h Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/include/cache.h Sat Jul 2 11:14:54 2011 (r223719) @@ -110,15 +110,16 @@ cache_flush_t spitfire_cache_flush; dcache_page_inval_t spitfire_dcache_page_inval; icache_page_inval_t spitfire_icache_page_inval; +cache_enable_t zeus_cache_enable; +cache_flush_t zeus_cache_flush; +dcache_page_inval_t zeus_dcache_page_inval; +icache_page_inval_t zeus_icache_page_inval; + extern cache_enable_t *cache_enable; extern cache_flush_t *cache_flush; extern dcache_page_inval_t *dcache_page_inval; extern icache_page_inval_t *icache_page_inval; -cache_flush_t zeus_cache_flush; -dcache_page_inval_t zeus_dcache_page_inval; -icache_page_inval_t zeus_icache_page_inval; - #endif /* KERNEL */ #endif /* !LOCORE */ Modified: head/sys/sparc64/include/cpu.h ============================================================================== --- head/sys/sparc64/include/cpu.h Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/include/cpu.h Sat Jul 2 11:14:54 2011 (r223719) @@ -57,6 +57,7 @@ void cpu_halt(void); void cpu_reset(void); void fork_trampoline(void); void swi_vm(void *v); +void zeus_init(u_int cpu_impl); static __inline u_int64_t get_cyclecount(void) Modified: head/sys/sparc64/include/tlb.h ============================================================================== --- head/sys/sparc64/include/tlb.h Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/include/tlb.h Sat Jul 2 11:14:54 2011 (r223719) @@ -44,7 +44,17 @@ (TD_V | TD_4M | (TLB_DIRECT_ADDRESS_MASK - TLB_DIRECT_PAGE_MASK)) #define TLB_DAR_SLOT_SHIFT (3) -#define TLB_DAR_SLOT(slot) ((slot) << TLB_DAR_SLOT_SHIFT) +#define TLB_DAR_TLB_SHIFT (16) +#define TLB_DAR_SLOT(tlb, slot) \ + ((tlb) << TLB_DAR_TLB_SHIFT | (slot) << TLB_DAR_SLOT_SHIFT) +#define TLB_DAR_T16 (0) /* US-III{,i,+}, IV{,+} */ +#define TLB_DAR_T32 (0) /* US-I, II{,e,i} */ +#define TLB_DAR_DT512_0 (2) /* US-III{,i,+}, IV{,+} */ +#define TLB_DAR_DT512_1 (3) /* US-III{,i,+}, IV{,+} */ +#define TLB_DAR_IT128 (2) /* US-III{,i,+}, IV */ +#define TLB_DAR_IT512 (2) /* US-IV+ */ +#define TLB_DAR_FTLB (0) /* SPARC64 V, VI, VII, VIIIfx */ +#define TLB_DAR_STLB (2) /* SPARC64 V, VI, VII, VIIIfx */ #define TAR_VPN_SHIFT (13) #define TAR_CTX_MASK ((1 << TAR_VPN_SHIFT) - 1) @@ -156,6 +166,9 @@ tlb_flush_user_t cheetah_tlb_flush_user; tlb_flush_nonlocked_t spitfire_tlb_flush_nonlocked; tlb_flush_user_t spitfire_tlb_flush_user; +tlb_flush_nonlocked_t zeus_tlb_flush_nonlocked; +tlb_flush_user_t zeus_tlb_flush_user; + extern tlb_flush_nonlocked_t *tlb_flush_nonlocked; extern tlb_flush_user_t *tlb_flush_user; Modified: head/sys/sparc64/sparc64/cache.c ============================================================================== --- head/sys/sparc64/sparc64/cache.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/cache.c Sat Jul 2 11:14:54 2011 (r223719) @@ -169,12 +169,12 @@ cache_init(struct pcpu *pcpu) tlb_flush_nonlocked = cheetah_tlb_flush_nonlocked; tlb_flush_user = cheetah_tlb_flush_user; } else if (pcpu->pc_impl == CPU_IMPL_SPARC64V) { - cache_enable = cheetah_cache_enable; + cache_enable = zeus_cache_enable; cache_flush = zeus_cache_flush; dcache_page_inval = zeus_dcache_page_inval; icache_page_inval = zeus_icache_page_inval; - tlb_flush_nonlocked = cheetah_tlb_flush_nonlocked; - tlb_flush_user = cheetah_tlb_flush_user; + tlb_flush_nonlocked = zeus_tlb_flush_nonlocked; + tlb_flush_user = zeus_tlb_flush_user; } else if (pcpu->pc_impl >= CPU_IMPL_ULTRASPARCI && pcpu->pc_impl < CPU_IMPL_ULTRASPARCIII) { cache_enable = spitfire_cache_enable; Modified: head/sys/sparc64/sparc64/cheetah.c ============================================================================== --- head/sys/sparc64/sparc64/cheetah.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/cheetah.c Sat Jul 2 11:14:54 2011 (r223719) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2003 Jake Burkholder. - * Copyright (c) 2005, 2008, 2010 Marius Strobl <marius@FreeBSD.org> + * Copyright (c) 2005 - 2011 Marius Strobl <marius@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -28,8 +28,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include "opt_pmap.h" - #include <sys/param.h> #include <sys/systm.h> #include <sys/lock.h> @@ -45,17 +43,19 @@ __FBSDID("$FreeBSD$"); #include <machine/cpufunc.h> #include <machine/dcr.h> #include <machine/lsu.h> -#include <machine/mcntl.h> #include <machine/smp.h> #include <machine/tlb.h> #include <machine/ver.h> #include <machine/vmparam.h> #define CHEETAH_ICACHE_TAG_LOWER 0x30 +#define CHEETAH_T16_ENTRIES 16 +#define CHEETAH_DT512_ENTRIES 512 +#define CHEETAH_IT128_ENTRIES 128 +#define CHEETAH_IT512_ENTRIES 512 /* - * CPU-specific initialization - this is used for both the Sun Cheetah and - * later as well as the Fujitsu Zeus and later CPUs. + * CPU-specific initialization for Sun Cheetah and later CPUs */ void cheetah_init(u_int cpu_impl) @@ -78,14 +78,6 @@ cheetah_init(u_int cpu_impl) stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0); membar(Sync); - if (cpu_impl == CPU_IMPL_SPARC64V) { - /* Ensure MCNTL_JPS1_TSBP is 0. */ - val = ldxa(AA_MCNTL, ASI_MCNTL); - val &= ~MCNTL_JPS1_TSBP; - stxa(AA_MCNTL, ASI_MCNTL, val); - return; - } - /* * Configure the first large dTLB to hold 4MB pages (e.g. for direct * mappings) for all three contexts and ensure the second one is set @@ -223,33 +215,92 @@ cheetah_icache_page_inval(vm_paddr_t pa } -#define cheetah_dmap_all() do { \ - stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); \ - stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); \ - flush(KERNBASE); \ -} while (0) - /* - * Flush all non-locked mappings from the TLB. + * Flush all non-locked mappings from the TLBs. */ void cheetah_tlb_flush_nonlocked(void) { - cheetah_dmap_all(); + stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); + stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); + flush(KERNBASE); } /* - * Flush all user mappings from the TLB. + * Flush all user mappings from the TLBs. */ void -cheetah_tlb_flush_user() +cheetah_tlb_flush_user(void) { + u_long data, tag; + register_t s; + u_int i, slot; /* - * Just use cheetah_dmap_all() and accept somes TLB misses - * rather than searching all 1040 D-TLB and 144 I-TLB slots - * for non-kernel mappings. + * We read ASI_{D,I}TLB_DATA_ACCESS_REG twice back-to-back in order + * to work around errata of USIII and beyond. */ - cheetah_dmap_all(); + for (i = 0; i < CHEETAH_T16_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_T16, i); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + for (i = 0; i < CHEETAH_DT512_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_DT512_0, i); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + slot = TLB_DAR_SLOT(TLB_DAR_DT512_1, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + } + if (PCPU_GET(impl) == CPU_IMPL_ULTRASPARCIVp) { + for (i = 0; i < CHEETAH_IT512_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_IT512, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + } else { + for (i = 0; i < CHEETAH_IT128_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_IT128, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + intr_restore(s); + if ((data & TD_V) != 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + } } Modified: head/sys/sparc64/sparc64/machdep.c ============================================================================== --- head/sys/sparc64/sparc64/machdep.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/machdep.c Sat Jul 2 11:14:54 2011 (r223719) @@ -348,9 +348,10 @@ sparc64_init(caddr_t mdp, u_long o1, u_l /* * Do CPU-specific initialization. */ - if (cpu_impl == CPU_IMPL_SPARC64V || - cpu_impl >= CPU_IMPL_ULTRASPARCIII) + if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(cpu_impl); + else if (cpu_impl == CPU_IMPL_SPARC64V) + zeus_init(cpu_impl); /* * Clear (S)TICK timer (including NPT). Modified: head/sys/sparc64/sparc64/mp_machdep.c ============================================================================== --- head/sys/sparc64/sparc64/mp_machdep.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/mp_machdep.c Sat Jul 2 11:14:54 2011 (r223719) @@ -428,9 +428,11 @@ cpu_mp_bootstrap(struct pcpu *pc) csa = &cpu_start_args; /* Do CPU-specific initialization. */ - if (pc->pc_impl == CPU_IMPL_SPARC64V || - pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) + if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(pc->pc_impl); + else if (pc->pc_impl == CPU_IMPL_SPARC64V) + zeus_init(pc->pc_impl); + /* * Enable the caches. Note that his may include applying workarounds. */ Modified: head/sys/sparc64/sparc64/pmap.c ============================================================================== --- head/sys/sparc64/sparc64/pmap.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/pmap.c Sat Jul 2 11:14:54 2011 (r223719) @@ -247,7 +247,7 @@ PMAP_STATS_VAR(pmap_ncopy_page_soc); PMAP_STATS_VAR(pmap_nnew_thread); PMAP_STATS_VAR(pmap_nnew_thread_oc); -static inline u_long dtlb_get_data(u_int slot); +static inline u_long dtlb_get_data(u_int tlb, u_int slot); /* * Quick sort callout for comparing memory regions @@ -288,15 +288,21 @@ om_cmp(const void *a, const void *b) } static inline u_long -dtlb_get_data(u_int slot) +dtlb_get_data(u_int tlb, u_int slot) { + u_long data; + register_t s; + slot = TLB_DAR_SLOT(tlb, slot); /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work - * around errata of USIII and beyond. + * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to + * work around errata of USIII and beyond. */ - (void)ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG); - return (ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG)); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + return (data); } /* @@ -392,7 +398,9 @@ pmap_bootstrap(u_int cpu_impl) } else { dtlb_slots_avail = 0; for (i = 0; i < dtlb_slots; i++) { - data = dtlb_get_data(i); + data = dtlb_get_data(cpu_impl == + CPU_IMPL_ULTRASPARCIII ? TLB_DAR_T16 : + TLB_DAR_T32, i); if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) dtlb_slots_avail++; } Modified: head/sys/sparc64/sparc64/spitfire.c ============================================================================== --- head/sys/sparc64/sparc64/spitfire.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/spitfire.c Sat Jul 2 11:14:54 2011 (r223719) @@ -140,47 +140,45 @@ spitfire_icache_page_inval(vm_paddr_t pa } /* - * Flush all non-locked mappings from the TLB. + * Flush all non-locked mappings from the TLBs. */ void spitfire_tlb_flush_nonlocked(void) { - int i; + u_int i; + u_int slot; for (i = 0; i < SPITFIRE_TLB_ENTRIES; i++) { - if ((ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG) & - TD_L) == 0) - stxa_sync(TLB_DAR_SLOT(i), - ASI_DTLB_DATA_ACCESS_REG, 0); - if ((ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG) & - TD_L) == 0) - stxa_sync(TLB_DAR_SLOT(i), - ASI_ITLB_DATA_ACCESS_REG, 0); + slot = TLB_DAR_SLOT(TLB_DAR_T32, i); + if ((ldxa(slot, ASI_DTLB_DATA_ACCESS_REG) & TD_L) == 0) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + if ((ldxa(slot, ASI_ITLB_DATA_ACCESS_REG) & TD_L) == 0) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); } } /* - * Flush all user mappings from the TLB. + * Flush all user mappings from the TLBs. */ void spitfire_tlb_flush_user(void) { u_long data; u_long tag; - int i; + u_int i; + u_int slot; for (i = 0; i < SPITFIRE_TLB_ENTRIES; i++) { - data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); - tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); + slot = TLB_DAR_SLOT(TLB_DAR_T32, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) - stxa_sync(TLB_DAR_SLOT(i), - ASI_DTLB_DATA_ACCESS_REG, 0); - data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); - tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) - stxa_sync(TLB_DAR_SLOT(i), - ASI_ITLB_DATA_ACCESS_REG, 0); + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); } } Modified: head/sys/sparc64/sparc64/zeus.c ============================================================================== --- head/sys/sparc64/sparc64/zeus.c Sat Jul 2 10:17:26 2011 (r223718) +++ head/sys/sparc64/sparc64/zeus.c Sat Jul 2 11:14:54 2011 (r223719) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2010 Marius Strobl <marius@FreeBSD.org> + * Copyright (c) 2010 - 2011 Marius Strobl <marius@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,7 +32,64 @@ __FBSDID("$FreeBSD$"); #include <machine/asi.h> #include <machine/cache.h> +#include <machine/cpu.h> #include <machine/cpufunc.h> +#include <machine/mcntl.h> +#include <machine/lsu.h> +#include <machine/tlb.h> +#include <machine/tte.h> +#include <machine/vmparam.h> + +#define ZEUS_FTLB_ENTRIES 32 +#define ZEUS_STLB_ENTRIES 2048 + +/* + * CPU-specific initialization for Fujitsu Zeus CPUs + */ +void +zeus_init(u_int cpu_impl) +{ + u_long val; + + /* Ensure the TSB Extension Registers hold 0 as TSB_Base. */ + + stxa(AA_DMMU_TSB_PEXT_REG, ASI_DMMU, 0); + stxa(AA_IMMU_TSB_PEXT_REG, ASI_IMMU, 0); + membar(Sync); + + stxa(AA_DMMU_TSB_SEXT_REG, ASI_DMMU, 0); + /* + * NB: the secondary context was removed from the iMMU. + */ + membar(Sync); + + stxa(AA_DMMU_TSB_NEXT_REG, ASI_DMMU, 0); + stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0); + membar(Sync); + + val = ldxa(AA_MCNTL, ASI_MCNTL); + /* Ensure MCNTL_JPS1_TSBP is 0. */ + val &= ~MCNTL_JPS1_TSBP; + /* + * Ensure 4-Mbyte page entries are stored in the 1024-entry, 2-way set + * associative TLB. + */ + val = (val & ~MCNTL_RMD_MASK) | MCNTL_RMD_1024; + stxa(AA_MCNTL, ASI_MCNTL, val); +} + +/* + * Enable level 1 caches. + */ +void +zeus_cache_enable(u_int cpu_impl) +{ + u_long lsu; + + lsu = ldxa(0, ASI_LSU_CTL_REG); + stxa(0, ASI_LSU_CTL_REG, lsu | LSU_IC | LSU_DC); + flush(KERNBASE); +} /* * Flush all lines from the level 1 caches. @@ -63,3 +120,52 @@ zeus_icache_page_inval(vm_paddr_t pa __u { } + +/* + * Flush all non-locked mappings from the TLBs. + */ +void +zeus_tlb_flush_nonlocked(void) +{ + + stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); + stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); + flush(KERNBASE); +} + +/* + * Flush all user mappings from the TLBs. + */ +void +zeus_tlb_flush_user(void) +{ + u_long data, tag; + u_int i, slot; + + for (i = 0; i < ZEUS_FTLB_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_FTLB, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + for (i = 0; i < ZEUS_STLB_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_STLB, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201107021114.p62BEssr037439>