Date: Thu, 6 Oct 2011 11:01:31 +0000 (UTC) From: Marius Strobl <marius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r226054 - in head/sys/sparc64: include sparc64 Message-ID: <201110061101.p96B1Vnq057627@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: marius Date: Thu Oct 6 11:01:31 2011 New Revision: 226054 URL: http://svn.freebsd.org/changeset/base/226054 Log: - Use atomic operations rather than sched_lock for safely assigning pm_active and pc_pmap for SMP. This is key to allowing adding support for SCHED_ULE. Thanks go to Peter Jeremy for additional testing. - Add support for SCHED_ULE to cpu_switch(). Committed from: 201110DevSummit Modified: head/sys/sparc64/include/asmacros.h head/sys/sparc64/sparc64/pmap.c head/sys/sparc64/sparc64/swtch.S Modified: head/sys/sparc64/include/asmacros.h ============================================================================== --- head/sys/sparc64/include/asmacros.h Thu Oct 6 07:56:18 2011 (r226053) +++ head/sys/sparc64/include/asmacros.h Thu Oct 6 11:01:31 2011 (r226054) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2001 Jake Burkholder. + * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -98,9 +99,67 @@ 9: andn r2, bits, r3 ; \ casxa [r1] ASI_N, r2, r3 ; \ cmp r2, r3 ; \ + bne,pn %xcc, 9b ; \ + mov r3, r2 + +/* + * Atomically load an integer from memory. + */ +#define ATOMIC_LOAD_INT(r1, val) \ + clr val ; \ + casa [r1] ASI_N, %g0, val + +/* + * Atomically load a long from memory. + */ +#define ATOMIC_LOAD_LONG(r1, val) \ + clr val ; \ + casxa [r1] ASI_N, %g0, val + +/* + * Atomically set a number of bits of an integer in memory. + */ +#define ATOMIC_SET_INT(r1, r2, r3, bits) \ + lduw [r1], r2 ; \ +9: or r2, bits, r3 ; \ + casa [r1] ASI_N, r2, r3 ; \ + cmp r2, r3 ; \ + bne,pn %icc, 9b ; \ + mov r3, r2 + +/* + * Atomically set a number of bits of a long in memory. + */ +#define ATOMIC_SET_LONG(r1, r2, r3, bits) \ + ldx [r1], r2 ; \ +9: or r2, bits, r3 ; \ + casxa [r1] ASI_N, r2, r3 ; \ + cmp r2, r3 ; \ + bne,pn %xcc, 9b ; \ + mov r3, r2 + +/* + * Atomically store an integer in memory. + */ +#define ATOMIC_STORE_INT(r1, r2, r3, val) \ + lduw [r1], r2 ; \ +9: mov val, r3 ; \ + casa [r1] ASI_N, r2, r3 ; \ + cmp r2, r3 ; \ bne,pn %icc, 9b ; \ mov r3, r2 +/* + * Atomically store a long in memory. + */ +#define ATOMIC_STORE_LONG(r1, r2, r3, val) \ + ldx [r1], r2 ; \ +9: mov val, r3 ; \ + casxa [r1] ASI_N, r2, r3 ; \ + cmp r2, r3 ; \ + bne,pn %xcc, 9b ; \ + mov r3, r2 + #define PCPU(member) PCPU_REG + PC_ ## member #define PCPU_ADDR(member, reg) \ add PCPU_REG, PC_ ## member, reg Modified: head/sys/sparc64/sparc64/pmap.c ============================================================================== --- head/sys/sparc64/sparc64/pmap.c Thu Oct 6 07:56:18 2011 (r226053) +++ head/sys/sparc64/sparc64/pmap.c Thu Oct 6 11:01:31 2011 (r226054) @@ -100,13 +100,6 @@ __FBSDID("$FreeBSD$"); #include <machine/tsb.h> #include <machine/ver.h> -/* XXX */ -#include "opt_sched.h" -#ifndef SCHED_4BSD -#error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." -#endif -extern struct mtx sched_lock; - /* * Virtual address of message buffer */ @@ -1232,11 +1225,9 @@ pmap_pinit(pmap_t pm) if (pm->pm_tsb_obj == NULL) pm->pm_tsb_obj = vm_object_allocate(OBJT_PHYS, TSB_PAGES); - mtx_lock_spin(&sched_lock); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = -1; CPU_ZERO(&pm->pm_active); - mtx_unlock_spin(&sched_lock); VM_OBJECT_LOCK(pm->pm_tsb_obj); for (i = 0; i < TSB_PAGES; i++) { @@ -1263,7 +1254,9 @@ pmap_release(pmap_t pm) { vm_object_t obj; vm_page_t m; +#ifdef SMP struct pcpu *pc; +#endif CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", pm->pm_context[curcpu], pm->pm_tsb); @@ -1283,11 +1276,18 @@ pmap_release(pmap_t pm) * - A process that referenced this pmap ran on a CPU, but we switched * to a kernel thread, leaving the pmap pointer unchanged. */ - mtx_lock_spin(&sched_lock); +#ifdef SMP + sched_pin(); STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) - if (pc->pc_pmap == pm) - pc->pc_pmap = NULL; - mtx_unlock_spin(&sched_lock); + atomic_cmpset_rel_ptr((uintptr_t *)&pc->pc_pmap, + (uintptr_t)pm, (uintptr_t)NULL); + sched_unpin(); +#else + critical_enter(); + if (PCPU_GET(pmap) == pm) + PCPU_SET(pmap, NULL); + critical_exit(); +#endif pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); obj = pm->pm_tsb_obj; @@ -2232,11 +2232,14 @@ pmap_activate(struct thread *td) } PCPU_SET(tlb_ctx, context + 1); - mtx_lock_spin(&sched_lock); pm->pm_context[curcpu] = context; +#ifdef SMP + CPU_SET_ATOMIC(PCPU_GET(cpuid), &pm->pm_active); + atomic_store_ptr((uintptr_t *)PCPU_PTR(pmap), (uintptr_t)pm); +#else CPU_SET(PCPU_GET(cpuid), &pm->pm_active); PCPU_SET(pmap, pm); - mtx_unlock_spin(&sched_lock); +#endif stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); Modified: head/sys/sparc64/sparc64/swtch.S ============================================================================== --- head/sys/sparc64/sparc64/swtch.S Thu Oct 6 07:56:18 2011 (r226053) +++ head/sys/sparc64/sparc64/swtch.S Thu Oct 6 11:01:31 2011 (r226054) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2001 Jake Burkholder. + * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include <machine/tstate.h> #include "assym.s" +#include "opt_sched.h" .register %g2, #ignore .register %g3, #ignore @@ -66,7 +68,7 @@ ENTRY(cpu_switch) nop call savefpctx add PCB_REG, PCB_KFP, %o0 - ba,a %xcc, 2f + ba,a,pt %xcc, 2f nop /* @@ -148,7 +150,7 @@ ENTRY(cpu_switch) * If they are the same we are done. */ cmp %l2, %l1 - be,a,pn %xcc, 7f + be,a,pn %xcc, 8f nop /* @@ -157,7 +159,7 @@ ENTRY(cpu_switch) */ SET(vmspace0, %i4, %i3) cmp %i5, %i3 - be,a,pn %xcc, 7f + be,a,pn %xcc, 8f nop /* @@ -180,9 +182,15 @@ ENTRY(cpu_switch) sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 +#ifdef SMP + add %l2, %l4, %l4 + membar #LoadStore | #StoreStore + ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5) +#else ldx [%l2 + %l4], %l6 andn %l6, %l5, %l6 stx %l6, [%l2 + %l4] +#endif /* * Take away its context number. @@ -194,14 +202,20 @@ ENTRY(cpu_switch) 3: cmp %i2, %g0 be,pn %xcc, 4f - lduw [PCPU(TLB_CTX_MAX)], %i4 - stx %i2, [%i0 + TD_LOCK] + add %i0, TD_LOCK, %l4 +#if defined(SCHED_ULE) && defined(SMP) + membar #LoadStore | #StoreStore + ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) +#else + stx %i2, [%l4] +#endif /* * Find a new TLB context. If we've run out we have to flush all * user mappings from the TLB and reset the context numbers. */ 4: lduw [PCPU(TLB_CTX)], %i3 + lduw [PCPU(TLB_CTX_MAX)], %i4 cmp %i3, %i4 bne,a,pt %xcc, 5f nop @@ -237,14 +251,24 @@ ENTRY(cpu_switch) sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 +#ifdef SMP + add %l1, %l4, %l4 + ATOMIC_SET_LONG(%l4, %l6, %l7, %l5) +#else ldx [%l1 + %l4], %l6 or %l6, %l5, %l6 stx %l6, [%l1 + %l4] +#endif /* * Make note of the change in pmap. */ +#ifdef SMP + PCPU_ADDR(PMAP, %l4) + ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1) +#else stx %l1, [PCPU(PMAP)] +#endif /* * Fiddle the hardware bits. Set the TSB registers and install the @@ -264,19 +288,35 @@ ENTRY(cpu_switch) stxa %i3, [%i5] ASI_DMMU flush %i4 +6: +#if defined(SCHED_ULE) && defined(SMP) + SET(blocked_lock, %l2, %l1) + add %i1, TD_LOCK, %l2 +7: + ATOMIC_LOAD_LONG(%l2, %l3) + cmp %l1, %l3 + be,a,pn %xcc, 7b + nop +#endif + /* * Done, return and load the new process's window from the stack. */ - -6: ret + ret restore -7: cmp %i2, %g0 - be,a,pn %xcc, 6b +8: cmp %i2, %g0 + be,pn %xcc, 6b + add %i0, TD_LOCK, %l4 +#if defined(SCHED_ULE) && defined(SMP) + membar #LoadStore | #StoreStore + ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) + ba,pt %xcc, 6b nop - stx %i2, [%i0 + TD_LOCK] - ret - restore +#else + ba,pt %xcc, 6b + stx %i2, [%l4] +#endif END(cpu_switch) ENTRY(savectx)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201110061101.p96B1Vnq057627>