Date: Tue, 4 Jun 2019 17:32:00 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r348646 - in stable/12/sys/riscv: include riscv Message-ID: <201906041732.x54HW0Ql002308@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Tue Jun 4 17:31:59 2019 New Revision: 348646 URL: https://svnweb.freebsd.org/changeset/base/348646 Log: MFC r344108: Implement per-CPU pmap activation tracking for RISC-V. Modified: stable/12/sys/riscv/include/pcb.h stable/12/sys/riscv/include/pcpu.h stable/12/sys/riscv/include/pmap.h stable/12/sys/riscv/riscv/genassym.c stable/12/sys/riscv/riscv/machdep.c stable/12/sys/riscv/riscv/mp_machdep.c stable/12/sys/riscv/riscv/pmap.c stable/12/sys/riscv/riscv/swtch.S stable/12/sys/riscv/riscv/vm_machdep.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/riscv/include/pcb.h ============================================================================== --- stable/12/sys/riscv/include/pcb.h Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/include/pcb.h Tue Jun 4 17:31:59 2019 (r348646) @@ -55,7 +55,6 @@ struct pcb { #define PCB_FP_STARTED 0x1 #define PCB_FP_USERMASK 0x1 uint64_t pcb_sepc; /* Supervisor exception pc */ - vm_offset_t pcb_l1addr; /* L1 page tables base address */ vm_offset_t pcb_onfault; /* Copyinout fault handler */ }; Modified: stable/12/sys/riscv/include/pcpu.h ============================================================================== --- stable/12/sys/riscv/include/pcpu.h Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/include/pcpu.h Tue Jun 4 17:31:59 2019 (r348646) @@ -45,6 +45,7 @@ #define ALT_STACK_SIZE 128 #define PCPU_MD_FIELDS \ + struct pmap *pc_curpmap; /* Currently active pmap */ \ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \ char __pad[61] Modified: stable/12/sys/riscv/include/pmap.h ============================================================================== --- stable/12/sys/riscv/include/pmap.h Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/include/pmap.h Tue Jun 4 17:31:59 2019 (r348646) @@ -41,6 +41,7 @@ #ifndef LOCORE #include <sys/queue.h> +#include <sys/_cpuset.h> #include <sys/_lock.h> #include <sys/_mutex.h> @@ -80,6 +81,8 @@ struct pmap { struct mtx pm_mtx; struct pmap_statistics pm_stats; /* pmap statictics */ pd_entry_t *pm_l1; + u_long pm_satp; /* value for SATP register */ + cpuset_t pm_active; /* active on cpus */ TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ LIST_ENTRY(pmap) pm_list; /* List of all pmaps */ struct vm_radix pm_root; @@ -137,6 +140,10 @@ extern vm_offset_t virtual_end; #define L1_MAPPABLE_P(va, pa, size) \ ((((va) | (pa)) & L1_OFFSET) == 0 && (size) >= L1_SIZE) +struct thread; + +void pmap_activate_boot(pmap_t); +void pmap_activate_sw(struct thread *); void pmap_bootstrap(vm_offset_t, vm_paddr_t, vm_size_t); void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t); vm_paddr_t pmap_kextract(vm_offset_t va); Modified: stable/12/sys/riscv/riscv/genassym.c ============================================================================== --- stable/12/sys/riscv/riscv/genassym.c Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/genassym.c Tue Jun 4 17:31:59 2019 (r348646) @@ -63,7 +63,6 @@ ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); -ASSYM(PCB_L1ADDR, offsetof(struct pcb, pcb_l1addr)); ASSYM(PCB_SIZE, sizeof(struct pcb)); ASSYM(PCB_RA, offsetof(struct pcb, pcb_ra)); ASSYM(PCB_SP, offsetof(struct pcb, pcb_sp)); Modified: stable/12/sys/riscv/riscv/machdep.c ============================================================================== --- stable/12/sys/riscv/riscv/machdep.c Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/machdep.c Tue Jun 4 17:31:59 2019 (r348646) @@ -878,10 +878,6 @@ initriscv(struct riscv_bootparams *rvbp) init_proc0(rvbp->kern_stack); - /* set page table base register for thread0 */ - thread0.td_pcb->pcb_l1addr = \ - (rvbp->kern_l1pt - KERNBASE + rvbp->kern_phys); - msgbufinit(msgbufp, msgbufsize); mutex_init(); init_param2(physmem); Modified: stable/12/sys/riscv/riscv/mp_machdep.c ============================================================================== --- stable/12/sys/riscv/riscv/mp_machdep.c Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/mp_machdep.c Tue Jun 4 17:31:59 2019 (r348646) @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <vm/vm_extern.h> #include <vm/vm_kern.h> +#include <vm/vm_map.h> #include <machine/intr.h> #include <machine/smp.h> @@ -254,6 +255,9 @@ init_secondary(uint64_t cpu) /* Enable external (PLIC) interrupts */ csr_set(sie, SIE_SEIE); + + /* Activate process 0's pmap. */ + pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); mtx_lock_spin(&ap_boot_mtx); Modified: stable/12/sys/riscv/riscv/pmap.c ============================================================================== --- stable/12/sys/riscv/riscv/pmap.c Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/pmap.c Tue Jun 4 17:31:59 2019 (r348646) @@ -118,9 +118,10 @@ __FBSDID("$FreeBSD$"); */ #include <sys/param.h> +#include <sys/systm.h> #include <sys/bitstring.h> #include <sys/bus.h> -#include <sys/systm.h> +#include <sys/cpuset.h> #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/lock.h> @@ -566,6 +567,8 @@ pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, rw_init(&pvh_global_lock, "pmap pv global"); + CPU_FILL(&kernel_pmap->pm_active); + /* Assume the address we were loaded to is a valid physical address. */ min_pa = max_pa = kernstart; @@ -723,9 +726,6 @@ pmap_init(void) * In general, the calling thread uses a plain fence to order the * writes to the page tables before invoking an SBI callback to invoke * sfence_vma() on remote CPUs. - * - * Since the riscv pmap does not yet have a pm_active field, IPIs are - * sent to all CPUs in the system. */ static void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) @@ -733,10 +733,11 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) cpuset_t mask; sched_pin(); - mask = all_cpus; + mask = pmap->pm_active; CPU_CLR(PCPU_GET(cpuid), &mask); fence(); - sbi_remote_sfence_vma(mask.__bits, va, 1); + if (!CPU_EMPTY(&mask) && smp_started) + sbi_remote_sfence_vma(mask.__bits, va, 1); sfence_vma_page(va); sched_unpin(); } @@ -747,10 +748,11 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm cpuset_t mask; sched_pin(); - mask = all_cpus; + mask = pmap->pm_active; CPU_CLR(PCPU_GET(cpuid), &mask); fence(); - sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1); + if (!CPU_EMPTY(&mask) && smp_started) + sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1); /* * Might consider a loop of sfence_vma_page() for a small @@ -766,16 +768,17 @@ pmap_invalidate_all(pmap_t pmap) cpuset_t mask; sched_pin(); - mask = all_cpus; + mask = pmap->pm_active; CPU_CLR(PCPU_GET(cpuid), &mask); - fence(); /* * XXX: The SBI doc doesn't detail how to specify x0 as the * address to perform a global fence. BBL currently treats * all sfence_vma requests as global however. */ - sbi_remote_sfence_vma(mask.__bits, 0, 0); + fence(); + if (!CPU_EMPTY(&mask) && smp_started) + sbi_remote_sfence_vma(mask.__bits, 0, 0); sfence_vma(); sched_unpin(); } @@ -1199,6 +1202,9 @@ pmap_pinit0(pmap_t pmap) PMAP_LOCK_INIT(pmap); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); pmap->pm_l1 = kernel_pmap->pm_l1; + pmap->pm_satp = SATP_MODE_SV39 | (vtophys(pmap->pm_l1) >> PAGE_SHIFT); + CPU_ZERO(&pmap->pm_active); + pmap_activate_boot(pmap); } int @@ -1216,12 +1222,15 @@ pmap_pinit(pmap_t pmap) l1phys = VM_PAGE_TO_PHYS(l1pt); pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); + pmap->pm_satp = SATP_MODE_SV39 | (l1phys >> PAGE_SHIFT); if ((l1pt->flags & PG_ZERO) == 0) pagezero(pmap->pm_l1); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); + CPU_ZERO(&pmap->pm_active); + /* Install kernel pagetables */ memcpy(pmap->pm_l1, kernel_pmap->pm_l1, PAGE_SIZE); @@ -1411,6 +1420,8 @@ pmap_release(pmap_t pmap) KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); + KASSERT(CPU_EMPTY(&pmap->pm_active), + ("releasing active pmap %p", pmap)); mtx_lock(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); @@ -4242,26 +4253,56 @@ done: } void -pmap_activate(struct thread *td) +pmap_activate_sw(struct thread *td) { - pmap_t pmap; - uint64_t reg; + pmap_t oldpmap, pmap; + u_int cpu; - critical_enter(); + oldpmap = PCPU_GET(curpmap); pmap = vmspace_pmap(td->td_proc->p_vmspace); - td->td_pcb->pcb_l1addr = vtophys(pmap->pm_l1); + if (pmap == oldpmap) + return; + load_satp(pmap->pm_satp); - reg = SATP_MODE_SV39; - reg |= (td->td_pcb->pcb_l1addr >> PAGE_SHIFT); - load_satp(reg); + cpu = PCPU_GET(cpuid); +#ifdef SMP + CPU_SET_ATOMIC(cpu, &pmap->pm_active); + CPU_CLR_ATOMIC(cpu, &oldpmap->pm_active); +#else + CPU_SET(cpu, &pmap->pm_active); + CPU_CLR(cpu, &oldpmap->pm_active); +#endif + PCPU_SET(curpmap, pmap); - pmap_invalidate_all(pmap); + sfence_vma(); +} + +void +pmap_activate(struct thread *td) +{ + + critical_enter(); + pmap_activate_sw(td); critical_exit(); } void -pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) +pmap_activate_boot(pmap_t pmap) { + u_int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef SMP + CPU_SET_ATOMIC(cpu, &pmap->pm_active); +#else + CPU_SET(cpu, &pmap->pm_active); +#endif + PCPU_SET(curpmap, pmap); +} + +void +pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) +{ cpuset_t mask; /* @@ -4276,7 +4317,8 @@ pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t mask = all_cpus; CPU_CLR(PCPU_GET(cpuid), &mask); fence(); - sbi_remote_fence_i(mask.__bits); + if (!CPU_EMPTY(&mask) && smp_started) + sbi_remote_fence_i(mask.__bits); sched_unpin(); } Modified: stable/12/sys/riscv/riscv/swtch.S ============================================================================== --- stable/12/sys/riscv/riscv/swtch.S Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/swtch.S Tue Jun 4 17:31:59 2019 (r348646) @@ -207,28 +207,21 @@ ENTRY(fpe_state_clear) END(fpe_state_clear) /* - * void cpu_throw(struct thread *old, struct thread *new) + * void cpu_throw(struct thread *old __unused, struct thread *new) */ ENTRY(cpu_throw) + /* Activate the new thread's pmap. */ + mv s0, a1 + mv a0, a1 + call _C_LABEL(pmap_activate_sw) + mv a0, s0 + /* Store the new curthread */ - sd a1, PC_CURTHREAD(gp) + sd a0, PC_CURTHREAD(gp) /* And the new pcb */ - ld x13, TD_PCB(a1) + ld x13, TD_PCB(a0) sd x13, PC_CURPCB(gp) - sfence.vma - - /* Switch to the new pmap */ - ld t0, PCB_L1ADDR(x13) - srli t0, t0, PAGE_SHIFT - li t1, SATP_MODE_SV39 - or t0, t0, t1 - csrw satp, t0 - - /* TODO: Invalidate the TLB */ - - sfence.vma - /* Load registers */ ld ra, (PCB_RA)(x13) ld sp, (PCB_SP)(x13) @@ -250,7 +243,7 @@ ENTRY(cpu_throw) #ifdef FPE /* Is FPE enabled for new thread? */ - ld t0, TD_FRAME(a1) + ld t0, TD_FRAME(a0) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 @@ -324,38 +317,27 @@ ENTRY(cpu_switch) 1: #endif - /* - * Restore the saved context. - */ - ld x13, TD_PCB(a1) + /* Activate the new thread's pmap */ + mv s0, a0 + mv s1, a1 + mv s2, a2 + mv a0, a1 + call _C_LABEL(pmap_activate_sw) + mv a1, s1 - /* - * TODO: We may need to flush the cache here if switching - * to a user process. - */ - - sfence.vma - - /* Switch to the new pmap */ - ld t0, PCB_L1ADDR(x13) - srli t0, t0, PAGE_SHIFT - li t1, SATP_MODE_SV39 - or t0, t0, t1 - csrw satp, t0 - - /* TODO: Invalidate the TLB */ - - sfence.vma - /* Release the old thread */ - sd a2, TD_LOCK(a0) + sd s2, TD_LOCK(s0) #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ - la a2, _C_LABEL(blocked_lock) + la s2, _C_LABEL(blocked_lock) 1: ld t0, TD_LOCK(a1) - beq t0, a2, 1b + beq t0, s2, 1b #endif + /* + * Restore the saved context. + */ + ld x13, TD_PCB(a1) /* Restore the registers */ ld tp, (PCB_TP)(x13) Modified: stable/12/sys/riscv/riscv/vm_machdep.c ============================================================================== --- stable/12/sys/riscv/riscv/vm_machdep.c Tue Jun 4 17:31:29 2019 (r348645) +++ stable/12/sys/riscv/riscv/vm_machdep.c Tue Jun 4 17:31:59 2019 (r348646) @@ -92,9 +92,6 @@ cpu_fork(struct thread *td1, struct proc *p2, struct t td2->td_pcb = pcb2; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); - td2->td_pcb->pcb_l1addr = - vtophys(vmspace_pmap(td2->td_proc->p_vmspace)->pm_l1); - tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1); bcopy(td1->td_frame, tf, sizeof(*tf));
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201906041732.x54HW0Ql002308>