Date: Fri, 30 Sep 2016 18:58:50 +0000 (UTC) From: "Conrad E. Meyer" <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r306521 - in head/sys: amd64/amd64 amd64/include x86/include x86/x86 Message-ID: <201609301858.u8UIwo8v086857@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Fri Sep 30 18:58:50 2016 New Revision: 306521 URL: https://svnweb.freebsd.org/changeset/base/306521 Log: Revert r306516 for now, it is incomplete on i386 Noted by: kib Modified: head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/include/pcpu.h head/sys/x86/include/x86_smp.h head/sys/x86/x86/mp_x86.c Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Fri Sep 30 18:47:34 2016 (r306520) +++ head/sys/amd64/amd64/mp_machdep.c Fri Sep 30 18:58:50 2016 (r306521) @@ -409,7 +409,6 @@ void invltlb_invpcid_handler(void) { struct invpcid_descr d; - uint64_t generation; #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; @@ -418,20 +417,17 @@ invltlb_invpcid_handler(void) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - generation = smp_tlb_generation; d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; d.pad = 0; d.addr = 0; invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB : INVPCID_CTX); - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); } void invltlb_pcid_handler(void) { - uint64_t generation; - #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -439,7 +435,6 @@ invltlb_pcid_handler(void) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - generation = smp_tlb_generation; if (smp_tlb_pmap == kernel_pmap) { invltlb_glob(); } else { @@ -455,5 +450,5 @@ invltlb_pcid_handler(void) smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid); } } - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); } Modified: head/sys/amd64/include/pcpu.h ============================================================================== --- head/sys/amd64/include/pcpu.h Fri Sep 30 18:47:34 2016 (r306520) +++ head/sys/amd64/include/pcpu.h Fri Sep 30 18:58:50 2016 (r306521) @@ -65,8 +65,7 @@ u_int pc_vcpu_id; /* Xen vCPU ID */ \ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ - uint64_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[141] /* be divisor of PAGE_SIZE \ + char __pad[149] /* be divisor of PAGE_SIZE \ after cache alignment */ #define PC_DBREG_CMD_NONE 0 Modified: head/sys/x86/include/x86_smp.h ============================================================================== --- head/sys/x86/include/x86_smp.h Fri Sep 30 18:47:34 2016 (r306520) +++ head/sys/x86/include/x86_smp.h Fri Sep 30 18:58:50 2016 (r306521) @@ -35,7 +35,7 @@ extern volatile int aps_ready; extern struct mtx ap_boot_mtx; extern int cpu_logical; extern int cpu_cores; -extern volatile uint64_t smp_tlb_generation; +extern volatile int smp_tlb_wait; extern struct pmap *smp_tlb_pmap; extern u_int xhits_gbl[]; extern u_int xhits_pg[]; Modified: head/sys/x86/x86/mp_x86.c ============================================================================== --- head/sys/x86/x86/mp_x86.c Fri Sep 30 18:47:34 2016 (r306520) +++ head/sys/x86/x86/mp_x86.c Fri Sep 30 18:58:50 2016 (r306521) @@ -1304,15 +1304,12 @@ cpususpend_handler(void) void invlcache_handler(void) { - uint64_t generation; - #ifdef COUNT_IPIS (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - generation = smp_tlb_generation; wbinvd(); - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); } /* @@ -1370,7 +1367,7 @@ SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ /* Variables needed for SMP tlb shootdown. */ static vm_offset_t smp_tlb_addr1, smp_tlb_addr2; pmap_t smp_tlb_pmap; -volatile uint64_t smp_tlb_generation; +volatile int smp_tlb_wait; #ifdef __amd64__ #define read_eflags() read_rflags() @@ -1380,16 +1377,15 @@ static void smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2) { - cpuset_t other_cpus; - volatile uint64_t *p_cpudone; - uint64_t generation; - int cpu; + int cpu, ncpu, othercpus; + + othercpus = mp_ncpus - 1; /* does not shootdown self */ /* * Check for other cpus. Return if none. */ if (CPU_ISFULLSET(&mask)) { - if (mp_ncpus <= 1) + if (othercpus < 1) return; } else { CPU_CLR(PCPU_GET(cpuid), &mask); @@ -1403,28 +1399,23 @@ smp_targeted_tlb_shootdown(cpuset_t mask smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; smp_tlb_pmap = pmap; - generation = ++smp_tlb_generation; + smp_tlb_wait = 0; if (CPU_ISFULLSET(&mask)) { + ncpu = othercpus; ipi_all_but_self(vector); - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); } else { - other_cpus = mask; + ncpu = 0; while ((cpu = CPU_FFS(&mask)) != 0) { cpu--; CPU_CLR(cpu, &mask); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, vector); ipi_send_cpu(cpu, vector); + ncpu++; } } - while ((cpu = CPU_FFS(&other_cpus)) != 0) { - cpu--; - CPU_CLR(cpu, &other_cpus); - p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done; - while (*p_cpudone != generation) - ia32_pause(); - } + while (smp_tlb_wait < ncpu) + ia32_pause(); mtx_unlock_spin(&smp_ipi_mtx); } @@ -1482,8 +1473,6 @@ smp_cache_flush(void) void invltlb_handler(void) { - uint64_t generation; - #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1491,19 +1480,16 @@ invltlb_handler(void) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - generation = smp_tlb_generation; if (smp_tlb_pmap == kernel_pmap) invltlb_glob(); else invltlb(); - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); } void invlpg_handler(void) { - uint64_t generation; - #ifdef COUNT_XINVLTLB_HITS xhits_pg[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1511,16 +1497,14 @@ invlpg_handler(void) (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - generation = smp_tlb_generation; invlpg(smp_tlb_addr1); - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); } void invlrng_handler(void) { - vm_offset_t addr, addr2; - uint64_t generation; + vm_offset_t addr; #ifdef COUNT_XINVLTLB_HITS xhits_rng[PCPU_GET(cpuid)]++; @@ -1530,12 +1514,10 @@ invlrng_handler(void) #endif /* COUNT_IPIS */ addr = smp_tlb_addr1; - addr2 = smp_tlb_addr2; - generation = smp_tlb_generation; do { invlpg(addr); addr += PAGE_SIZE; - } while (addr < addr2); + } while (addr < smp_tlb_addr2); - PCPU_SET(smp_tlb_done, generation); + atomic_add_int(&smp_tlb_wait, 1); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201609301858.u8UIwo8v086857>