From owner-svn-src-all@FreeBSD.ORG Fri Apr 4 19:17:34 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 825E02C9; Fri, 4 Apr 2014 19:17:34 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 6EC1E6E9; Fri, 4 Apr 2014 19:17:34 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s34JHYkD048022; Fri, 4 Apr 2014 19:17:34 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s34JHYeL048020; Fri, 4 Apr 2014 19:17:34 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201404041917.s34JHYeL048020@svn.freebsd.org> From: Konstantin Belousov Date: Fri, 4 Apr 2014 19:17:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r264134 - stable/10/sys/amd64/amd64 X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 04 Apr 2014 19:17:34 -0000 Author: kib Date: Fri Apr 4 19:17:33 2014 New Revision: 264134 URL: http://svnweb.freebsd.org/changeset/base/264134 Log: MFC r263878: Several fixes for the PCID implementation. Modified: stable/10/sys/amd64/amd64/mp_machdep.c stable/10/sys/amd64/amd64/pmap.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/amd64/amd64/mp_machdep.c ============================================================================== --- stable/10/sys/amd64/amd64/mp_machdep.c Fri Apr 4 18:10:11 2014 (r264133) +++ stable/10/sys/amd64/amd64/mp_machdep.c Fri Apr 4 19:17:33 2014 (r264134) @@ -1257,7 +1257,7 @@ smp_masked_invltlb(cpuset_t mask, pmap_t { if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, NULL, 0, 0); + smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_global++; #endif @@ -1517,6 +1517,7 @@ void invltlb_pcid_handler(void) { uint64_t cr3; + u_int cpuid; #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1524,14 +1525,13 @@ invltlb_pcid_handler(void) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - cr3 = rcr3(); if (smp_tlb_invpcid.pcid != (uint64_t)-1 && smp_tlb_invpcid.pcid != 0) { - if (invpcid_works) { invpcid(&smp_tlb_invpcid, INVPCID_CTX); } else { /* Otherwise reload %cr3 twice. */ + cr3 = rcr3(); if (cr3 != pcid_cr3) { load_cr3(pcid_cr3); cr3 |= CR3_PCID_SAVE; @@ -1541,8 +1541,11 @@ invltlb_pcid_handler(void) } else { invltlb_globpcid(); } - if (smp_tlb_pmap != NULL) - CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); + if (smp_tlb_pmap != NULL) { + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &smp_tlb_pmap->pm_save); + } atomic_add_int(&smp_tlb_wait, 1); } @@ -1608,7 +1611,10 @@ invlpg_range(vm_offset_t start, vm_offse void invlrng_handler(void) { + struct invpcid_descr d; vm_offset_t addr; + uint64_t cr3; + u_int cpuid; #ifdef COUNT_XINVLTLB_HITS xhits_rng[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1618,15 +1624,7 @@ invlrng_handler(void) addr = smp_tlb_invpcid.addr; if (pmap_pcid_enabled) { - if (invpcid_works) { - struct invpcid_descr d; - - d = smp_tlb_invpcid; - do { - invpcid(&d, INVPCID_ADDR); - d.addr += PAGE_SIZE; - } while (d.addr < smp_tlb_addr2); - } else if (smp_tlb_invpcid.pcid == 0) { + if (smp_tlb_invpcid.pcid == 0) { /* * kernel pmap - use invlpg to invalidate * global mapping. @@ -1635,12 +1633,18 @@ invlrng_handler(void) } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { invltlb_globpcid(); if (smp_tlb_pmap != NULL) { - CPU_CLR_ATOMIC(PCPU_GET(cpuid), - &smp_tlb_pmap->pm_save); + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, + &smp_tlb_pmap->pm_save); } + } else if (invpcid_works) { + d = smp_tlb_invpcid; + do { + invpcid(&d, INVPCID_ADDR); + d.addr += PAGE_SIZE; + } while (d.addr <= smp_tlb_addr2); } else { - uint64_t cr3; - cr3 = rcr3(); if (cr3 != pcid_cr3) load_cr3(pcid_cr3 | CR3_PCID_SAVE); Modified: stable/10/sys/amd64/amd64/pmap.c ============================================================================== --- stable/10/sys/amd64/amd64/pmap.c Fri Apr 4 18:10:11 2014 (r264133) +++ stable/10/sys/amd64/amd64/pmap.c Fri Apr 4 19:17:33 2014 (r264134) @@ -838,7 +838,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_cr3 = KPML4phys; CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ - CPU_ZERO(&kernel_pmap->pm_save); + CPU_FILL(&kernel_pmap->pm_save); /* always superset of pm_active */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); kernel_pmap->pm_flags = pmap_flags; @@ -1492,7 +1492,8 @@ pmap_invalidate_all(pmap_t pmap) } else { invltlb_globpcid(); } - CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); + if (!CPU_ISSET(cpuid, &pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); smp_invltlb(pmap); } else { other_cpus = all_cpus; @@ -1526,7 +1527,8 @@ pmap_invalidate_all(pmap_t pmap) } } else if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); - CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); + if (!CPU_ISSET(cpuid, &pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); if (pmap_pcid_enabled) CPU_AND(&other_cpus, &pmap->pm_save); else