Date: Sun, 13 May 2018 23:24:48 +0000 (UTC) From: Matt Macy <mmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r333591 - in head/sys: kern sys Message-ID: <201805132324.w4DNOm3B097320@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mmacy Date: Sun May 13 23:24:48 2018 New Revision: 333591 URL: https://svnweb.freebsd.org/changeset/base/333591 Log: epoch(9): cleanups, additional debug checks, and add global_epoch - GC the _nopreempt routines - to really benefit we'd need a separate routine - they're not currently in use - they complicate the API for no benefit at this time - check that we're actually in a epoch section at exit - handle epoch_call() early in boot - Fix copyright declaration language Approved by: sbruno@ Modified: head/sys/kern/subr_epoch.c head/sys/sys/epoch.h Modified: head/sys/kern/subr_epoch.c ============================================================================== --- head/sys/kern/subr_epoch.c Sun May 13 23:16:04 2018 (r333590) +++ head/sys/kern/subr_epoch.c Sun May 13 23:24:48 2018 (r333591) @@ -1,27 +1,29 @@ /*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Neither the name of Matthew Macy nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * */ #include <sys/cdefs.h> @@ -49,7 +51,7 @@ __FBSDID("$FreeBSD$"); #include <ck_epoch.h> -MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); +static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); /* arbitrary --- needs benchmarking */ #define MAX_ADAPTIVE_SPIN 5000 @@ -116,6 +118,7 @@ struct epoch { static __read_mostly int domcount[MAXMEMDOM]; static __read_mostly int domoffsets[MAXMEMDOM]; static __read_mostly int inited; +__read_mostly epoch_t global_epoch; static void epoch_call_task(void *context); @@ -136,10 +139,8 @@ epoch_init(void *arg __unused) migrate_count = counter_u64_alloc(M_WAITOK); turnstile_count = counter_u64_alloc(M_WAITOK); switch_count = counter_u64_alloc(M_WAITOK); - if (usedomains == false) { - inited = 1; - return; - } + if (usedomains == false) + goto done; count = domain = 0; domoffsets[0] = 0; for (domain = 0; domain < vm_ndomains; domain++) { @@ -156,9 +157,11 @@ epoch_init(void *arg __unused) break; } } + done: inited = 1; + global_epoch = epoch_alloc(); } -SYSINIT(epoch, SI_SUB_CPU + 1, SI_ORDER_FIRST, epoch_init, NULL); +SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL); static void epoch_init_numa(epoch_t epoch) @@ -311,19 +314,6 @@ epoch_enter(epoch_t epoch) } void -epoch_enter_nopreempt(epoch_t epoch) -{ - struct epoch_pcpu_state *eps; - - INIT_CHECK(epoch); - critical_enter(); - eps = epoch->e_pcpu[curcpu]; - curthread->td_epochnest++; - MPASS(curthread->td_epochnest < UCHAR_MAX - 2); - ck_epoch_begin(&eps->eps_record.er_record, NULL); -} - -void epoch_exit(epoch_t epoch) { struct epoch_pcpu_state *eps; @@ -331,6 +321,7 @@ epoch_exit(epoch_t epoch) td = curthread; INIT_CHECK(epoch); + MPASS(td->td_epochnest); critical_enter(); eps = epoch->e_pcpu[curcpu]; sched_unpin(); @@ -342,19 +333,6 @@ epoch_exit(epoch_t epoch) critical_exit(); } -void -epoch_exit_nopreempt(epoch_t epoch) -{ - struct epoch_pcpu_state *eps; - - INIT_CHECK(epoch); - MPASS(curthread->td_critnest); - eps = epoch->e_pcpu[curcpu]; - ck_epoch_end(&eps->eps_record.er_record, NULL); - curthread->td_epochnest--; - critical_exit(); -} - /* * epoch_block_handler is a callback from the ck code when another thread is * currently in an epoch section. @@ -541,12 +519,17 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (* cb = (void *)ctx; + MPASS(callback); + /* too early in boot to have epoch set up */ + if (__predict_false(epoch == NULL)) { + callback(ctx); + return; + } MPASS(cb->ec_callback == NULL); MPASS(cb->ec_link.stqe_next == NULL); - MPASS(epoch); - MPASS(callback); cb->ec_callback = callback; counter_u64_add(epoch->e_frees, 1); + critical_enter(); eps = epoch->e_pcpu[curcpu]; STAILQ_INSERT_HEAD(&eps->eps_cblist, cb, ec_link); Modified: head/sys/sys/epoch.h ============================================================================== --- head/sys/sys/epoch.h Sun May 13 23:16:04 2018 (r333590) +++ head/sys/sys/epoch.h Sun May 13 23:24:48 2018 (r333591) @@ -1,27 +1,28 @@ /*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Neither the name of Matthew Macy nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. * * $FreeBSD$ */ @@ -32,6 +33,8 @@ struct epoch; typedef struct epoch *epoch_t; +extern epoch_t global_epoch; + struct epoch_context { void *data[2]; } __aligned(sizeof(void *)); @@ -42,8 +45,6 @@ epoch_t epoch_alloc(void); void epoch_free(epoch_t epoch); void epoch_enter(epoch_t epoch); void epoch_exit(epoch_t epoch); -void epoch_enter_nopreempt(epoch_t epoch); -void epoch_exit_nopreempt(epoch_t epoch); void epoch_wait(epoch_t epoch); void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)); int in_epoch(void);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201805132324.w4DNOm3B097320>