From owner-dev-commits-src-branches@freebsd.org Wed Aug 11 15:22:44 2021 Return-Path: Delivered-To: dev-commits-src-branches@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 215B8656065; Wed, 11 Aug 2021 15:22:44 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4GlD8g5ddZz3LMg; Wed, 11 Aug 2021 15:22:43 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id A1DEC11A5F; Wed, 11 Aug 2021 15:22:43 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 17BFMhl3064298; Wed, 11 Aug 2021 15:22:43 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 17BFMhuj064297; Wed, 11 Aug 2021 15:22:43 GMT (envelope-from git) Date: Wed, 11 Aug 2021 15:22:43 GMT Message-Id: <202108111522.17BFMhuj064297@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mateusz Guzik Subject: git: fff80a6992f1 - stable/13 - pf: add hybrid 32- an 64- bit counters MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: mjg X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: fff80a6992f1ba13f2261b066d70a3a97d8837e4 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-branches@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commits to the stable branches of the FreeBSD src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 11 Aug 2021 15:22:44 -0000 The branch stable/13 has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=fff80a6992f1ba13f2261b066d70a3a97d8837e4 commit fff80a6992f1ba13f2261b066d70a3a97d8837e4 Author: Mateusz Guzik AuthorDate: 2021-07-22 20:47:24 +0000 Commit: Mateusz Guzik CommitDate: 2021-08-11 13:37:53 +0000 pf: add hybrid 32- an 64- bit counters Numerous counters got migrated from straight uint64_t to the counter(9) API. Unfortunately the implementation comes with a significiant performance hit on some platforms and cannot be easily fixed. Work around the problem by implementing a pf-specific variant. Reviewed by: kp Sponsored by: Rubicon Communications, LLC ("Netgate") (cherry picked from commit defdcdd5648dc1ea789bc54bb45108fcab546a6b) --- sys/net/pfvar.h | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++ sys/netpfil/pf/pf.c | 28 ++++++- 2 files changed, 245 insertions(+), 1 deletion(-) diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index 55ecbecf657a..e381279a113f 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -43,9 +43,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -64,6 +66,222 @@ #ifdef _KERNEL +#if defined(__arm__) +#define PF_WANT_32_TO_64_COUNTER +#endif + +/* + * A hybrid of 32-bit and 64-bit counters which can be used on platforms where + * counter(9) is very expensive. + * + * As 32-bit counters are expected to overflow, a periodic job sums them up to + * a saved 64-bit state. Fetching the value still walks all CPUs to get the most + * current snapshot. + */ +#ifdef PF_WANT_32_TO_64_COUNTER +struct pf_counter_u64_pcpu { + u_int32_t current; + u_int32_t snapshot; +}; + +struct pf_counter_u64 { + struct pf_counter_u64_pcpu *pfcu64_pcpu; + u_int64_t pfcu64_value; + seqc_t pfcu64_seqc; +}; + +static inline int +pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags) +{ + + pfcu64->pfcu64_value = 0; + pfcu64->pfcu64_seqc = 0; + pfcu64->pfcu64_pcpu = uma_zalloc_pcpu(pcpu_zone_8, flags | M_ZERO); + if (__predict_false(pfcu64->pfcu64_pcpu == NULL)) + return (ENOMEM); + return (0); +} + +static inline void +pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64) +{ + + uma_zfree_pcpu(pcpu_zone_8, pfcu64->pfcu64_pcpu); +} + +static inline void +pf_counter_u64_critical_enter(void) +{ + + critical_enter(); +} + +static inline void +pf_counter_u64_critical_exit(void) +{ + + critical_exit(); +} + +static inline void +pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int32_t val; + + MPASS(curthread->td_critnest > 0); + pcpu = zpcpu_get(pfcu64->pfcu64_pcpu); + val = atomic_load_int(&pcpu->current); + atomic_store_int(&pcpu->current, val + n); +} + +static inline void +pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + critical_enter(); + pf_counter_u64_add_protected(pfcu64, n); + critical_exit(); +} + +static inline u_int64_t +pf_counter_u64_periodic(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int64_t sum; + u_int32_t val; + int cpu; + + MPASS(curthread->td_critnest > 0); + seqc_write_begin(&pfcu64->pfcu64_seqc); + sum = pfcu64->pfcu64_value; + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + val = atomic_load_int(&pcpu->current); + sum += (uint32_t)(val - pcpu->snapshot); + pcpu->snapshot = val; + } + pfcu64->pfcu64_value = sum; + seqc_write_end(&pfcu64->pfcu64_seqc); + return (sum); +} + +static inline u_int64_t +pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int64_t sum; + seqc_t seqc; + int cpu; + + for (;;) { + seqc = seqc_read(&pfcu64->pfcu64_seqc); + sum = 0; + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + sum += (uint32_t)(atomic_load_int(&pcpu->current) -pcpu->snapshot); + } + sum += pfcu64->pfcu64_value; + if (seqc_consistent(&pfcu64->pfcu64_seqc, seqc)) + break; + } + return (sum); +} + +static inline void +pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + int cpu; + + MPASS(curthread->td_critnest > 0); + seqc_write_begin(&pfcu64->pfcu64_seqc); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + pcpu->snapshot = atomic_load_int(&pcpu->current); + } + pfcu64->pfcu64_value = 0; + seqc_write_end(&pfcu64->pfcu64_seqc); +} + +static inline void +pf_counter_u64_zero(struct pf_counter_u64 *pfcu64) +{ + + critical_enter(); + pf_counter_u64_zero_protected(pfcu64); + critical_exit(); +} +#else +struct pf_counter_u64 { + counter_u64_t counter; +}; + +static inline int +pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags) +{ + + pfcu64->counter = counter_u64_alloc(flags); + if (__predict_false(pfcu64->counter == NULL)) + return (ENOMEM); + return (0); +} + +static inline void +pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64) +{ + + counter_u64_free(pfcu64->counter); +} + +static inline void +pf_counter_u64_critical_enter(void) +{ + +} + +static inline void +pf_counter_u64_critical_exit(void) +{ + +} + +static inline void +pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + counter_u64_add(pfcu64->counter, n); +} + +static inline void +pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + pf_counter_u64_add_protected(pfcu64, n); +} + +static inline u_int64_t +pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64) +{ + + return (counter_u64_fetch(pfcu64->counter)); +} + +static inline void +pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64) +{ + + counter_u64_zero(pfcu64->counter); +} + +static inline void +pf_counter_u64_zero(struct pf_counter_u64 *pfcu64) +{ + + pf_counter_u64_zero_protected(pfcu64); +} +#endif + SYSCTL_DECL(_net_pf); MALLOC_DECLARE(M_PFHASH); diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 0db9eaed1ac4..b74491bb88d0 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -154,6 +154,11 @@ VNET_DECLARE(int, pf_vnet_active); VNET_DEFINE_STATIC(uint32_t, pf_purge_idx); #define V_pf_purge_idx VNET(pf_purge_idx) +#ifdef PF_WANT_32_TO_64_COUNTER +VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter); +#define V_pf_counter_periodic_iter VNET(pf_counter_periodic_iter) +#endif + /* * Queue for pf_intr() sends. */ @@ -1509,6 +1514,25 @@ pf_intr(void *v) CURVNET_RESTORE(); } +#define pf_purge_thread_period (hz / 10) + +#ifdef PF_WANT_32_TO_64_COUNTER +static void +pf_counter_u64_periodic_main(void) +{ + PF_RULES_RLOCK_TRACKER; + + V_pf_counter_periodic_iter++; + + PF_RULES_RLOCK(); + pf_counter_u64_critical_enter(); + pf_counter_u64_critical_exit(); + PF_RULES_RUNLOCK(); +} +#else +#define pf_counter_u64_periodic_main() do { } while (0) +#endif + void pf_purge_thread(void *unused __unused) { @@ -1516,7 +1540,7 @@ pf_purge_thread(void *unused __unused) sx_xlock(&pf_end_lock); while (pf_end_threads == 0) { - sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", hz / 10); + sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period); VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { @@ -1528,6 +1552,8 @@ pf_purge_thread(void *unused __unused) continue; } + pf_counter_u64_periodic_main(); + /* * Process 1/interval fraction of the state * table every run.