Date: Mon, 15 Apr 2013 20:11:00 +0400 From: "Alexander V. Chernikov" <melifaro@FreeBSD.org> To: Gleb Smirnoff <glebius@FreeBSD.org> Cc: svn-src-head@freebsd.org, svn-src-all@freebsd.org, src-committers@freebsd.org Subject: Re: svn commit: r249506 - head/sys/net Message-ID: <516C2694.9090906@FreeBSD.org> In-Reply-To: <201304151300.r3FD0gDV021140@svn.freebsd.org> References: <201304151300.r3FD0gDV021140@svn.freebsd.org>
next in thread | previous in thread | raw e-mail | index | archive | help
This is a multi-part message in MIME format. --------------020404010101070001050708 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit On 15.04.2013 17:00, Gleb Smirnoff wrote: > Author: glebius > Date: Mon Apr 15 13:00:42 2013 > New Revision: 249506 > URL: http://svnweb.freebsd.org/changeset/base/249506 > > Log: > Switch lagg(4) statistics to counter(9). > > The lagg(4) is often used to bond high speed links, so basic per-packet += > on statistics cause cache misses and statistics loss. Lagg is probably the only interface where we really do not need to account (most) per-packet traffic. It is probably better to import statistics from underlying interfaces with callout. Additionally, this approach allows us not to use lagg RX path at all skipping lock (reference patch to ixgbe in the -net@ follows). > > Perfect solution would be to convert ifnet(9) to counters(9), but this > requires much more work, and unfortunately ABI change, so temporarily > patch lagg(4) manually. > > We store counters in the softc, and once per second push their values > to legacy ifnet counters. > > Sponsored by: Nginx, Inc. > > Modified: > head/sys/net/if_lagg.c > head/sys/net/if_lagg.h > > Modified: head/sys/net/if_lagg.c > ============================================================================== > --- head/sys/net/if_lagg.c Mon Apr 15 12:16:24 2013 (r249505) > +++ head/sys/net/if_lagg.c Mon Apr 15 13:00:42 2013 (r249506) > @@ -153,6 +153,8 @@ static struct mbuf *lagg_lacp_input(stru > struct mbuf *); > static void lagg_lacp_lladdr(struct lagg_softc *); > > +static void lagg_callout(void *); > + > /* lagg protocol table */ > static const struct { > int ti_proto; > @@ -278,6 +280,11 @@ lagg_clone_create(struct if_clone *ifc, > return (ENOSPC); > } > > + sc->sc_ipackets = counter_u64_alloc(M_WAITOK); > + sc->sc_opackets = counter_u64_alloc(M_WAITOK); > + sc->sc_ibytes = counter_u64_alloc(M_WAITOK); > + sc->sc_obytes = counter_u64_alloc(M_WAITOK); > + > sysctl_ctx_init(&sc->ctx); > snprintf(num, sizeof(num), "%u", unit); > sc->use_flowid = def_use_flowid; > @@ -307,6 +314,7 @@ lagg_clone_create(struct if_clone *ifc, > LAGG_LOCK_INIT(sc); > SLIST_INIT(&sc->sc_ports); > TASK_INIT(&sc->sc_lladdr_task, 0, lagg_port_setlladdr, sc); > + callout_init_rw(&sc->sc_callout, &sc->sc_mtx, CALLOUT_SHAREDLOCK); > > /* Initialise pseudo media types */ > ifmedia_init(&sc->sc_media, 0, lagg_media_change, > @@ -338,6 +346,8 @@ lagg_clone_create(struct if_clone *ifc, > SLIST_INSERT_HEAD(&lagg_list, sc, sc_entries); > mtx_unlock(&lagg_list_mtx); > > + callout_reset(&sc->sc_callout, hz, lagg_callout, sc); > + > return (0); > } > > @@ -369,6 +379,12 @@ lagg_clone_destroy(struct ifnet *ifp) > ether_ifdetach(ifp); > if_free(ifp); > > + callout_drain(&sc->sc_callout); > + counter_u64_free(sc->sc_ipackets); > + counter_u64_free(sc->sc_opackets); > + counter_u64_free(sc->sc_ibytes); > + counter_u64_free(sc->sc_obytes); > + > mtx_lock(&lagg_list_mtx); > SLIST_REMOVE(&lagg_list, sc, lagg_softc, sc_entries); > mtx_unlock(&lagg_list_mtx); > @@ -1243,9 +1259,9 @@ lagg_transmit(struct ifnet *ifp, struct > LAGG_RUNLOCK(sc); > > if (error == 0) { > - ifp->if_opackets++; > + counter_u64_add(sc->sc_opackets, 1); > + counter_u64_add(sc->sc_obytes, len); > ifp->if_omcasts += mcast; > - ifp->if_obytes += len; > } else > ifp->if_oerrors++; > > @@ -1281,8 +1297,8 @@ lagg_input(struct ifnet *ifp, struct mbu > m = (*sc->sc_input)(sc, lp, m); > > if (m != NULL) { > - scifp->if_ipackets++; > - scifp->if_ibytes += m->m_pkthdr.len; > + counter_u64_add(sc->sc_ipackets, 1); > + counter_u64_add(sc->sc_ibytes, m->m_pkthdr.len); > > if (scifp->if_flags & IFF_MONITOR) { > m_freem(m); > @@ -1892,3 +1908,17 @@ lagg_lacp_input(struct lagg_softc *sc, s > m->m_pkthdr.rcvif = ifp; > return (m); > } > + > +static void > +lagg_callout(void *arg) > +{ > + struct lagg_softc *sc = (struct lagg_softc *)arg; > + struct ifnet *ifp = sc->sc_ifp; > + > + ifp->if_ipackets = counter_u64_fetch(sc->sc_ipackets); > + ifp->if_opackets = counter_u64_fetch(sc->sc_opackets); > + ifp->if_ibytes = counter_u64_fetch(sc->sc_ibytes); > + ifp->if_obytes = counter_u64_fetch(sc->sc_obytes); > + > + callout_reset(&sc->sc_callout, hz, lagg_callout, sc); > +} > > Modified: head/sys/net/if_lagg.h > ============================================================================== > --- head/sys/net/if_lagg.h Mon Apr 15 12:16:24 2013 (r249505) > +++ head/sys/net/if_lagg.h Mon Apr 15 13:00:42 2013 (r249506) > @@ -21,8 +21,6 @@ > #ifndef _NET_LAGG_H > #define _NET_LAGG_H > > -#include <sys/sysctl.h> > - > /* > * Global definitions > */ > @@ -137,6 +135,9 @@ struct lagg_reqflags { > #define SIOCSLAGGHASH _IOW('i', 146, struct lagg_reqflags) > > #ifdef _KERNEL > + > +#include <sys/counter.h> > + > /* > * Internal kernel part > */ > @@ -195,6 +196,11 @@ struct lagg_softc { > uint32_t sc_seq; /* sequence counter */ > uint32_t sc_flags; > > + counter_u64_t sc_ipackets; > + counter_u64_t sc_opackets; > + counter_u64_t sc_ibytes; > + counter_u64_t sc_obytes; > + > SLIST_HEAD(__tplhd, lagg_port) sc_ports; /* list of interfaces */ > SLIST_ENTRY(lagg_softc) sc_entries; > > @@ -217,6 +223,7 @@ struct lagg_softc { > void (*sc_portreq)(struct lagg_port *, caddr_t); > eventhandler_tag vlan_attach; > eventhandler_tag vlan_detach; > + struct callout sc_callout; > struct sysctl_ctx_list ctx; /* sysctl variables */ > int use_flowid; /* use M_FLOWID */ > }; > -- WBR, Alexander --------------020404010101070001050708 Content-Type: text/plain; charset=UTF-8; name="if_lagg.diff" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="if_lagg.diff" SW5kZXg6IHN5cy9uZXQvaWZfbGFnZy5jCj09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KLS0tIHN5cy9uZXQvaWZf bGFnZy5jCShyZXZpc2lvbiAyNDg3MDQpCisrKyBzeXMvbmV0L2lmX2xhZ2cuYwkod29ya2lu ZyBjb3B5KQpAQCAtMjYxLDYgKzI2MSw0MSBAQCBsYWdnX3VucmVnaXN0ZXJfdmxhbih2b2lk ICphcmcsIHN0cnVjdCBpZm5ldCAqaWZwLAogICAgICAgICBMQUdHX1JVTkxPQ0soc2MpOwog fQogCisjZGVmaW5lCUxBR0dfVVBEQVRFX0NPVU5URVIoX25hbWUpCWRvIHsJCVwKKwlpZiAo cF9pZnAtPmlmX2RhdGEuX25hbWUgPj0gbHAtPl9uYW1lKSB7CVwKKwkJaSA9IHBfaWZwLT5p Zl9kYXRhLl9uYW1lIC0gbHAtPl9uYW1lOwlcCisJCWxhZ2dfaWZwLT5pZl9kYXRhLl9uYW1l ICs9IGk7CQlcCisJCWxwLT5fbmFtZSArPSBpOwkJCQlcCisJfSBlbHNlIAkJCQkJCVwKKwkJ bHAtPl9uYW1lID0gcF9pZnAtPmlmX2RhdGEuX25hbWU7CVwKK30gd2hpbGUgKDApCisKK3N0 YXRpYyB2b2lkCitsYWdnX3VwZGF0ZV9jb3VudGVycyh2b2lkICphcmcpCit7CisJc3RydWN0 IGxhZ2dfc29mdGMJKnNjID0gYXJnOworICAgICAgICBzdHJ1Y3QgbGFnZ19wb3J0ICAgICAg ICAqbHA7CisJc3RydWN0IGlmbmV0CQkqbGFnZ19pZnAsICpwX2lmcDsKKwl1X2xvbmcgaTsK KworCWxhZ2dfaWZwID0gc2MtPnNjX2lmcDsKKworCUxBR0dfUkxPQ0soc2MpOworCisJU0xJ U1RfRk9SRUFDSChscCwgJnNjLT5zY19wb3J0cywgbHBfZW50cmllcykgeworCQlwX2lmcCA9 IGxwLT5scF9pZnA7CisKKwkJTEFHR19VUERBVEVfQ09VTlRFUihpZmlfaXBhY2tldHMpOwor CQlMQUdHX1VQREFURV9DT1VOVEVSKGlmaV9vcGFja2V0cyk7CisJCUxBR0dfVVBEQVRFX0NP VU5URVIoaWZpX2lieXRlcyk7CisJCUxBR0dfVVBEQVRFX0NPVU5URVIoaWZpX29ieXRlcyk7 CisJfQorCisJTEFHR19SVU5MT0NLKHNjKTsKKworCWNhbGxvdXRfcmVzZXQoJnNjLT5jb3Vu dGVyc19zeW5jLCBoeiwgbGFnZ191cGRhdGVfY291bnRlcnMsIHNjKTsKK30KKwogc3RhdGlj IGludAogbGFnZ19jbG9uZV9jcmVhdGUoc3RydWN0IGlmX2Nsb25lICppZmMsIGludCB1bml0 LCBjYWRkcl90IHBhcmFtcykKIHsKQEAgLTMwNyw2ICszNDIsNyBAQCBsYWdnX2Nsb25lX2Ny ZWF0ZShzdHJ1Y3QgaWZfY2xvbmUgKmlmYywgaW50IHVuaXQsCiAJTEFHR19MT0NLX0lOSVQo c2MpOwogCVNMSVNUX0lOSVQoJnNjLT5zY19wb3J0cyk7CiAJVEFTS19JTklUKCZzYy0+c2Nf bGxhZGRyX3Rhc2ssIDAsIGxhZ2dfcG9ydF9zZXRsbGFkZHIsIHNjKTsKKwljYWxsb3V0X2lu aXQoJnNjLT5jb3VudGVyc19zeW5jLCAwKTsKIAogCS8qIEluaXRpYWxpc2UgcHNldWRvIG1l ZGlhIHR5cGVzICovCiAJaWZtZWRpYV9pbml0KCZzYy0+c2NfbWVkaWEsIDAsIGxhZ2dfbWVk aWFfY2hhbmdlLApAQCAtMzM4LDYgKzM3NCw4IEBAIGxhZ2dfY2xvbmVfY3JlYXRlKHN0cnVj dCBpZl9jbG9uZSAqaWZjLCBpbnQgdW5pdCwKIAlTTElTVF9JTlNFUlRfSEVBRCgmbGFnZ19s aXN0LCBzYywgc2NfZW50cmllcyk7CiAJbXR4X3VubG9jaygmbGFnZ19saXN0X210eCk7CiAK KwljYWxsb3V0X3Jlc2V0KCZzYy0+Y291bnRlcnNfc3luYywgaHogLyAxMCwgbGFnZ191cGRh dGVfY291bnRlcnMsIHNjKTsKKwogCXJldHVybiAoMCk7CiB9CiAKQEAgLTM0Nyw2ICszODUs OSBAQCBsYWdnX2Nsb25lX2Rlc3Ryb3koc3RydWN0IGlmbmV0ICppZnApCiAJc3RydWN0IGxh Z2dfc29mdGMgKnNjID0gKHN0cnVjdCBsYWdnX3NvZnRjICopaWZwLT5pZl9zb2Z0YzsKIAlz dHJ1Y3QgbGFnZ19wb3J0ICpscDsKIAorCS8qIFN0b3AgY29sbGVjdGluZyBjb3VudGVycyAq LworCWNhbGxvdXRfZHJhaW4oJnNjLT5jb3VudGVyc19zeW5jKTsKKwogCUxBR0dfV0xPQ0so c2MpOwogCiAJbGFnZ19zdG9wKHNjKTsKQEAgLTU3MSw2ICs2MTIsMTIgQEAgbGFnZ19wb3J0 X2NyZWF0ZShzdHJ1Y3QgbGFnZ19zb2Z0YyAqc2MsIHN0cnVjdCBpZm4KIAlscC0+bHBfaWZw ID0gaWZwOwogCWxwLT5scF9zb2Z0YyA9IHNjOwogCisJLyogU2V0IGNvdW50ZXJzICovCisJ bHAtPmlmaV9pYnl0ZXMgPSBpZnAtPmlmX2lieXRlczsKKwlscC0+aWZpX29ieXRlcyA9IGlm cC0+aWZfb2J5dGVzOworCWxwLT5pZmlfaXBhY2tldHMgPSBpZnAtPmlmX2lwYWNrZXRzOwor CWxwLT5pZmlfb3BhY2tldHMgPSBpZnAtPmlmX29wYWNrZXRzOworCiAJLyogU2F2ZSBwb3J0 IGxpbmsgbGF5ZXIgYWRkcmVzcyAqLwogCWJjb3B5KElGX0xMQUREUihpZnApLCBscC0+bHBf bGxhZGRyLCBFVEhFUl9BRERSX0xFTik7CiAKQEAgLTEyNDMsOSArMTI5MCw3IEBAIGxhZ2df dHJhbnNtaXQoc3RydWN0IGlmbmV0ICppZnAsIHN0cnVjdCBtYnVmICptKQogCUxBR0dfUlVO TE9DSyhzYyk7CiAKIAlpZiAoZXJyb3IgPT0gMCkgewotCQlpZnAtPmlmX29wYWNrZXRzKys7 CiAJCWlmcC0+aWZfb21jYXN0cyArPSBtY2FzdDsKLQkJaWZwLT5pZl9vYnl0ZXMgKz0gbGVu OwogCX0gZWxzZQogCQlpZnAtPmlmX29lcnJvcnMrKzsKIApAQCAtMTI4MSw5ICsxMzI2LDYg QEAgbGFnZ19pbnB1dChzdHJ1Y3QgaWZuZXQgKmlmcCwgc3RydWN0IG1idWYgKm0pCiAJbSA9 ICgqc2MtPnNjX2lucHV0KShzYywgbHAsIG0pOwogCiAJaWYgKG0gIT0gTlVMTCkgewotCQlz Y2lmcC0+aWZfaXBhY2tldHMrKzsKLQkJc2NpZnAtPmlmX2lieXRlcyArPSBtLT5tX3BrdGhk ci5sZW47Ci0KIAkJaWYgKHNjaWZwLT5pZl9mbGFncyAmIElGRl9NT05JVE9SKSB7CiAJCQlt X2ZyZWVtKG0pOwogCQkJbSA9IE5VTEw7CkluZGV4OiBzeXMvbmV0L2lmX2xhZ2cuaAo9PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09Ci0tLSBzeXMvbmV0L2lmX2xhZ2cuaAkocmV2aXNpb24gMjQ4NzA0KQorKysg c3lzL25ldC9pZl9sYWdnLmgJKHdvcmtpbmcgY29weSkKQEAgLTIxOSw2ICsyMTksNyBAQCBz dHJ1Y3QgbGFnZ19zb2Z0YyB7CiAJZXZlbnRoYW5kbGVyX3RhZyB2bGFuX2RldGFjaDsKIAlz dHJ1Y3Qgc3lzY3RsX2N0eF9saXN0CQljdHg7CQkvKiBzeXNjdGwgdmFyaWFibGVzICovCiAJ aW50CQkJCXVzZV9mbG93aWQ7CS8qIHVzZSBNX0ZMT1dJRCAqLworCXN0cnVjdCBjYWxsb3V0 CQkJY291bnRlcnNfc3luYzsJLyogY291bnRlcnMgcGVyaW9kaWMgKi8KIH07CiAKIHN0cnVj dCBsYWdnX3BvcnQgewpAQCAtMjQxLDYgKzI0MiwxMSBAQCBzdHJ1Y3QgbGFnZ19wb3J0IHsK IAlpbnQJKCpscF9vdXRwdXQpKHN0cnVjdCBpZm5ldCAqLCBzdHJ1Y3QgbWJ1ZiAqLCBzdHJ1 Y3Qgc29ja2FkZHIgKiwKIAkJICAgICBzdHJ1Y3Qgcm91dGUgKik7CiAKKwl1X2xvbmcJCQkJ aWZpX2lwYWNrZXRzOwkvKiBwYWNrZXRzIHJlY2VpdmVkIG9uIGludGVyZmFjZSAqLworCXVf bG9uZwkJCQlpZmlfb3BhY2tldHM7CS8qIHBhY2tldHMgc2VudCBvbiBpbnRlcmZhY2UgKi8K Kwl1X2xvbmcJCQkJaWZpX2lieXRlczsJLyogdG90YWwgbnVtYmVyIG9mIG9jdGV0cyByZWNl aXZlZCAqLworCXVfbG9uZwkJCQlpZmlfb2J5dGVzOwkvKiB0b3RhbCBudW1iZXIgb2Ygb2N0 ZXRzIHNlbnQgKi8KKwogCVNMSVNUX0VOVFJZKGxhZ2dfcG9ydCkJCWxwX2VudHJpZXM7CiB9 OwogCkluZGV4OiBzeXMvbmV0L2lmX3ZsYW4uYwo9PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09Ci0tLSBzeXMvbmV0 L2lmX3ZsYW4uYwkocmV2aXNpb24gMjQ4NzA0KQorKysgc3lzL25ldC9pZl92bGFuLmMJKHdv cmtpbmcgY29weSkKQEAgLTEzOCw2ICsxMzgsMTIgQEAgc3RhdGljIGludCBzb2Z0X3BhZCA9 IDA7CiBTWVNDVExfSU5UKF9uZXRfbGlua192bGFuLCBPSURfQVVUTywgc29mdF9wYWQsIENU TEZMQUdfUlcsICZzb2Z0X3BhZCwgMCwKIAkgICAicGFkIHNob3J0IGZyYW1lcyBiZWZvcmUg dGFnZ2luZyIpOwogCitzdGF0aWMgaW50IHN5c2N0bF9wcmlvcml0eV92YWwoU1lTQ1RMX0hB TkRMRVJfQVJHUyk7CitzdGF0aWMgaW50IHByaW9yaXR5ID0gaHRvbnMoMSA8PCAxMyk7CitT WVNDVExfVk5FVF9QUk9DKF9uZXRfbGlua192bGFuLCBPSURfQVVUTywgZG90MXBfcHJpb3Jp dHksCisgICAgQ1RMVFlQRV9VSU5UfENUTEZMQUdfUlcsIDAsIDAsIHN5c2N0bF9wcmlvcml0 eV92YWwsICJJVSIsCisJICAgInNldCBkb3QxcCBwcmlvcml0eSIpOworCiBzdGF0aWMgY29u c3QgY2hhciB2bGFubmFtZVtdID0gInZsYW4iOwogc3RhdGljIE1BTExPQ19ERUZJTkUoTV9W TEFOLCB2bGFubmFtZSwgIjgwMi4xUSBWaXJ0dWFsIExBTiBJbnRlcmZhY2UiKTsKIApAQCAt MjE3LDYgKzIyMywyNyBAQCBzdGF0aWMgVk5FVF9ERUZJTkUoc3RydWN0IGlmX2Nsb25lICos IHZsYW5fY2xvbmVyKQogI2lmbmRlZiBWTEFOX0FSUkFZCiAjZGVmaW5lIEhBU0gobiwgbSkJ KCgoKG4pID4+IDgpIF4gKChuKSA+PiA0KSBeIChuKSkgJiAobSkpCiAKKy8qCisgKiBTZXQg bWF4aW11bSBudW1iZXIgb2YgdGFibGVzIHRoYXQgY2FuIGJlIHVzZWQgaW4gZ2l2ZW4gVk5F VCBpcGZ3IGluc3RhbmNlLgorICovCitzdGF0aWMgaW50CitzeXNjdGxfcHJpb3JpdHlfdmFs KFNZU0NUTF9IQU5ETEVSX0FSR1MpCit7CisJaW50IGVycm9yOworCXVuc2lnbmVkIGludCBu cHJpbzsKKworCW5wcmlvID0gbnRvaHMocHJpb3JpdHkpID4+IDEzOworCisJZXJyb3IgPSBz eXNjdGxfaGFuZGxlX2ludChvaWRwLCAmbnByaW8sIDAsIHJlcSk7CisJLyogUmVhZCBvcGVy YXRpb24gb3Igc29tZSBlcnJvciAqLworCWlmICgoZXJyb3IgIT0gMCkgfHwgKHJlcS0+bmV3 cHRyID09IE5VTEwpKQorCQlyZXR1cm4gKGVycm9yKTsKKwkKKwlwcmlvcml0eSA9IGh0b25z KChucHJpbyAmIDB4NykgPDwgMTMpOworCisJcmV0dXJuICgwKTsKK30KKwogc3RhdGljIHZv aWQKIHZsYW5faW5pdGhhc2goc3RydWN0IGlmdmxhbnRydW5rICp0cnVuaykKIHsKQEAgLTEw NzYsNyArMTEwMyw3IEBAIHZsYW5fdHJhbnNtaXQoc3RydWN0IGlmbmV0ICppZnAsIHN0cnVj dCBtYnVmICptKQogCSAqIHBhY2tldCB0YWcgdGhhdCBob2xkcyBpdC4KIAkgKi8KIAlpZiAo cC0+aWZfY2FwZW5hYmxlICYgSUZDQVBfVkxBTl9IV1RBR0dJTkcpIHsKLQkJbS0+bV9wa3Ro ZHIuZXRoZXJfdnRhZyA9IGlmdi0+aWZ2X3ZpZDsKKwkJbS0+bV9wa3RoZHIuZXRoZXJfdnRh ZyA9IGlmdi0+aWZ2X3ZpZCB8IHByaW9yaXR5OwogCQltLT5tX2ZsYWdzIHw9IE1fVkxBTlRB RzsKIAl9IGVsc2UgewogCQltID0gZXRoZXJfdmxhbmVuY2FwKG0sIGlmdi0+aWZ2X3ZpZCk7 Cg== --------------020404010101070001050708--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?516C2694.9090906>