Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 7 Oct 2019 22:40:06 +0000 (UTC)
From:      Gleb Smirnoff <glebius@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r353292 - in head/sys: contrib/ipfilter/netinet dev/firewire dev/iicbus dev/usb/net kern net netgraph netinet netinet6 netipsec netpfil/ipfw netpfil/pf ofed/drivers/infiniband/ulp/ipoib
Message-ID:  <201910072240.x97Me60x065650@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: glebius
Date: Mon Oct  7 22:40:05 2019
New Revision: 353292
URL: https://svnweb.freebsd.org/changeset/base/353292

Log:
  Widen NET_EPOCH coverage.
  
  When epoch(9) was introduced to network stack, it was basically
  dropped in place of existing locking, which was mutexes and
  rwlocks. For the sake of performance mutex covered areas were
  as small as possible, so became epoch covered areas.
  
  However, epoch doesn't introduce any contention, it just delays
  memory reclaim. So, there is no point to minimise epoch covered
  areas in sense of performance. Meanwhile entering/exiting epoch
  also has non-zero CPU usage, so doing this less often is a win.
  
  Not the least is also code maintainability. In the new paradigm
  we can assume that at any stage of processing a packet, we are
  inside network epoch. This makes coding both input and output
  path way easier.
  
  On output path we already enter epoch quite early - in the
  ip_output(), in the ip6_output().
  
  This patch does the same for the input path. All ISR processing,
  network related callouts, other ways of packet injection to the
  network stack shall be performed in net_epoch. Any leaf function
  that walks network configuration now asserts epoch.
  
  Tricky part is configuration code paths - ioctls, sysctls. They
  also call into leaf functions, so some need to be changed.
  
  This patch would introduce more epoch recursions (see EPOCH_TRACE)
  than we had before. They will be cleaned up separately, as several
  of them aren't trivial. Note, that unlike a lock recursion the
  epoch recursion is safe and just wastes a bit of resources.
  
  Reviewed by:	gallatin, hselasky, cy, adrian, kristof
  Differential Revision:	https://reviews.freebsd.org/D19111

Modified:
  head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
  head/sys/dev/firewire/if_fwip.c
  head/sys/dev/iicbus/if_ic.c
  head/sys/dev/usb/net/if_usie.c
  head/sys/dev/usb/net/uhso.c
  head/sys/dev/usb/net/usb_ethernet.c
  head/sys/kern/uipc_socket.c
  head/sys/net/if.c
  head/sys/net/if_ethersubr.c
  head/sys/net/if_gif.c
  head/sys/net/if_me.c
  head/sys/net/if_stf.c
  head/sys/net/if_tuntap.c
  head/sys/net/if_vlan.c
  head/sys/net/netisr.c
  head/sys/net/route.c
  head/sys/net/rtsock.c
  head/sys/netgraph/ng_ether.c
  head/sys/netgraph/ng_iface.c
  head/sys/netgraph/ng_ip_input.c
  head/sys/netinet/if_ether.c
  head/sys/netinet/igmp.c
  head/sys/netinet/in.c
  head/sys/netinet/in_mcast.c
  head/sys/netinet/in_rmx.c
  head/sys/netinet/in_var.h
  head/sys/netinet/ip_carp.c
  head/sys/netinet/ip_encap.c
  head/sys/netinet/ip_icmp.c
  head/sys/netinet/ip_input.c
  head/sys/netinet/ip_mroute.c
  head/sys/netinet/ip_options.c
  head/sys/netinet/ip_output.c
  head/sys/netinet/sctp_bsd_addr.c
  head/sys/netinet6/icmp6.c
  head/sys/netinet6/in6.c
  head/sys/netinet6/in6_ifattach.c
  head/sys/netinet6/in6_mcast.c
  head/sys/netinet6/in6_var.h
  head/sys/netinet6/ip6_output.c
  head/sys/netinet6/mld6.c
  head/sys/netinet6/nd6.c
  head/sys/netinet6/nd6_nbr.c
  head/sys/netinet6/nd6_rtr.c
  head/sys/netinet6/raw_ip6.c
  head/sys/netipsec/xform_ipcomp.c
  head/sys/netpfil/ipfw/ip_dn_io.c
  head/sys/netpfil/pf/pf_if.c
  head/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c

Modified: head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
==============================================================================
--- head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -1312,8 +1312,10 @@ ipf_inject(fin, m)
 	fr_info_t *fin;
 	mb_t *m;
 {
+	struct epoch_tracker et;
 	int error = 0;
 
+	NET_EPOCH_ENTER(et);
 	if (fin->fin_out == 0) {
 		netisr_dispatch(NETISR_IP, m);
 	} else {
@@ -1321,6 +1323,7 @@ ipf_inject(fin, m)
 		fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off);
 		error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL);
 	}
+	NET_EPOCH_EXIT(et);
 
 	return error;
 }

Modified: head/sys/dev/firewire/if_fwip.c
==============================================================================
--- head/sys/dev/firewire/if_fwip.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/dev/firewire/if_fwip.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count)
 static void
 fwip_stream_input(struct fw_xferq *xferq)
 {
+	struct epoch_tracker et;
 	struct mbuf *m, *m0;
 	struct m_tag *mtag;
 	struct ifnet *ifp;
@@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq)
 	fwip = (struct fwip_softc *)xferq->sc;
 	ifp = fwip->fw_softc.fwip_ifp;
 
+	NET_EPOCH_ENTER(et);
 	while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) {
 		STAILQ_REMOVE_HEAD(&xferq->stvalid, link);
 		fp = mtod(sxfer->mbuf, struct fw_pkt *);
@@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq)
 		firewire_input(ifp, m, src);
 		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 	}
+	NET_EPOCH_EXIT(et);
 	if (STAILQ_FIRST(&xferq->stfree) != NULL)
 		fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch);
 }

Modified: head/sys/dev/iicbus/if_ic.c
==============================================================================
--- head/sys/dev/iicbus/if_ic.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/dev/iicbus/if_ic.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr)
 		BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN);
 		top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0);
 		if (top) {
+			struct epoch_tracker et;
+
 			mtx_unlock(&sc->ic_lock);
 			M_SETFIB(top, sc->ic_ifp->if_fib);
+			NET_EPOCH_ENTER(et);
 			netisr_dispatch(NETISR_IP, top);
+			NET_EPOCH_EXIT(et);
 			mtx_lock(&sc->ic_lock);
 		}
 		break;

Modified: head/sys/dev/usb/net/if_usie.c
==============================================================================
--- head/sys/dev/usb/net/if_usie.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/dev/usb/net/if_usie.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -773,6 +773,7 @@ tr_setup:
 static void
 usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 {
+	struct epoch_tracker et;
 	struct usie_softc *sc = usbd_xfer_softc(xfer);
 	struct ifnet *ifp = sc->sc_ifp;
 	struct mbuf *m0;
@@ -852,6 +853,7 @@ tr_setup:
 	err = pkt = 0;
 
 	/* HW can aggregate multiple frames in a single USB xfer */
+	NET_EPOCH_ENTER(et);
 	for (;;) {
 		rxd = mtod(m, struct usie_desc *);
 
@@ -918,6 +920,7 @@ tr_setup:
 		m->m_data += diff;
 		m->m_pkthdr.len = (m->m_len -= diff);
 	}
+	NET_EPOCH_EXIT(et);
 
 	mtx_lock(&sc->sc_mtx);
 

Modified: head/sys/dev/usb/net/uhso.c
==============================================================================
--- head/sys/dev/usb/net/uhso.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/dev/usb/net/uhso.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -1664,6 +1664,7 @@ tr_setup:
 static void
 uhso_if_rxflush(void *arg)
 {
+	struct epoch_tracker et;
 	struct uhso_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	uint8_t *cp;
@@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg)
 
 	m = NULL;
 	mwait = sc->sc_mwait;
+	NET_EPOCH_ENTER(et);
 	for (;;) {
 		if (m == NULL) {
 			if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL)
@@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg)
 		m = m0 != NULL ? m0 : NULL;
 		mtx_lock(&sc->sc_mtx);
 	}
+	NET_EPOCH_EXIT(et);
 	sc->sc_mwait = mwait;
 }
 

Modified: head/sys/dev/usb/net/usb_ethernet.c
==============================================================================
--- head/sys/dev/usb/net/usb_ethernet.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/dev/usb/net/usb_ethernet.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -645,22 +645,21 @@ void
 uether_rxflush(struct usb_ether *ue)
 {
 	struct ifnet *ifp = ue->ue_ifp;
-	struct mbuf *m;
+	struct epoch_tracker et;
+	struct mbuf *m, *n;
 
 	UE_LOCK_ASSERT(ue, MA_OWNED);
 
-	for (;;) {
-		m = mbufq_dequeue(&ue->ue_rxq);
-		if (m == NULL)
-			break;
-
-		/*
-		 * The USB xfer has been resubmitted so its safe to unlock now.
-		 */
-		UE_UNLOCK(ue);
+	n = mbufq_flush(&ue->ue_rxq);
+	UE_UNLOCK(ue);
+	NET_EPOCH_ENTER(et);
+	while ((m = n) != NULL) {
+		n = STAILQ_NEXT(m, m_stailqpkt);
+		m->m_nextpkt = NULL;
 		ifp->if_input(ifp, m);
-		UE_LOCK(ue);
 	}
+	NET_EPOCH_EXIT(et);
+	UE_LOCK(ue);
 }
 
 /*

Modified: head/sys/kern/uipc_socket.c
==============================================================================
--- head/sys/kern/uipc_socket.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/kern/uipc_socket.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp.h>
 
 #include <net/vnet.h>
+#include <net/if.h>	/* XXXGL: net_epoch should move out there */
+#include <net/if_var.h>	/* XXXGL: net_epoch should move out there */
 
 #include <security/mac/mac_framework.h>
 

Modified: head/sys/net/if.c
==============================================================================
--- head/sys/net/if.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -351,17 +351,14 @@ ifnet_byindex(u_short idx)
 struct ifnet *
 ifnet_byindex_ref(u_short idx)
 {
-	struct epoch_tracker et;
 	struct ifnet *ifp;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	ifp = ifnet_byindex_locked(idx);
-	if (ifp == NULL || (ifp->if_flags & IFF_DYING)) {
-		NET_EPOCH_EXIT(et);
+	if (ifp == NULL || (ifp->if_flags & IFF_DYING))
 		return (NULL);
-	}
 	if_ref(ifp);
-	NET_EPOCH_EXIT(et);
 	return (ifp);
 }
 
@@ -425,15 +422,14 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp)
 struct ifaddr *
 ifaddr_byindex(u_short idx)
 {
-	struct epoch_tracker et;
 	struct ifnet *ifp;
 	struct ifaddr *ifa = NULL;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	ifp = ifnet_byindex_locked(idx);
 	if (ifp != NULL && (ifa = ifp->if_addr) != NULL)
 		ifa_ref(ifa);
-	NET_EPOCH_EXIT(et);
 	return (ifa);
 }
 
@@ -1640,39 +1636,32 @@ ifgr_groups_get(void *ifgrp)
 static int
 if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp)
 {
-	struct epoch_tracker	 et;
 	int			 len, error;
 	struct ifg_list		*ifgl;
 	struct ifg_req		 ifgrq, *ifgp;
 
+	NET_EPOCH_ASSERT();
+
 	if (ifgr->ifgr_len == 0) {
-		NET_EPOCH_ENTER(et);
 		CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
 			ifgr->ifgr_len += sizeof(struct ifg_req);
-		NET_EPOCH_EXIT(et);
 		return (0);
 	}
 
 	len = ifgr->ifgr_len;
 	ifgp = ifgr_groups_get(ifgr);
 	/* XXX: wire */
-	NET_EPOCH_ENTER(et);
 	CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
-		if (len < sizeof(ifgrq)) {
-			NET_EPOCH_EXIT(et);
+		if (len < sizeof(ifgrq))
 			return (EINVAL);
-		}
 		bzero(&ifgrq, sizeof ifgrq);
 		strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group,
 		    sizeof(ifgrq.ifgrq_group));
-		if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) {
-		    	NET_EPOCH_EXIT(et);
+		if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req))))
 			return (error);
-		}
 		len -= sizeof(ifgrq);
 		ifgp++;
 	}
-	NET_EPOCH_EXIT(et);
 
 	return (0);
 }
@@ -1972,7 +1961,8 @@ ifa_ifwithaddr(const struct sockaddr *addr)
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	MPASS(in_epoch(net_epoch_preempt));
+	NET_EPOCH_ASSERT();
+
 	CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 			if (ifa->ifa_addr->sa_family != addr->sa_family)
@@ -2324,17 +2314,22 @@ if_link_state_change(struct ifnet *ifp, int link_state
 
 	ifp->if_link_state = link_state;
 
+	/* XXXGL: reference ifp? */
 	taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask);
 }
 
 static void
 do_link_state_change(void *arg, int pending)
 {
-	struct ifnet *ifp = (struct ifnet *)arg;
-	int link_state = ifp->if_link_state;
-	CURVNET_SET(ifp->if_vnet);
+	struct epoch_tracker et;
+	struct ifnet *ifp;
+	int link_state;
 
-	/* Notify that the link state has changed. */
+	NET_EPOCH_ENTER(et);
+	ifp = arg;
+	link_state = ifp->if_link_state;
+
+	CURVNET_SET(ifp->if_vnet);
 	rt_ifmsg(ifp);
 	if (ifp->if_vlantrunk != NULL)
 		(*vlan_link_state_p)(ifp);
@@ -2360,6 +2355,7 @@ do_link_state_change(void *arg, int pending)
 		    (link_state == LINK_STATE_UP) ? "UP" : "DOWN" );
 	EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state);
 	CURVNET_RESTORE();
+	NET_EPOCH_EXIT(et);
 }
 
 /*
@@ -2912,9 +2908,14 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data,
 		break;
 
 	case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP):
-		if ((error = if_getgroup((struct ifgroupreq *)data, ifp)))
-			return (error);
+	{
+		struct epoch_tracker et;
+
+		NET_EPOCH_ENTER(et);
+		error = if_getgroup((struct ifgroupreq *)data, ifp);
+		NET_EPOCH_EXIT(et);
 		break;
+	}
 
 	case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP):
 		error = priv_check(td, PRIV_NET_DELIFGROUP);
@@ -2983,7 +2984,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
 #ifdef COMPAT_FREEBSD32
 	caddr_t saved_data = NULL;
 	struct ifmediareq ifmr;
-	struct ifmediareq *ifmrp;
+	struct ifmediareq *ifmrp = NULL;
 #endif
 	struct ifnet *ifp;
 	struct ifreq *ifr;
@@ -2999,12 +3000,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
 	}
 #endif
 
-
 	switch (cmd) {
 	case SIOCGIFCONF:
 		error = ifconf(cmd, data);
-		CURVNET_RESTORE();
-		return (error);
+		goto out_noref;
 
 #ifdef COMPAT_FREEBSD32
 	case SIOCGIFCONF32:
@@ -3017,16 +3016,14 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
 			ifc.ifc_buf = PTRIN(ifc32->ifc_buf);
 
 			error = ifconf(SIOCGIFCONF, (void *)&ifc);
-			CURVNET_RESTORE();
 			if (error == 0)
 				ifc32->ifc_len = ifc.ifc_len;
-			return (error);
+			goto out_noref;
 		}
 #endif
 	}
 
 #ifdef COMPAT_FREEBSD32
-	ifmrp = NULL;
 	switch (cmd) {
 	case SIOCGIFMEDIA32:
 	case SIOCGIFXMEDIA32:
@@ -3618,16 +3615,15 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
 	struct ifmultiaddr *ifma;
 	int lastref;
 #ifdef INVARIANTS
-	struct epoch_tracker et;
 	struct ifnet *oifp;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
 		if (ifp == oifp)
 			break;
 	if (ifp != oifp)
 		ifp = NULL;
-	NET_EPOCH_EXIT(et);
 
 	KASSERT(ifp != NULL, ("%s: ifnet went away", __func__));
 #endif
@@ -3693,16 +3689,15 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int f
 	if (ifp == NULL) {
 		printf("%s: ifma_ifp seems to be detached\n", __func__);
 	} else {
-		struct epoch_tracker et;
 		struct ifnet *oifp;
 
-		NET_EPOCH_ENTER(et);
+		NET_EPOCH_ASSERT();
+
 		CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
 			if (ifp == oifp)
 				break;
 		if (ifp != oifp)
 			ifp = NULL;
-		NET_EPOCH_EXIT(et);
 	}
 #endif
 	/*
@@ -3826,11 +3821,11 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, 
 	struct sockaddr_dl *sdl;
 	struct ifaddr *ifa;
 	struct ifreq ifr;
-	struct epoch_tracker et;
 	int rc;
 
+	NET_EPOCH_ASSERT();
+
 	rc = 0;
-	NET_EPOCH_ENTER(et);
 	ifa = ifp->if_addr;
 	if (ifa == NULL) {
 		rc = EINVAL;
@@ -3864,7 +3859,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, 
 	 * to re-init it in order to reprogram its
 	 * address filter.
 	 */
-	NET_EPOCH_EXIT(et);
 	if ((ifp->if_flags & IFF_UP) != 0) {
 		if (ifp->if_ioctl) {
 			ifp->if_flags &= ~IFF_UP;
@@ -3879,8 +3873,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, 
 	}
 	EVENTHANDLER_INVOKE(iflladdr_event, ifp);
 	return (0);
- out:
-	NET_EPOCH_EXIT(et);
+out:
 	return (rc);
 }
 

Modified: head/sys/net/if_ethersubr.c
==============================================================================
--- head/sys/net/if_ethersubr.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_ethersubr.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -800,6 +800,7 @@ VNET_SYSUNINIT(vnet_ether_uninit, SI_SUB_PROTO_IF, SI_
 static void
 ether_input(struct ifnet *ifp, struct mbuf *m)
 {
+	struct epoch_tracker et;
 	struct mbuf *mn;
 
 	/*
@@ -807,22 +808,24 @@ ether_input(struct ifnet *ifp, struct mbuf *m)
 	 * m_nextpkt. We split them up into separate packets here and pass
 	 * them up. This allows the drivers to amortize the receive lock.
 	 */
+	CURVNET_SET_QUIET(ifp->if_vnet);
+	NET_EPOCH_ENTER(et);
 	while (m) {
 		mn = m->m_nextpkt;
 		m->m_nextpkt = NULL;
 
 		/*
-		 * We will rely on rcvif being set properly in the deferred context,
-		 * so assert it is correct here.
+		 * We will rely on rcvif being set properly in the deferred
+		 * context, so assert it is correct here.
 		 */
 		MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
 		KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p "
 		    "rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp));
-		CURVNET_SET_QUIET(ifp->if_vnet);
 		netisr_dispatch(NETISR_ETHER, m);
-		CURVNET_RESTORE();
 		m = mn;
 	}
+	NET_EPOCH_EXIT(et);
+	CURVNET_RESTORE();
 }
 
 /*
@@ -835,6 +838,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m)
 	int i, isr;
 	u_short ether_type;
 
+	NET_EPOCH_ASSERT();
 	KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__));
 
 	/* Do not grab PROMISC frames in case we are re-entered. */

Modified: head/sys/net/if_gif.c
==============================================================================
--- head/sys/net/if_gif.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_gif.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -415,6 +415,8 @@ gif_input(struct mbuf *m, struct ifnet *ifp, int proto
 	struct ifnet *oldifp;
 	int isr, n, af;
 
+	NET_EPOCH_ASSERT();
+
 	if (ifp == NULL) {
 		/* just in case */
 		m_freem(m);

Modified: head/sys/net/if_me.c
==============================================================================
--- head/sys/net/if_me.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_me.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -451,6 +451,8 @@ me_input(struct mbuf *m, int off, int proto, void *arg
 	struct ip *ip;
 	int hlen;
 
+	NET_EPOCH_ASSERT();
+
 	ifp = ME2IFP(sc);
 	/* checks for short packets */
 	hlen = sizeof(struct mobhdr);

Modified: head/sys/net/if_stf.c
==============================================================================
--- head/sys/net/if_stf.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_stf.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -613,6 +613,8 @@ in_stf_input(struct mbuf *m, int off, int proto, void 
 	u_int8_t otos, itos;
 	struct ifnet *ifp;
 
+	NET_EPOCH_ASSERT();
+
 	if (proto != IPPROTO_IPV6) {
 		m_freem(m);
 		return (IPPROTO_DONE);

Modified: head/sys/net/if_tuntap.c
==============================================================================
--- head/sys/net/if_tuntap.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_tuntap.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -1662,6 +1662,7 @@ tunwrite_l2(struct tuntap_softc *tp, struct mbuf *m)
 static int
 tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m)
 {
+	struct epoch_tracker et;
 	struct ifnet *ifp;
 	int family, isr;
 
@@ -1702,7 +1703,9 @@ tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m)
 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 	CURVNET_SET(ifp->if_vnet);
 	M_SETFIB(m, ifp->if_fib);
+	NET_EPOCH_ENTER(et);
 	netisr_dispatch(isr, m);
+	NET_EPOCH_EXIT(et);
 	CURVNET_RESTORE();
 	return (0);
 }

Modified: head/sys/net/if_vlan.c
==============================================================================
--- head/sys/net/if_vlan.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/if_vlan.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -255,7 +255,6 @@ static struct sx _VLAN_SX_ID;
 #define	TRUNK_LOCK_DESTROY(trunk)	mtx_destroy(&(trunk)->lock)
 #define	TRUNK_WLOCK(trunk)		mtx_lock(&(trunk)->lock)
 #define	TRUNK_WUNLOCK(trunk)		mtx_unlock(&(trunk)->lock)
-#define	TRUNK_LOCK_ASSERT(trunk)	MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock))
 #define	TRUNK_WLOCK_ASSERT(trunk)	mtx_assert(&(trunk)->lock, MA_OWNED);
 
 /*
@@ -704,18 +703,17 @@ vlan_ifdetach(void *arg __unused, struct ifnet *ifp)
 static struct ifnet  *
 vlan_trunkdev(struct ifnet *ifp)
 {
-	struct epoch_tracker et;
 	struct ifvlan *ifv;
 
+	NET_EPOCH_ASSERT();
+
 	if (ifp->if_type != IFT_L2VLAN)
 		return (NULL);
 
-	NET_EPOCH_ENTER(et);
 	ifv = ifp->if_softc;
 	ifp = NULL;
 	if (ifv->ifv_trunk)
 		ifp = PARENT(ifv);
-	NET_EPOCH_EXIT(et);
 	return (ifp);
 }
 
@@ -787,21 +785,18 @@ vlan_setcookie(struct ifnet *ifp, void *cookie)
 static struct ifnet *
 vlan_devat(struct ifnet *ifp, uint16_t vid)
 {
-	struct epoch_tracker et;
 	struct ifvlantrunk *trunk;
 	struct ifvlan *ifv;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	trunk = ifp->if_vlantrunk;
-	if (trunk == NULL) {
-		NET_EPOCH_EXIT(et);
+	if (trunk == NULL)
 		return (NULL);
-	}
 	ifp = NULL;
 	ifv = vlan_gethash(trunk, vid);
 	if (ifv)
 		ifp = ifv->ifv_ifp;
-	NET_EPOCH_EXIT(et);
 	return (ifp);
 }
 
@@ -1140,16 +1135,15 @@ vlan_init(void *foo __unused)
 static int
 vlan_transmit(struct ifnet *ifp, struct mbuf *m)
 {
-	struct epoch_tracker et;
 	struct ifvlan *ifv;
 	struct ifnet *p;
 	int error, len, mcast;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	ifv = ifp->if_softc;
 	if (TRUNK(ifv) == NULL) {
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
-		NET_EPOCH_EXIT(et);
 		m_freem(m);
 		return (ENETDOWN);
 	}
@@ -1169,7 +1163,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
 		vst = mst_to_vst(mst);
 		if (vst->tag->ifp != p) {
 			if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
-			NET_EPOCH_EXIT(et);
 			m_freem(m);
 			return (EAGAIN);
 		}
@@ -1185,14 +1178,12 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
 	 */
 	if (!UP_AND_RUNNING(p)) {
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
-		NET_EPOCH_EXIT(et);
 		m_freem(m);
 		return (ENETDOWN);
 	}
 
 	if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) {
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
-		NET_EPOCH_EXIT(et);
 		return (0);
 	}
 
@@ -1206,7 +1197,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
 		if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast);
 	} else
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
-	NET_EPOCH_EXIT(et);
 	return (error);
 }
 
@@ -1214,19 +1204,17 @@ static int
 vlan_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
     struct route *ro)
 {
-	struct epoch_tracker et;
 	struct ifvlan *ifv;
 	struct ifnet *p;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	ifv = ifp->if_softc;
 	if (TRUNK(ifv) == NULL) {
-		NET_EPOCH_EXIT(et);
 		m_freem(m);
 		return (ENETDOWN);
 	}
 	p = PARENT(ifv);
-	NET_EPOCH_EXIT(et);
 	return p->if_output(ifp, m, dst, ro);
 }
 
@@ -1242,16 +1230,15 @@ vlan_qflush(struct ifnet *ifp __unused)
 static void
 vlan_input(struct ifnet *ifp, struct mbuf *m)
 {
-	struct epoch_tracker et;
 	struct ifvlantrunk *trunk;
 	struct ifvlan *ifv;
 	struct m_tag *mtag;
 	uint16_t vid, tag;
 
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	trunk = ifp->if_vlantrunk;
 	if (trunk == NULL) {
-		NET_EPOCH_EXIT(et);
 		m_freem(m);
 		return;
 	}
@@ -1274,7 +1261,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
 			if (m->m_len < sizeof(*evl) &&
 			    (m = m_pullup(m, sizeof(*evl))) == NULL) {
 				if_printf(ifp, "cannot pullup VLAN header\n");
-				NET_EPOCH_EXIT(et);
 				return;
 			}
 			evl = mtod(m, struct ether_vlan_header *);
@@ -1297,7 +1283,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
 			      __func__, ifp->if_xname, ifp->if_type);
 #endif
 			if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
-			NET_EPOCH_EXIT(et);
 			m_freem(m);
 			return;
 		}
@@ -1307,7 +1292,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
 
 	ifv = vlan_gethash(trunk, vid);
 	if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) {
-		NET_EPOCH_EXIT(et);
 		if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
 		m_freem(m);
 		return;
@@ -1327,7 +1311,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
 			    sizeof(uint8_t), M_NOWAIT);
 			if (mtag == NULL) {
 				if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
-				NET_EPOCH_EXIT(et);
 				m_freem(m);
 				return;
 			}
@@ -1338,7 +1321,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
 
 	m->m_pkthdr.rcvif = ifv->ifv_ifp;
 	if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1);
-	NET_EPOCH_EXIT(et);
 
 	/* Pass it back through the parent's input routine. */
 	(*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m);
@@ -1364,11 +1346,12 @@ vlan_lladdr_fn(void *arg, int pending __unused)
 static int
 vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid)
 {
-	struct epoch_tracker et;
 	struct ifvlantrunk *trunk;
 	struct ifnet *ifp;
 	int error = 0;
 
+	NET_EPOCH_ASSERT();
+
 	/*
 	 * We can handle non-ethernet hardware types as long as
 	 * they handle the tagging and headers themselves.
@@ -1469,9 +1452,7 @@ vlan_config(struct ifvlan *ifv, struct ifnet *p, uint1
 
 	ifp->if_link_state = p->if_link_state;
 
-	NET_EPOCH_ENTER(et);
 	vlan_capabilities(ifv);
-	NET_EPOCH_EXIT(et);
 
 	/*
 	 * Set up our interface address to reflect the underlying
@@ -1643,17 +1624,14 @@ vlan_setflags(struct ifnet *ifp, int status)
 static void
 vlan_link_state(struct ifnet *ifp)
 {
-	struct epoch_tracker et;
 	struct ifvlantrunk *trunk;
 	struct ifvlan *ifv;
 
-	/* Called from a taskqueue_swi task, so we cannot sleep. */
-	NET_EPOCH_ENTER(et);
+	NET_EPOCH_ASSERT();
+
 	trunk = ifp->if_vlantrunk;
-	if (trunk == NULL) {
-		NET_EPOCH_EXIT(et);
+	if (trunk == NULL)
 		return;
-	}
 
 	TRUNK_WLOCK(trunk);
 	VLAN_FOREACH(ifv, trunk) {
@@ -1662,7 +1640,6 @@ vlan_link_state(struct ifnet *ifp)
 		    trunk->parent->if_link_state);
 	}
 	TRUNK_WUNLOCK(trunk);
-	NET_EPOCH_EXIT(et);
 }
 
 static void
@@ -1674,8 +1651,9 @@ vlan_capabilities(struct ifvlan *ifv)
 	int cap = 0, ena = 0, mena;
 	u_long hwa = 0;
 
-	VLAN_SXLOCK_ASSERT();
 	NET_EPOCH_ASSERT();
+	VLAN_SXLOCK_ASSERT();
+
 	p = PARENT(ifv);
 	ifp = ifv->ifv_ifp;
 
@@ -1791,7 +1769,6 @@ vlan_capabilities(struct ifvlan *ifv)
 static void
 vlan_trunk_capabilities(struct ifnet *ifp)
 {
-	struct epoch_tracker et;
 	struct ifvlantrunk *trunk;
 	struct ifvlan *ifv;
 
@@ -1801,11 +1778,8 @@ vlan_trunk_capabilities(struct ifnet *ifp)
 		VLAN_SUNLOCK();
 		return;
 	}
-	NET_EPOCH_ENTER(et);
-	VLAN_FOREACH(ifv, trunk) {
+	VLAN_FOREACH(ifv, trunk)
 		vlan_capabilities(ifv);
-	}
-	NET_EPOCH_EXIT(et);
 	VLAN_SUNLOCK();
 }
 
@@ -1820,6 +1794,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data
 	struct vlanreq vlr;
 	int error = 0;
 
+	NET_EPOCH_ASSERT();
+
 	ifr = (struct ifreq *)data;
 	ifa = (struct ifaddr *) data;
 	ifv = ifp->if_softc;
@@ -1996,13 +1972,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data
 		VLAN_SLOCK();
 		ifv->ifv_capenable = ifr->ifr_reqcap;
 		trunk = TRUNK(ifv);
-		if (trunk != NULL) {
-			struct epoch_tracker et;
-
-			NET_EPOCH_ENTER(et);
+		if (trunk != NULL)
 			vlan_capabilities(ifv);
-			NET_EPOCH_EXIT(et);
-		}
 		VLAN_SUNLOCK();
 		break;
 

Modified: head/sys/net/netisr.c
==============================================================================
--- head/sys/net/netisr.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/netisr.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -861,6 +861,7 @@ static u_int
 netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto)
 {
 	struct netisr_work local_npw, *npwp;
+	struct epoch_tracker et;
 	u_int handled;
 	struct mbuf *m;
 
@@ -890,6 +891,7 @@ netisr_process_workstream_proto(struct netisr_workstre
 	npwp->nw_len = 0;
 	nwsp->nws_pendingbits &= ~(1 << proto);
 	NWS_UNLOCK(nwsp);
+	NET_EPOCH_ENTER(et);
 	while ((m = local_npw.nw_head) != NULL) {
 		local_npw.nw_head = m->m_nextpkt;
 		m->m_nextpkt = NULL;
@@ -902,6 +904,7 @@ netisr_process_workstream_proto(struct netisr_workstre
 		netisr_proto[proto].np_handler(m);
 		CURVNET_RESTORE();
 	}
+	NET_EPOCH_EXIT(et);
 	KASSERT(local_npw.nw_len == 0,
 	    ("%s(%u): len %u", __func__, proto, local_npw.nw_len));
 	if (netisr_proto[proto].np_drainedcpu)
@@ -1088,6 +1091,7 @@ netisr_dispatch_src(u_int proto, uintptr_t source, str
 	int dosignal, error;
 	u_int cpuid, dispatch_policy;
 
+	NET_EPOCH_ASSERT();
 	KASSERT(proto < NETISR_MAXPROT,
 	    ("%s: invalid proto %u", __func__, proto));
 #ifdef NETISR_LOCKING

Modified: head/sys/net/route.c
==============================================================================
--- head/sys/net/route.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/route.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -593,12 +593,12 @@ rtredirect_fib(struct sockaddr *dst,
 	int error = 0;
 	short *stat = NULL;
 	struct rt_addrinfo info;
-	struct epoch_tracker et;
 	struct ifaddr *ifa;
 	struct rib_head *rnh;
 
+	NET_EPOCH_ASSERT();
+
 	ifa = NULL;
-	NET_EPOCH_ENTER(et);
 	rnh = rt_tables_get_rnh(fibnum, dst->sa_family);
 	if (rnh == NULL) {
 		error = EAFNOSUPPORT;
@@ -693,7 +693,6 @@ done:
 	if (rt)
 		RTFREE_LOCKED(rt);
  out:
-	NET_EPOCH_EXIT(et);
 	if (error)
 		V_rtstat.rts_badredirect++;
 	else if (stat != NULL)

Modified: head/sys/net/rtsock.c
==============================================================================
--- head/sys/net/rtsock.c	Mon Oct  7 20:41:55 2019	(r353291)
+++ head/sys/net/rtsock.c	Mon Oct  7 22:40:05 2019	(r353292)
@@ -560,6 +560,7 @@ route_output(struct mbuf *m, struct socket *so, ...)
 	struct rib_head *rnh;
 	struct rt_addrinfo info;
 	struct sockaddr_storage ss;
+	struct epoch_tracker et;
 #ifdef INET6
 	struct sockaddr_in6 *sin6;
 	int i, rti_need_deembed = 0;
@@ -579,6 +580,7 @@ route_output(struct mbuf *m, struct socket *so, ...)
 		return (ENOBUFS);
 	if ((m->m_flags & M_PKTHDR) == 0)
 		panic("route_output");
+	NET_EPOCH_ENTER(et);
 	len = m->m_pkthdr.len;
 	if (len < sizeof(*rtm) ||
 	    len != mtod(m, struct rt_msghdr *)->rtm_msglen)
@@ -803,11 +805,11 @@ route_output(struct mbuf *m, struct socket *so, ...)
 				NET_EPOCH_ENTER(et);
 				ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1,
 						RT_ALL_FIBS);
+				NET_EPOCH_EXIT(et);
 				if (ifa != NULL)
 					rt_maskedcopy(ifa->ifa_addr,
 						      &laddr,
 						      ifa->ifa_netmask);
-				NET_EPOCH_EXIT(et);
 			} else
 				rt_maskedcopy(rt->rt_ifa->ifa_addr,
 					      &laddr,
@@ -898,6 +900,7 @@ report:
 	}
 
 flush:
+	NET_EPOCH_EXIT(et);
 	if (rt != NULL)
 		RTFREE(rt);
 	/*
@@ -1761,11 +1764,9 @@ sysctl_iflist(int af, struct walkarg *w)
 	struct rt_addrinfo info;
 	int len, error = 0;
 	struct sockaddr_storage ss;
-	struct epoch_tracker et;
 
 	bzero((caddr_t)&info, sizeof(info));
 	bzero(&ifd, sizeof(ifd));
-	NET_EPOCH_ENTER(et);
 	CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (w->w_arg && w->w_arg != ifp->if_index)
 			continue;
@@ -1815,7 +1816,6 @@ sysctl_iflist(int af, struct walkarg *w)
 		info.rti_info[RTAX_BRD] = NULL;
 	}
 done:
-	NET_EPOCH_EXIT(et);
 	return (error);
 }
 
@@ -1823,16 +1823,16 @@ static int
 sysctl_ifmalist(int af, struct walkarg *w)
 {
 	struct rt_addrinfo info;
-	struct epoch_tracker et;
 	struct ifaddr *ifa;
 	struct ifmultiaddr *ifma;
 	struct ifnet *ifp;
 	int error, len;
 
+	NET_EPOCH_ASSERT();
+
 	error = 0;
 	bzero((caddr_t)&info, sizeof(info));
 
-	NET_EPOCH_ENTER(et);
 	CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (w->w_arg && w->w_arg != ifp->if_index)
 			continue;
@@ -1867,7 +1867,6 @@ sysctl_ifmalist(int af, struct walkarg *w)
 		if (error != 0)
 			break;
 	}
-	NET_EPOCH_EXIT(et);
 	return (error);
 }
 
@@ -1875,6 +1874,7 @@ static int
 sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 {
 	RIB_RLOCK_TRACKER;
+	struct epoch_tracker et;
 	int	*name = (int *)arg1;
 	u_int	namelen = arg2;
 	struct rib_head *rnh = NULL; /* silence compiler. */
@@ -1918,8 +1918,8 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 	w.w_tmemsize = 65536;
 	w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK);
 
+	NET_EPOCH_ENTER(et);
 	switch (w.w_op) {
-
 	case NET_RT_DUMP:
 	case NET_RT_FLAGS:
 		if (af == 0) {			/* dump all tables */
@@ -1946,13 +1946,9 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 		for (error = 0; error == 0 && i <= lim; i++) {
 			rnh = rt_tables_get_rnh(fib, i);
 			if (rnh != NULL) {
-				struct epoch_tracker et;
-
 				RIB_RLOCK(rnh); 
-				NET_EPOCH_ENTER(et);
 			    	error = rnh->rnh_walktree(&rnh->head,
 				    sysctl_dumpentry, &w);
-				NET_EPOCH_EXIT(et);
 				RIB_RUNLOCK(rnh);
 			} else if (af != 0)
 				error = EAFNOSUPPORT;
@@ -1968,6 +1964,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 		error = sysctl_ifmalist(af, &w);
 		break;
 	}
+	NET_EPOCH_EXIT(et);
 
 	free(w.w_tmem, M_TEMP);
 	return (error);

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910072240.x97Me60x065650>