Date: Sat, 22 Nov 2014 19:53:37 +0000 (UTC) From: "Alexander V. Chernikov" <melifaro@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r274887 - in projects/routing/sys: net netinet netinet6 Message-ID: <201411221953.sAMJrb16064087@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: melifaro Date: Sat Nov 22 19:53:36 2014 New Revision: 274887 URL: https://svnweb.freebsd.org/changeset/base/274887 Log: Use less-invasive approach for IF_AFDATA lock: convert into 2 locks: use rwlock accessible via external functions (IF_AFDATA_CFG_* -> if_afdata_cfg_*()) for all control plane tasks use rmlock (IF_AFDATA_RUN_*) for fast-path lookups. Modified: projects/routing/sys/net/if.c projects/routing/sys/net/if_var.h projects/routing/sys/net/rt_nhops.c projects/routing/sys/netinet/if_ether.c projects/routing/sys/netinet/in.c projects/routing/sys/netinet6/in6.c Modified: projects/routing/sys/net/if.c ============================================================================== --- projects/routing/sys/net/if.c Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/net/if.c Sat Nov 22 19:53:36 2014 (r274887) @@ -51,6 +51,7 @@ #include <sys/lock.h> #include <sys/refcount.h> #include <sys/module.h> +#include <sys/rmlock.h> #include <sys/rwlock.h> #include <sys/sockio.h> #include <sys/syslog.h> @@ -773,7 +774,7 @@ if_attachdomain1(struct ifnet *ifp) * Since dp->dom_ifattach calls malloc() with M_WAITOK, we * cannot lock ifp->if_afdata initialization, entirely. */ - if (IF_AFDATA_TRYLOCK(ifp) == 0) + if (IF_AFDATA_TRY_WLOCK(ifp) == 0) return; if (ifp->if_afdata_initialized >= domain_init_status) { IF_AFDATA_UNLOCK(ifp); @@ -3937,3 +3938,65 @@ drbr_enqueue_drv(if_t ifh, struct buf_ri return drbr_enqueue(ifh, br, m); } + +void +if_afdata_cfg_rlock(struct ifnet *ifp) +{ + + rw_rlock(&ifp->if_afdata_cfg_lock); +} + +void +if_afdata_cfg_runlock(struct ifnet *ifp) +{ + + rw_runlock(&ifp->if_afdata_cfg_lock); +} + +void +if_afdata_cfg_wlock(struct ifnet *ifp) +{ + + rw_wlock(&ifp->if_afdata_cfg_lock); +} + +void +if_afdata_cfg_wunlock(struct ifnet *ifp) +{ + + rw_wunlock(&ifp->if_afdata_cfg_lock); +} + +void +if_afdata_cfg_lock_assert(struct ifnet *ifp, int what) +{ + + rw_assert(&ifp->if_afdata_cfg_lock, what); +} + +void +if_afdata_wlock(struct ifnet *ifp) +{ + + if_afdata_cfg_wlock(ifp); + IF_AFDATA_RUN_WLOCK(ifp); +} + +void +if_afdata_wunlock(struct ifnet *ifp) +{ + + if_afdata_cfg_wunlock(ifp); + IF_AFDATA_RUN_WUNLOCK(ifp); +} + +int +if_afdata_try_wlock(struct ifnet *ifp) +{ + if (rw_try_wlock(&ifp->if_afdata_cfg_lock) == 0) + return (0); + + IF_AFDATA_RUN_WLOCK(ifp); + return (1); +} + Modified: projects/routing/sys/net/if_var.h ============================================================================== --- projects/routing/sys/net/if_var.h Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/net/if_var.h Sat Nov 22 19:53:36 2014 (r274887) @@ -192,7 +192,7 @@ struct ifnet { int if_amcount; /* number of all-multicast requests */ struct ifaddr *if_addr; /* pointer to link-level address */ const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */ - struct rwlock if_afdata_lock; + struct rmlock if_afdata_run_lock; void *if_afdata[AF_MAX]; int if_afdata_initialized; @@ -253,6 +253,7 @@ struct ifnet { u_int if_hw_tsomaxsegcount; /* TSO maximum segment count */ u_int if_hw_tsomaxsegsize; /* TSO maximum segment size in bytes */ + struct rwlock if_afdata_cfg_lock; /* * Spare fields to be added before branching a stable branch, so * that structure can be enhanced without changing the kernel @@ -339,22 +340,59 @@ typedef void (*group_change_event_handle EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t); #endif /* _SYS_EVENTHANDLER_H_ */ -#define IF_AFDATA_LOCK_INIT(ifp) \ - rw_init(&(ifp)->if_afdata_lock, "if_afdata") +#define IF_AFDATA_LOCK_INIT(ifp) do { \ + rw_init(&(ifp)->if_afdata_cfg_lock, "if_afdata_cfg"); \ + rm_init(&(ifp)->if_afdata_run_lock, "if_afdata_run"); \ +} while (0) + +#define IF_AFDATA_DESTROY(ifp) do { \ + rw_destroy(&(ifp)->if_afdata_cfg_lock); \ + rm_destroy(&(ifp)->if_afdata_run_lock); \ +} while(0) + +/* if_afdata lock: control plane functions */ +#define IF_AFDATA_CFG_RLOCK(ifp) if_afdata_cfg_rlock(ifp) +#define IF_AFDATA_CFG_RUNLOCK(ifp) if_afdata_cfg_runlock(ifp) +#define IF_AFDATA_CFG_WLOCK(ifp) if_afdata_cfg_wlock(ifp) +#define IF_AFDATA_CFG_WUNLOCK(ifp) if_afdata_cfg_wunlock(ifp) + +#define IF_AFDATA_CFG_LOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_LOCKED) +#define IF_AFDATA_CFG_RLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_RLOCKED) +#define IF_AFDATA_CFG_WLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_WLOCKED) +#define IF_AFDATA_CFG_UNLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i,RA_UNLOCKED) + +void if_afdata_cfg_rlock(struct ifnet *ifp); +void if_afdata_cfg_runlock(struct ifnet *ifp); +void if_afdata_cfg_wlock(struct ifnet *ifp); +void if_afdata_cfg_wunlock(struct ifnet *ifp); +void if_afdata_cfg_lock_assert(struct ifnet *ifp, int what); + +/* if_afdata lock: fast path */ +#define IF_AFDATA_RUN_WLOCK(ifp) rm_wlock(&(ifp)->if_afdata_run_lock) +#define IF_AFDATA_RUN_WUNLOCK(ifp) rm_wunlock(&(ifp)->if_afdata_run_lock) +#define IF_AFDATA_RUN_RLOCK(ifp) \ + rm_rlock(&(ifp)->if_afdata_run_lock, &if_afdata_tracker) +#define IF_AFDATA_RUN_RUNLOCK(ifp) \ + rm_runlock(&(ifp)->if_afdata_run_lock, &if_afdata_tracker) +#define IF_AFDATA_RUN_TRACKER struct rm_priotracker if_afdata_tracker + +/* Common wrappers */ +#define IF_AFDATA_RLOCK(ifp) IF_AFDATA_CFG_RLOCK(ifp) +#define IF_AFDATA_RUNLOCK(ifp) IF_AFDATA_CFG_RUNLOCK(ifp) +#define IF_AFDATA_WLOCK(ifp) if_afdata_wlock(ifp) +#define IF_AFDATA_WUNLOCK(ifp) if_afdata_wunlock(ifp) -#define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_TRY_WLOCK(ifp) if_afdata_try_wlock(ifp) #define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp) #define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp) -#define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock) - -#define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED) -#define IF_AFDATA_RLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_RLOCKED) -#define IF_AFDATA_WLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_WLOCKED) -#define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED) +void if_afdata_wlock(struct ifnet *ifp); +void if_afdata_wunlock(struct ifnet *ifp); +int if_afdata_try_wlock(struct ifnet *ifp); + +#define IF_AFDATA_LOCK_ASSERT(ifp) IF_AFDATA_CFG_LOCK_ASSERT(ifp) +#define IF_AFDATA_RLOCK_ASSERT(ifp) IF_AFDATA_CFG_RLOCK_ASSERT(ifp) +#define IF_AFDATA_WLOCK_ASSERT(ifp) IF_AFDATA_CFG_WLOCK_ASSERT(ifp) +#define IF_AFDATA_UNLOCK_ASSERT(ifp) IF_AFDATA_CFG_UNLOCK_ASSERT(ifp) /* * 72 was chosen below because it is the size of a TCP/IP Modified: projects/routing/sys/net/rt_nhops.c ============================================================================== --- projects/routing/sys/net/rt_nhops.c Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/net/rt_nhops.c Sat Nov 22 19:53:36 2014 (r274887) @@ -681,6 +681,7 @@ fib6_storelladdr(struct ifnet *ifp, stru { struct llentry *ln; struct sockaddr_in6 dst_sa; + IF_AFDATA_RUN_TRACKER; if (mm_flags & M_MCAST) { ETHER_MAP_IPV6_MULTICAST(&dst, desten); @@ -697,7 +698,7 @@ fib6_storelladdr(struct ifnet *ifp, stru /* * the entry should have been created in nd6_store_lladdr */ - IF_AFDATA_RLOCK(ifp); + IF_AFDATA_RUN_RLOCK(ifp); ln = lla_lookup(LLTABLE6(ifp), 0, (struct sockaddr *)&dst_sa); /* @@ -712,12 +713,12 @@ fib6_storelladdr(struct ifnet *ifp, stru ln->ln_state != ND6_LLINFO_DELAY)) { if (ln != NULL) LLE_RUNLOCK(ln); - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); return (1); } bcopy(&ln->ll_addr, desten, ifp->if_addrlen); LLE_RUNLOCK(ln); - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); return (0); } Modified: projects/routing/sys/netinet/if_ether.c ============================================================================== --- projects/routing/sys/netinet/if_ether.c Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/netinet/if_ether.c Sat Nov 22 19:53:36 2014 (r274887) @@ -47,6 +47,8 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/mbuf.h> #include <sys/malloc.h> +#include <sys/lock.h> +#include <sys/rmlock.h> #include <sys/proc.h> #include <sys/socket.h> #include <sys/syslog.h> @@ -354,6 +356,7 @@ arpresolve_fast(struct ifnet *ifp, struc struct llentry *la; struct sockaddr_in sin; const struct sockaddr *sa_dst; + IF_AFDATA_RUN_TRACKER; if (mflags & M_BCAST) { memcpy(dst_addr, ifp->if_broadcastaddr, ifp->if_addrlen); @@ -370,17 +373,17 @@ arpresolve_fast(struct ifnet *ifp, struc sin.sin_len = sizeof(sin); sa_dst = (const struct sockaddr *)&sin; - IF_AFDATA_RLOCK(ifp); + IF_AFDATA_RUN_RLOCK(ifp); la = lla_lookup(LLTABLE(ifp), LLE_UNLOCKED, sa_dst); if (la != NULL && (la->r_flags & RLLE_VALID) != 0) { /* Entry found, let's copy lle info */ bcopy(&la->ll_addr, dst_addr, ifp->if_addrlen); if (la->r_kick != 0) la->r_kick = 0; /* Notify that entry was used */ - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); return (0); } - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); return (EAGAIN); @@ -438,6 +441,7 @@ arpresolve(struct ifnet *ifp, struct rte { struct llentry *la = NULL; int is_gw; + IF_AFDATA_RUN_TRACKER; *lle = NULL; if (m != NULL) { @@ -454,18 +458,18 @@ arpresolve(struct ifnet *ifp, struct rte } } - IF_AFDATA_RLOCK(ifp); + IF_AFDATA_RUN_RLOCK(ifp); la = lla_lookup(LLTABLE(ifp), LLE_UNLOCKED, dst); if (la != NULL && (la->r_flags & RLLE_VALID) != 0) { /* Entry found, let's copy lle info */ bcopy(&la->ll_addr, desten, ifp->if_addrlen); if (la->r_kick != 0) la->r_kick = 0; /* Notify that entry was used */ - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); *lle = la; return (0); } - IF_AFDATA_RUNLOCK(ifp); + IF_AFDATA_RUN_RUNLOCK(ifp); is_gw = (rt0 != NULL && (rt0->rt_flags & RTF_GATEWAY)) ? 1 : 0; return (arpresolve_slow(ifp, is_gw, m, dst, desten, lle)); Modified: projects/routing/sys/netinet/in.c ============================================================================== --- projects/routing/sys/netinet/in.c Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/netinet/in.c Sat Nov 22 19:53:36 2014 (r274887) @@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$"); #include <sys/sockio.h> #include <sys/malloc.h> #include <sys/priv.h> +#include <sys/lock.h> +#include <sys/rmlock.h> #include <sys/socket.h> #include <sys/jail.h> #include <sys/kernel.h> @@ -1249,11 +1251,14 @@ in_lltable_unlink(struct llentry *lle) static struct llentry * in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) { - struct ifnet *ifp = llt->llt_ifp; struct llentry *lle; struct in_addr dst; - IF_AFDATA_LOCK_ASSERT(ifp); + /* + * Do not check for AFDATA lock since search can be protected + * by different locks. + * IF_AFDATA_LOCK_ASSERT(llt->llt_ifp); + */ KASSERT(l3addr->sa_family == AF_INET, ("sin_family %d", l3addr->sa_family)); Modified: projects/routing/sys/netinet6/in6.c ============================================================================== --- projects/routing/sys/netinet6/in6.c Sat Nov 22 19:48:14 2014 (r274886) +++ projects/routing/sys/netinet6/in6.c Sat Nov 22 19:53:36 2014 (r274887) @@ -2180,10 +2180,9 @@ in6_lltable_delete(struct lltable *llt, const struct sockaddr *l3addr) { const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr; - struct ifnet *ifp = llt->llt_ifp; struct llentry *lle; - IF_AFDATA_LOCK_ASSERT(ifp); + IF_AFDATA_LOCK_ASSERT(llt->llt_ifp); KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family)); @@ -2287,10 +2286,13 @@ in6_lltable_lookup(struct lltable *llt, const struct sockaddr *l3addr) { const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr; - struct ifnet *ifp = llt->llt_ifp; struct llentry *lle; - IF_AFDATA_LOCK_ASSERT(ifp); + /* + * Do not check for AFDATA lock since search can be protected + * by different locks. + * IF_AFDATA_LOCK_ASSERT(llt->llt_ifp); + */ KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family));
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201411221953.sAMJrb16064087>