Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 21 Mar 2018 15:54:46 +0000 (UTC)
From:      "Jonathan T. Looney" <jtl@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r331309 - in head/sys: netinet netinet6
Message-ID:  <201803211554.w2LFskv7027707@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: jtl
Date: Wed Mar 21 15:54:46 2018
New Revision: 331309
URL: https://svnweb.freebsd.org/changeset/base/331309

Log:
  If the INP lock is uncontested, avoid taking a reference and jumping
  through the lock-switching hoops.
  
  A few of the INP lookup operations that lock INPs after the lookup do
  so using this mechanism (to maintain lock ordering):
  
  1. Lock lookup structure.
  2. Find INP.
  3. Acquire reference on INP.
  4. Drop lock on lookup structure.
  5. Acquire INP lock.
  6. Drop reference on INP.
  
  This change provides a slightly shorter path for cases where the INP
  lock is uncontested:
  
  1. Lock lookup structure.
  2. Find INP.
  3. Try to acquire the INP lock.
  4. If successful, drop lock on lookup structure.
  
  Of course, if the INP lock is contested, the functions will need to
  revert to the previous way of switching locks safely.
  
  This saves a few atomic operations when the INP lock is uncontested.
  
  Discussed with:	gallatin, rrs, rwatson
  MFC after:	2 weeks
  Sponsored by:	Netflix, Inc.
  Differential Revision:	https://reviews.freebsd.org/D12911

Modified:
  head/sys/netinet/in_pcb.c
  head/sys/netinet6/in6_pcb.c

Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c	Wed Mar 21 15:43:57 2018	(r331308)
+++ head/sys/netinet/in_pcb.c	Wed Mar 21 15:54:46 2018	(r331309)
@@ -1632,6 +1632,7 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct i
 	struct inpcbhead *head;
 	struct inpcb *inp, *tmpinp;
 	u_short fport = fport_arg, lport = lport_arg;
+	bool locked;
 
 	/*
 	 * First look for an exact match.
@@ -1818,18 +1819,32 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct i
 	return (NULL);
 
 found:
-	in_pcbref(inp);
-	INP_GROUP_UNLOCK(pcbgroup);
-	if (lookupflags & INPLOOKUP_WLOCKPCB) {
-		INP_WLOCK(inp);
-		if (in_pcbrele_wlocked(inp))
-			return (NULL);
-	} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
-		INP_RLOCK(inp);
-		if (in_pcbrele_rlocked(inp))
-			return (NULL);
-	} else
+	if (lookupflags & INPLOOKUP_WLOCKPCB)
+		locked = TRY_INP_WLOCK(inp);
+	else if (lookupflags & INPLOOKUP_RLOCKPCB)
+		locked = TRY_INP_RLOCK(inp);
+	else
 		panic("%s: locking bug", __func__);
+	if (!locked)
+		in_pcbref(inp);
+	INP_GROUP_UNLOCK(pcbgroup);
+	if (!locked) {
+		if (lookupflags & INPLOOKUP_WLOCKPCB) {
+			INP_WLOCK(inp);
+			if (in_pcbrele_wlocked(inp))
+				return (NULL);
+		} else {
+			INP_RLOCK(inp);
+			if (in_pcbrele_rlocked(inp))
+				return (NULL);
+		}
+	}
+#ifdef INVARIANTS
+	if (lookupflags & INPLOOKUP_WLOCKPCB)
+		INP_WLOCK_ASSERT(inp);
+	else
+		INP_RLOCK_ASSERT(inp);
+#endif
 	return (inp);
 }
 #endif /* PCBGROUP */
@@ -1968,23 +1983,38 @@ in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in
     struct ifnet *ifp)
 {
 	struct inpcb *inp;
+	bool locked;
 
 	INP_HASH_RLOCK(pcbinfo);
 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
 	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
 	if (inp != NULL) {
-		in_pcbref(inp);
-		INP_HASH_RUNLOCK(pcbinfo);
-		if (lookupflags & INPLOOKUP_WLOCKPCB) {
-			INP_WLOCK(inp);
-			if (in_pcbrele_wlocked(inp))
-				return (NULL);
-		} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
-			INP_RLOCK(inp);
-			if (in_pcbrele_rlocked(inp))
-				return (NULL);
-		} else
+		if (lookupflags & INPLOOKUP_WLOCKPCB)
+			locked = INP_TRY_WLOCK(inp);
+		else if (lookupflags & INPLOOKUP_RLOCKPCB)
+			locked = INP_TRY_RLOCK(inp);
+		else
 			panic("%s: locking bug", __func__);
+		if (!locked)
+			in_pcbref(inp);
+		INP_HASH_RUNLOCK(pcbinfo);
+		if (!locked) {
+			if (lookupflags & INPLOOKUP_WLOCKPCB) {
+				INP_WLOCK(inp);
+				if (in_pcbrele_wlocked(inp))
+					return (NULL);
+			} else {
+				INP_RLOCK(inp);
+				if (in_pcbrele_rlocked(inp))
+					return (NULL);
+			}
+		}
+#ifdef INVARIANTS
+		if (lookupflags & INPLOOKUP_WLOCKPCB)
+			INP_WLOCK_ASSERT(inp);
+		else
+			INP_RLOCK_ASSERT(inp);
+#endif
 	} else
 		INP_HASH_RUNLOCK(pcbinfo);
 	return (inp);

Modified: head/sys/netinet6/in6_pcb.c
==============================================================================
--- head/sys/netinet6/in6_pcb.c	Wed Mar 21 15:43:57 2018	(r331308)
+++ head/sys/netinet6/in6_pcb.c	Wed Mar 21 15:54:46 2018	(r331309)
@@ -868,6 +868,7 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct 
 	struct inpcbhead *head;
 	struct inpcb *inp, *tmpinp;
 	u_short fport = fport_arg, lport = lport_arg;
+	bool locked;
 
 	/*
 	 * First look for an exact match.
@@ -1026,18 +1027,32 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct 
 	return (NULL);
 
 found:
-	in_pcbref(inp);
-	INP_GROUP_UNLOCK(pcbgroup);
-	if (lookupflags & INPLOOKUP_WLOCKPCB) {
-		INP_WLOCK(inp);
-		if (in_pcbrele_wlocked(inp))
-			return (NULL);
-	} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
-		INP_RLOCK(inp);
-		if (in_pcbrele_rlocked(inp))
-			return (NULL);
-	} else
+	if (lookupflags & INPLOOKUP_WLOCKPCB)
+		locked = INP_TRY_WLOCK(inp);
+	else if (lookupflags & INPLOOKUP_RLOCKPCB)
+		locked = INP_TRY_RLOCK(inp);
+	else
 		panic("%s: locking buf", __func__);
+	if (!locked)
+		in_pcbref(inp);
+	INP_GROUP_UNLOCK(pcbgroup);
+	if (!locked) {
+		if (lookupflags & INPLOOKUP_WLOCKPCB) {
+			INP_WLOCK(inp);
+			if (in_pcbrele_wlocked(inp))
+				return (NULL);
+		} else {
+			INP_RLOCK(inp);
+			if (in_pcbrele_rlocked(inp))
+				return (NULL);
+		}
+	}
+#ifdef INVARIANTS
+	if (lookupflags & INPLOOKUP_WLOCKPCB)
+		INP_WLOCK_ASSERT(inp);
+	else
+		INP_RLOCK_ASSERT(inp);
+#endif
 	return (inp);
 }
 #endif /* PCBGROUP */
@@ -1163,23 +1178,38 @@ in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct i
     struct ifnet *ifp)
 {
 	struct inpcb *inp;
+	bool locked;
 
 	INP_HASH_RLOCK(pcbinfo);
 	inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
 	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
 	if (inp != NULL) {
-		in_pcbref(inp);
-		INP_HASH_RUNLOCK(pcbinfo);
-		if (lookupflags & INPLOOKUP_WLOCKPCB) {
-			INP_WLOCK(inp);
-			if (in_pcbrele_wlocked(inp))
-				return (NULL);
-		} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
-			INP_RLOCK(inp);
-			if (in_pcbrele_rlocked(inp))
-				return (NULL);
-		} else
+		if (lookupflags & INPLOOKUP_WLOCKPCB)
+			locked = INP_TRY_WLOCK(inp);
+		else if (lookupflags & INPLOOKUP_RLOCKPCB)
+			locked = INP_TRY_RLOCK(inp);
+		else
 			panic("%s: locking bug", __func__);
+		if (!locked)
+			in_pcbref(inp);
+		INP_HASH_RUNLOCK(pcbinfo);
+		if (!locked) {
+			if (lookupflags & INPLOOKUP_WLOCKPCB) {
+				INP_WLOCK(inp);
+				if (in_pcbrele_wlocked(inp))
+					return (NULL);
+			} else {
+				INP_RLOCK(inp);
+				if (in_pcbrele_rlocked(inp))
+					return (NULL);
+			}
+		}
+#ifdef INVARIANTS
+		if (lookupflags & INPLOOKUP_WLOCKPCB)
+			INP_WLOCK_ASSERT(inp);
+		else
+			INP_RLOCK_ASSERT(inp);
+#endif
 	} else
 		INP_HASH_RUNLOCK(pcbinfo);
 	return (inp);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803211554.w2LFskv7027707>