Date: Wed, 4 Mar 2009 00:56:48 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r189326 - in user/kmacy/HEAD_fast_net_merge/sys: net netinet Message-ID: <200903040056.n240um2r026157@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Wed Mar 4 00:56:48 2009 New Revision: 189326 URL: http://svn.freebsd.org/changeset/base/189326 Log: IFC HEAD_fast_net inpcb rtentry & llentry caching 186573, 186575, 186576, 186578, 186579, 186580 186573: - Cache rtentry in inpcb for connected sockets 186575: - Add cached llentry to inpcb - reduce header pollution by removing net/route.h from in_pcb.h 186576: - install cached llentry in the inpcb and then pass down to ether_output in the struct route 186578: - don't LLE_FREE if llentry is unchanged 186579: - cleanup upgrade / downgrade checks for route and lle 186580: - don't check INP_WLOCKED if inp is NULL Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h user/kmacy/HEAD_fast_net_merge/sys/net/route.h user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:56:48 2009 (r189326) @@ -669,6 +669,8 @@ struct ifmultiaddr { }; #ifdef _KERNEL +#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET]) + #define IFAFREE(ifa) \ do { \ IFA_LOCK(ifa); \ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:56:48 2009 (r189326) @@ -46,8 +46,9 @@ * in their control blocks, e.g. inpcb. */ struct route { - struct rtentry *ro_rt; struct sockaddr ro_dst; + struct rtentry *ro_rt; + struct llentry *ro_lle; }; /* Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:56:48 2009 (r189326) @@ -81,7 +81,6 @@ __FBSDID("$FreeBSD$"); #define SIN(s) ((struct sockaddr_in *)s) #define SDL(s) ((struct sockaddr_dl *)s) -#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET]) SYSCTL_DECL(_net_link_ether); SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, ""); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:56:48 2009 (r189326) @@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$"); #include <net/if.h> #include <net/if_types.h> +#include <net/if_llatbl.h> #include <net/route.h> #include <netinet/in.h> @@ -497,6 +498,72 @@ in_pcbbind_setup(struct inpcb *inp, stru return (0); } +void +in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro) +{ + struct sockaddr_in *sin; + struct sockaddr *dst; + struct llentry *la; + struct rtentry *rt; + struct ifnet *ifp; + int flags = LLE_EXCLUSIVE; + + INP_WLOCK_ASSERT(inp); + bzero(sro, sizeof(*sro)); + sin = (struct sockaddr_in *)&sro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = faddr; + /* + * If route is known our src addr is taken from the i/f, + * else punt. + * + * Find out route to destination. + */ + if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) { +#ifdef RADIX_MPATH + rtalloc_mpath_fib(sro, ntohl(faddr->s_addr), + inp->inp_inc.inc_fibnum); +#else + in_rtalloc_ign(sro, 0, inp->inp_inc.inc_fibnum); +#endif + } + + rt = sro->ro_rt; + if (rt == NULL) + return; + + inp->inp_rt = rt; + inp->inp_vflag |= INP_RT_VALID; + + if (rt->rt_ifp == NULL) + return; + + ifp = rt->rt_ifp; + dst = &sro->ro_dst; + if (rt->rt_flags & RTF_GATEWAY) + dst = rt->rt_gateway; + + IF_AFDATA_RLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), flags, dst); + IF_AFDATA_RUNLOCK(ifp); + if ((la == NULL) && + (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { + flags |= (LLE_CREATE | LLE_EXCLUSIVE); + IF_AFDATA_WLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), flags, dst); + IF_AFDATA_WUNLOCK(ifp); + } + if (la == NULL) + return; + + LLE_ADDREF(la); + LLE_WUNLOCK(la); + + inp->inp_lle = la; + inp->inp_flags |= INP_LLE_VALID; +} + /* * Connect from a socket to a specified address. * Both address and port must be specified in argument sin. @@ -509,6 +576,7 @@ in_pcbconnect(struct inpcb *inp, struct u_short lport, fport; in_addr_t laddr, faddr; int anonport, error; + struct route sro; INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); @@ -532,6 +600,7 @@ in_pcbconnect(struct inpcb *inp, struct } } + in_pcbrtalloc(inp, faddr, &sro); /* Commit the remaining changes. */ inp->inp_lport = lport; inp->inp_laddr.s_addr = laddr; @@ -870,6 +939,17 @@ in_pcbdisconnect(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + inp->inp_vflag &= ~INP_RT_VALID; + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + } + if (inp->inp_flags & INP_LLE_VALID) { + inp->inp_flags &= ~INP_LLE_VALID; + LLE_FREE(inp->inp_lle); + inp->inp_lle = NULL; + } + inp->inp_faddr.s_addr = INADDR_ANY; inp->inp_fport = 0; in_pcbrehash(inp); @@ -907,6 +987,17 @@ in_pcbfree_internal(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(ipi); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + inp->inp_vflag &= ~INP_RT_VALID; + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + } + if (inp->inp_flags & INP_LLE_VALID) { + inp->inp_flags &= ~INP_LLE_VALID; + LLE_FREE(inp->inp_lle); + inp->inp_lle = NULL; + } + #ifdef IPSEC if (inp->inp_sp != NULL) ipsec_delete_pcbpolicy(inp); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:56:48 2009 (r189326) @@ -39,8 +39,6 @@ #include <sys/_mutex.h> #include <sys/_rwlock.h> -#include <net/route.h> - #ifdef _KERNEL #include <sys/rwlock.h> #endif @@ -49,6 +47,10 @@ #define in6p_sp inp_sp /* for KAME src sync over BSD*'s */ struct inpcbpolicy; +struct rtentry; +struct llentry; +struct route; + /* * struct inpcb is the common protocol control block structure used in most * IP transport protocols. @@ -169,7 +171,8 @@ struct inpcb { u_char inp_ip_minttl; /* (i) minimum TTL or drop */ uint32_t inp_ispare1; /* (x) connection id / queue id */ u_int inp_refcount; /* (i) refcount */ - void *inp_pspare[2]; /* (x) rtentry / general use */ + struct llentry *inp_lle; /* L2 information */ + struct rtentry *inp_rt; /* L3 information */ /* Local and foreign ports, local and foreign addr. */ struct in_conninfo inp_inc; /* (i/p) list for PCB's local port */ @@ -324,6 +327,9 @@ struct inpcbinfo { #define INP_RLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_RLOCKED) #define INP_WLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_WLOCKED) #define INP_UNLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_UNLOCKED) +#define INP_TRY_UPGRADE(inp) rw_try_upgrade(&(inp)->inp_lock) +#define INP_DOWNGRADE(inp) rw_downgrade(&(inp)->inp_lock) +#define INP_WLOCKED(inp) rw_wowned(&(inp)->inp_lock) #ifdef _KERNEL /* @@ -395,6 +401,7 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_ONESBCAST 0x10 /* send all-ones broadcast */ #define INP_DROPPED 0x20 /* protocol drop flag */ #define INP_SOCKREF 0x40 /* strong socket reference */ +#define INP_RT_VALID 0x80 /* rtentry is set */ /* * Flags for inp_flag. @@ -412,7 +419,7 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_RECVTTL 0x400 /* receive incoming IP TTL */ #define INP_DONTFRAG 0x800 /* don't fragment packet */ #define INP_NONLOCALOK 0x1000 /* Allow bind to spoof any address */ - /* - requires options IP_NONLOCALBIND */ +#define INP_LLE_VALID 0x2000 /* L2 entry is set */ #define IN6P_IPV6_V6ONLY 0x008000 /* restrict AF_INET6 socket for v6 */ @@ -489,6 +496,7 @@ void in_pcbdisconnect(struct inpcb *); void in_pcbdrop(struct inpcb *); void in_pcbfree(struct inpcb *); int in_pcbinshash(struct inpcb *); +void in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro); struct inpcb * in_pcblookup_local(struct inpcbinfo *, struct in_addr, u_short, int, struct ucred *); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:56:48 2009 (r189326) @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vimage.h> #include <net/if.h> +#include <net/if_llatbl.h> #include <net/netisr.h> #include <net/pfil.h> #include <net/route.h> @@ -134,11 +135,13 @@ ip_output(struct mbuf *m, struct mbuf *o int hlen = sizeof (struct ip); int mtu; int len, error = 0; + int neednewroute = 0, neednewlle = 0; struct sockaddr_in *dst = NULL; /* keep compiler happy */ struct in_ifaddr *ia = NULL; int isbroadcast, sw_csum; struct route iproute; struct in_addr odst; + struct sockaddr_in *sin; #ifdef IPFIREWALL_FORWARD struct m_tag *fwd_tag = NULL; #endif @@ -152,6 +155,22 @@ ip_output(struct mbuf *m, struct mbuf *o if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); INP_LOCK_ASSERT(inp); + if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { + if (inp->inp_rt->rt_flags & RTF_UP) { + sin = (struct sockaddr_in *)&ro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = inp->inp_faddr.s_addr; + ro->ro_rt = inp->inp_rt; + } else + neednewroute = 1; + } + if ((ro == &iproute) && (inp->inp_flags & INP_LLE_VALID)) { + if (inp->inp_lle->la_flags & LLE_VALID) { + ro->ro_lle = inp->inp_lle; + } else + neednewlle = 1; + } } if (opt) { @@ -194,7 +213,8 @@ again: if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || dst->sin_family != AF_INET || dst->sin_addr.s_addr != ip->ip_dst.s_addr)) { - RTFREE(ro->ro_rt); + if (inp == NULL || (ro->ro_rt != inp->inp_rt)) + RTFREE(ro->ro_rt); ro->ro_rt = (struct rtentry *)NULL; } #ifdef IPFIREWALL_FORWARD @@ -637,9 +657,48 @@ passout: V_ipstat.ips_fragmented++; done: - if (ro == &iproute && ro->ro_rt) { - RTFREE(ro->ro_rt); + if (ro == &iproute && ro->ro_rt != NULL) { + int wlocked = 0; + struct llentry *la; + + if (neednewlle || neednewroute) { + wlocked = INP_WLOCKED(inp); + if (!wlocked && INP_TRY_UPGRADE(inp) == 0) + return (error); + } + + if (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0) + RTFREE(ro->ro_rt); + else if (neednewroute && ro->ro_rt != inp->inp_rt) { + RTFREE(inp->inp_rt); + inp->inp_rt = ro->ro_rt; + + } + if (neednewlle) { + IF_AFDATA_RLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, + (struct sockaddr *)dst); + IF_AFDATA_RUNLOCK(ifp); + if ((la == NULL) && + (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { + IF_AFDATA_WLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), + (LLE_CREATE | LLE_EXCLUSIVE), + (struct sockaddr *)dst); + IF_AFDATA_WUNLOCK(ifp); + } + if (la != NULL && (inp->inp_lle != la)) { + LLE_FREE(inp->inp_lle); + LLE_ADDREF(la); + LLE_WUNLOCK(la); + inp->inp_lle = la; + } else if (la != NULL) + LLE_WUNLOCK(la); + } + if ((neednewlle || neednewroute) && !wlocked) + INP_DOWNGRADE(inp); } + return (error); bad: m_freem(m); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:56:48 2009 (r189326) @@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vimage.h> #include <net/if.h> +#include <net/route.h> #include <netinet/in.h> #include <netinet/in_systm.h> Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:56:48 2009 (r189326) @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <net/if.h> #include <net/if_types.h> #include <net/if_var.h> +#include <net/route.h> #include <netinet/in.h> #include <netinet/in_systm.h> Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:56:48 2009 (r189326) @@ -1075,6 +1075,7 @@ tcp_connect(struct tcpcb *tp, struct soc struct socket *so = inp->inp_socket; INIT_VNET_INET(so->so_vnet); struct in_addr laddr; + struct route sro; u_short lport; int error; @@ -1103,6 +1104,7 @@ tcp_connect(struct tcpcb *tp, struct soc inp->inp_laddr = laddr; in_pcbrehash(inp); + in_pcbrtalloc(inp, inp->inp_faddr.s_addr, &sro); /* * Compute window scaling to request: * Scale to fit into sweet spot. See tcp_syncache.c.
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200903040056.n240um2r026157>