Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 15 Jun 1999 23:18:24 +0100
From:      Ian Dowse <iedowse@maths.tcd.ie>
To:        freebsd-current@freebsd.org
Cc:        iedowse@maths.tcd.ie
Subject:   NFS with IP aliases or multiple interfaces
Message-ID:   <9906152318.aa15606@salmon.maths.tcd.ie>

next in thread | raw e-mail | index | archive | help

In certain setups with multiple network interfaces or IP aliases, nfsd(8)
can send a reply using a source IP address that is not the same as the
destination address of the request. This is explained in more detail in
PRs 2858, 5964, 6412 and 9612.

We just came across this problem locally, so I decided to attempt a fix. The
patches included below appear to solve the problem in our configuration, but
it has not been extensively tested, so I would be interested in any comments
or suggestions. I'm particularly unsure about the new get_ifaddrs() function
in nfsd.c, as much of it was copied from rwhod.c.

(These patches are against -stable, and while they appear to apply cleanly
to -current, I haven't actually tested them there)

Essentially these changes cause nfsd to register one UDP socket for each
interface as well as the wildcard socket. On the kernel side, I have
removed 'nfs_udpsock' completely, because there is no longer just one UDP
socket.

Ian

--- nfs_syscalls.c.orig	Tue Jun 15 21:07:49 1999
+++ nfs_syscalls.c	Tue Jun 15 22:22:48 1999
@@ -86,7 +86,7 @@
 extern struct nfsstats nfsstats;
 extern int nfsrvw_procrastinate;
 extern int nfsrvw_procrastinate_v3;
-struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock;
+struct nfssvc_sock *nfs_cltpsock;
 static int nuidhash_max = NFS_MAXUIDHASH;
 
 static void	nfsrv_zapsock __P((struct nfssvc_sock *slp));
@@ -368,28 +368,24 @@
 	/*
 	 * Add it to the list, as required.
 	 */
-	if (so->so_proto->pr_protocol == IPPROTO_UDP) {
-		tslp = nfs_udpsock;
-		if (tslp->ns_flag & SLP_VALID) {
-			FREE(mynam, M_SONAME);
-			return (EPERM);
-		}
 #ifdef ISO
-	} else if (so->so_proto->pr_protocol == ISOPROTO_CLTP) {
+	if (so->so_proto->pr_protocol == ISOPROTO_CLTP) {
 		tslp = nfs_cltpsock;
 		if (tslp->ns_flag & SLP_VALID) {
-			FREE(mynam, M_SONAME);
+			if (mynam != NULL)
+				FREE(mynam, M_SONAME);
 			return (EPERM);
 		}
-#endif /* ISO */
 	}
+#endif /* ISO */
 	if (so->so_type == SOCK_STREAM)
 		siz = NFS_MAXPACKET + sizeof (u_long);
 	else
 		siz = NFS_MAXPACKET;
 	error = soreserve(so, siz, siz);
 	if (error) {
-		FREE(mynam, M_SONAME);
+		if (mynam != NULL)
+			FREE(mynam, M_SONAME);
 		return (error);
 	}
 
@@ -867,13 +863,6 @@
 
 	TAILQ_INIT(&nfsd_head);
 	nfsd_head_flag &= ~NFSD_CHECKSLP;
-
-	nfs_udpsock = (struct nfssvc_sock *)
-	    malloc(sizeof (struct nfssvc_sock), M_NFSSVC, M_WAITOK);
-	bzero((caddr_t)nfs_udpsock, sizeof (struct nfssvc_sock));
-	STAILQ_INIT(&nfs_udpsock->ns_rec);
-	TAILQ_INIT(&nfs_udpsock->ns_uidlruhead);
-	TAILQ_INSERT_HEAD(&nfssvc_sockhead, nfs_udpsock, ns_chain);
 
 	nfs_cltpsock = (struct nfssvc_sock *)
 	    malloc(sizeof (struct nfssvc_sock), M_NFSSVC, M_WAITOK);
--- nfs_nqlease.c.orig	Tue Jun 15 21:07:39 1999
+++ nfs_nqlease.c	Tue Jun 15 21:23:21 1999
@@ -134,7 +134,7 @@
 
 extern nfstype nfsv2_type[9];
 extern nfstype nfsv3_type[9];
-extern struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock;
+extern struct nfssvc_sock *nfs_cltpsock;
 extern int nfsd_waiting;
 extern struct nfsstats nfsstats;
 
@@ -381,11 +381,13 @@
 
 	if (slp == NQLOCALSLP)
 		lph->lph_flag |= (LC_VALID | LC_LOCAL);
-	else if (slp == nfs_udpsock) {
+	else if (slp->ns_so->so_proto->pr_protocol == IPPROTO_UDP) {
 		saddr = (struct sockaddr_in *)nam;
-		lph->lph_flag |= (LC_VALID | LC_UDP);
+		lph->lph_flag |= (LC_VALID | LC_UDP | LC_SREF);
 		lph->lph_inetaddr = saddr->sin_addr.s_addr;
 		lph->lph_port = saddr->sin_port;
+		lph->lph_slp = slp;
+		slp->ns_sref++;
 	} else if (slp == nfs_cltpsock) {
 		lph->lph_nam = dup_sockaddr(nam, 1);
 		lph->lph_flag |= (LC_VALID | LC_CLTP);
@@ -455,7 +457,8 @@
 		else
 			return (0);
 	}
-	if (slp == nfs_udpsock || slp == nfs_cltpsock)
+	if (slp->ns_so->so_proto->pr_protocol == IPPROTO_UDP ||
+	    slp == nfs_cltpsock)
 		addr = nam;
 	else
 		addr = slp->ns_nam;
@@ -514,7 +517,7 @@
 				saddr->sin_family = AF_INET;
 				saddr->sin_addr.s_addr = lph->lph_inetaddr;
 				saddr->sin_port = lph->lph_port;
-				so = nfs_udpsock->ns_so;
+				so = lph->lph_slp->ns_so;
 			} else if (lph->lph_flag & LC_CLTP) {
 				nam2 = lph->lph_nam;
 				so = nfs_cltpsock->ns_so;
--- nfsd.c.orig	Tue Jun 15 14:15:50 1999
+++ nfsd.c	Tue Jun 15 22:31:50 1999
@@ -52,6 +52,7 @@
 #include <sys/syslog.h>
 #include <sys/wait.h>
 #include <sys/mount.h>
+#include <sys/sysctl.h>
 
 #include <rpc/rpc.h>
 #include <rpc/pmap_clnt.h>
@@ -62,12 +63,17 @@
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
+#include <net/if.h>
+#include <net/route.h>
+#include <arpa/inet.h>
+
 
 #ifdef NFSKERB
 #include <kerberosIV/des.h>
 #include <kerberosIV/krb.h>
 #endif
 
+
 #include <err.h>
 #include <errno.h>
 #include <libutil.h>
@@ -84,6 +90,10 @@
 int	debug = 0;
 #endif
 
+struct in_addr *ifaddr;
+int ifaddr_count;
+
+
 struct	nfsd_srvargs nsd;
 #ifdef OLD_SETPROCTITLE
 char	**Argv = NULL;		/* pointer to argument vector */
@@ -110,6 +120,7 @@
 #endif
 #endif
 void	usage __P((void));
+void	get_ifaddrs __P((void));
 
 /*
  * Nfs server daemon mostly just a user context for nfssvc()
@@ -143,7 +154,7 @@
 #endif
 	fd_set ready, sockbits;
 	int ch, cltpflag, connect_type_cnt, i, len, maxsock, msgsock;
-	int nfsdcnt, nfssvc_flag, on, reregister, sock, tcpflag, tcpsock;
+	int nfsdcnt, nfssvc_flag, on, reregister, tcpflag, tcpsock;
 	int tp4cnt, tp4flag, tpipcnt, tpipflag, udpflag;
 #ifdef notyet
 	int tp4sock, tpipsock;
@@ -375,32 +386,55 @@
 
 	/* If we are serving udp, set up the socket. */
 	if (udpflag) {
-		if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
-			syslog(LOG_ERR, "can't create udp socket");
-			exit(1);
-		}
-		inetaddr.sin_family = AF_INET;
-		inetaddr.sin_addr.s_addr = INADDR_ANY;
-		inetaddr.sin_port = htons(NFS_PORT);
-		inetaddr.sin_len = sizeof(inetaddr);
-		if (bind(sock,
-		    (struct sockaddr *)&inetaddr, sizeof(inetaddr)) < 0) {
-			syslog(LOG_ERR, "can't bind udp addr");
-			exit(1);
+		int i;
+		int *sock;
+
+		get_ifaddrs();
+
+		sock = malloc(ifaddr_count * sizeof(*sock));
+
+		for (i = 0; i < ifaddr_count; i++) {
+			int on = 1;
+
+			if ((sock[i] = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
+				syslog(LOG_ERR, "can't create udp socket: %m");
+				exit(1);
+			}
+			if (setsockopt(sock[i], SOL_SOCKET, SO_REUSEADDR,
+			    &on, sizeof(on)) != 0) {
+				syslog(LOG_ERR, "setsockopt failed: %m");
+				exit(1);
+			}
+			bzero(&inetaddr, sizeof(inetaddr));
+			inetaddr.sin_family = AF_INET;
+			inetaddr.sin_addr.s_addr = ifaddr[i].s_addr;
+			inetaddr.sin_port = htons(NFS_PORT);
+			inetaddr.sin_len = sizeof(inetaddr);
+			if (bind(sock[i], (struct sockaddr *)&inetaddr,
+			    sizeof(inetaddr)) < 0) {
+				syslog(LOG_ERR, "can't bind udp addr %s: %m",
+				    inet_ntoa(ifaddr[i]));
+				exit(1);
+			}
 		}
 		if (!pmap_set(RPCPROG_NFS, 2, IPPROTO_UDP, NFS_PORT) ||
 		    !pmap_set(RPCPROG_NFS, 3, IPPROTO_UDP, NFS_PORT)) {
 			syslog(LOG_ERR, "can't register with udp portmap");
 			exit(1);
 		}
-		nfsdargs.sock = sock;
-		nfsdargs.name = NULL;
-		nfsdargs.namelen = 0;
-		if (nfssvc(NFSSVC_ADDSOCK, &nfsdargs) < 0) {
-			syslog(LOG_ERR, "can't Add UDP socket");
-			exit(1);
+
+		for (i = 0; i < ifaddr_count; i++) {
+			nfsdargs.sock = sock[i];
+			nfsdargs.name = NULL;
+			nfsdargs.namelen = 0;
+			if (nfssvc(NFSSVC_ADDSOCK, &nfsdargs) < 0) {
+				syslog(LOG_ERR, "can't Add UDP socket");
+				exit(1);
+			}
+			(void)close(sock[i]);
 		}
-		(void)close(sock);
+
+		free(sock);
 	}
 
 #ifdef ISO
@@ -671,3 +705,87 @@
 }
 #endif	/* __FreeBSD__ */
 #endif
+
+
+
+#define ROUNDUP(a) \
+	((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
+#define ADVANCE(x, n) (x += ROUNDUP((n)->sa_len))
+
+/* Get a list of local IP addresses, and store them in ifaddr[].
+ * The first address will be INADDR_ANY as a fallback entry.
+ * The number of entries in the list is stored in ifaddr_count.
+ */
+void get_ifaddrs() {
+	register struct if_msghdr *ifm;
+	register struct ifa_msghdr *ifam;
+	size_t needed;
+	int mib[6], flags = 0;
+	char *buf, *lim, *next;
+	struct rt_addrinfo info;
+
+	/* Create first address as wildcard */
+	ifaddr = realloc(ifaddr, sizeof(*ifaddr) * (ifaddr_count + 1));
+	ifaddr[ifaddr_count++].s_addr = INADDR_ANY;
+
+	mib[0] = CTL_NET;
+	mib[1] = PF_ROUTE;
+	mib[2] = 0;
+	mib[3] = AF_INET;
+	mib[4] = NET_RT_IFLIST;
+	mib[5] = 0;
+	if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
+		errx(1, "route-sysctl-estimate");
+	if ((buf = malloc(needed)) == NULL)
+		errx(1, "malloc");
+	if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0)
+		errx(1, "actual retrieval of interface table");
+	lim = buf + needed;
+
+	for (next = buf; next < lim; next += ifm->ifm_msglen) {
+		int i;
+		char *cp, *cplim;
+		struct sockaddr *saa[RTAX_MAX], *sa;
+
+		ifm = (struct if_msghdr *)next;
+		if (ifm->ifm_type == RTM_IFINFO) {
+			flags = ifm->ifm_flags;
+			continue;
+		}
+		if ((flags & IFF_UP) == 0)
+			continue;
+		if (ifm->ifm_type != RTM_NEWADDR)
+			errx(1, "out of sync parsing NET_RT_IFLIST");
+		ifam = (struct ifa_msghdr *)ifm;
+		info.rti_addrs = ifam->ifam_addrs;
+		cp = (char *)(ifam + 1);
+		cplim = ifam->ifam_msglen + (char *)ifam;
+
+		bzero(saa, sizeof(saa));
+		for (i = 0; i < RTAX_MAX && cp < cplim; i++) {
+			saa[i] = (struct sockaddr *)cp;
+			ADVANCE(cp, saa[i]);
+		}
+
+		sa = saa[RTAX_GATEWAY];
+		if (sa == 0 || sa->sa_family != AF_INET)
+			continue;
+
+#define SA2SIN(sa) (((struct sockaddr_in *)(sa))->sin_addr)
+
+		/* Check if we've seen this address before */
+		for (i = 0; i < ifaddr_count; i++)
+			if (ifaddr[i].s_addr == SA2SIN(sa).s_addr) {
+				sa = NULL;
+				break;
+			}
+
+		if (sa == NULL)
+			continue;
+
+		ifaddr = realloc(ifaddr, sizeof(*ifaddr) * (ifaddr_count + 1));
+		ifaddr[ifaddr_count] = SA2SIN(sa);
+		ifaddr_count++;
+	}
+	free(buf);
+}


To Unsubscribe: send mail to majordomo@FreeBSD.org
with "unsubscribe freebsd-current" in the body of the message




Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi? <9906152318.aa15606>