From owner-freebsd-current Tue Jun 15 15:18:34 1999 Delivered-To: freebsd-current@freebsd.org Received: from salmon.maths.tcd.ie (salmon.maths.tcd.ie [134.226.81.11]) by hub.freebsd.org (Postfix) with SMTP id 5FFC814D84 for ; Tue, 15 Jun 1999 15:18:25 -0700 (PDT) (envelope-from iedowse@maths.tcd.ie) Received: from bell.maths.tcd.ie by salmon.maths.tcd.ie with SMTP id ; 15 Jun 99 23:18:24 +0100 (BST) To: freebsd-current@freebsd.org Cc: iedowse@maths.tcd.ie Subject: NFS with IP aliases or multiple interfaces Date: Tue, 15 Jun 1999 23:18:24 +0100 From: Ian Dowse Message-ID: <9906152318.aa15606@salmon.maths.tcd.ie> Sender: owner-freebsd-current@FreeBSD.ORG Precedence: bulk X-Loop: FreeBSD.ORG In certain setups with multiple network interfaces or IP aliases, nfsd(8) can send a reply using a source IP address that is not the same as the destination address of the request. This is explained in more detail in PRs 2858, 5964, 6412 and 9612. We just came across this problem locally, so I decided to attempt a fix. The patches included below appear to solve the problem in our configuration, but it has not been extensively tested, so I would be interested in any comments or suggestions. I'm particularly unsure about the new get_ifaddrs() function in nfsd.c, as much of it was copied from rwhod.c. (These patches are against -stable, and while they appear to apply cleanly to -current, I haven't actually tested them there) Essentially these changes cause nfsd to register one UDP socket for each interface as well as the wildcard socket. On the kernel side, I have removed 'nfs_udpsock' completely, because there is no longer just one UDP socket. Ian --- nfs_syscalls.c.orig Tue Jun 15 21:07:49 1999 +++ nfs_syscalls.c Tue Jun 15 22:22:48 1999 @@ -86,7 +86,7 @@ extern struct nfsstats nfsstats; extern int nfsrvw_procrastinate; extern int nfsrvw_procrastinate_v3; -struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock; +struct nfssvc_sock *nfs_cltpsock; static int nuidhash_max = NFS_MAXUIDHASH; static void nfsrv_zapsock __P((struct nfssvc_sock *slp)); @@ -368,28 +368,24 @@ /* * Add it to the list, as required. */ - if (so->so_proto->pr_protocol == IPPROTO_UDP) { - tslp = nfs_udpsock; - if (tslp->ns_flag & SLP_VALID) { - FREE(mynam, M_SONAME); - return (EPERM); - } #ifdef ISO - } else if (so->so_proto->pr_protocol == ISOPROTO_CLTP) { + if (so->so_proto->pr_protocol == ISOPROTO_CLTP) { tslp = nfs_cltpsock; if (tslp->ns_flag & SLP_VALID) { - FREE(mynam, M_SONAME); + if (mynam != NULL) + FREE(mynam, M_SONAME); return (EPERM); } -#endif /* ISO */ } +#endif /* ISO */ if (so->so_type == SOCK_STREAM) siz = NFS_MAXPACKET + sizeof (u_long); else siz = NFS_MAXPACKET; error = soreserve(so, siz, siz); if (error) { - FREE(mynam, M_SONAME); + if (mynam != NULL) + FREE(mynam, M_SONAME); return (error); } @@ -867,13 +863,6 @@ TAILQ_INIT(&nfsd_head); nfsd_head_flag &= ~NFSD_CHECKSLP; - - nfs_udpsock = (struct nfssvc_sock *) - malloc(sizeof (struct nfssvc_sock), M_NFSSVC, M_WAITOK); - bzero((caddr_t)nfs_udpsock, sizeof (struct nfssvc_sock)); - STAILQ_INIT(&nfs_udpsock->ns_rec); - TAILQ_INIT(&nfs_udpsock->ns_uidlruhead); - TAILQ_INSERT_HEAD(&nfssvc_sockhead, nfs_udpsock, ns_chain); nfs_cltpsock = (struct nfssvc_sock *) malloc(sizeof (struct nfssvc_sock), M_NFSSVC, M_WAITOK); --- nfs_nqlease.c.orig Tue Jun 15 21:07:39 1999 +++ nfs_nqlease.c Tue Jun 15 21:23:21 1999 @@ -134,7 +134,7 @@ extern nfstype nfsv2_type[9]; extern nfstype nfsv3_type[9]; -extern struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock; +extern struct nfssvc_sock *nfs_cltpsock; extern int nfsd_waiting; extern struct nfsstats nfsstats; @@ -381,11 +381,13 @@ if (slp == NQLOCALSLP) lph->lph_flag |= (LC_VALID | LC_LOCAL); - else if (slp == nfs_udpsock) { + else if (slp->ns_so->so_proto->pr_protocol == IPPROTO_UDP) { saddr = (struct sockaddr_in *)nam; - lph->lph_flag |= (LC_VALID | LC_UDP); + lph->lph_flag |= (LC_VALID | LC_UDP | LC_SREF); lph->lph_inetaddr = saddr->sin_addr.s_addr; lph->lph_port = saddr->sin_port; + lph->lph_slp = slp; + slp->ns_sref++; } else if (slp == nfs_cltpsock) { lph->lph_nam = dup_sockaddr(nam, 1); lph->lph_flag |= (LC_VALID | LC_CLTP); @@ -455,7 +457,8 @@ else return (0); } - if (slp == nfs_udpsock || slp == nfs_cltpsock) + if (slp->ns_so->so_proto->pr_protocol == IPPROTO_UDP || + slp == nfs_cltpsock) addr = nam; else addr = slp->ns_nam; @@ -514,7 +517,7 @@ saddr->sin_family = AF_INET; saddr->sin_addr.s_addr = lph->lph_inetaddr; saddr->sin_port = lph->lph_port; - so = nfs_udpsock->ns_so; + so = lph->lph_slp->ns_so; } else if (lph->lph_flag & LC_CLTP) { nam2 = lph->lph_nam; so = nfs_cltpsock->ns_so; --- nfsd.c.orig Tue Jun 15 14:15:50 1999 +++ nfsd.c Tue Jun 15 22:31:50 1999 @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -62,12 +63,17 @@ #include #include #include +#include +#include +#include + #ifdef NFSKERB #include #include #endif + #include #include #include @@ -84,6 +90,10 @@ int debug = 0; #endif +struct in_addr *ifaddr; +int ifaddr_count; + + struct nfsd_srvargs nsd; #ifdef OLD_SETPROCTITLE char **Argv = NULL; /* pointer to argument vector */ @@ -110,6 +120,7 @@ #endif #endif void usage __P((void)); +void get_ifaddrs __P((void)); /* * Nfs server daemon mostly just a user context for nfssvc() @@ -143,7 +154,7 @@ #endif fd_set ready, sockbits; int ch, cltpflag, connect_type_cnt, i, len, maxsock, msgsock; - int nfsdcnt, nfssvc_flag, on, reregister, sock, tcpflag, tcpsock; + int nfsdcnt, nfssvc_flag, on, reregister, tcpflag, tcpsock; int tp4cnt, tp4flag, tpipcnt, tpipflag, udpflag; #ifdef notyet int tp4sock, tpipsock; @@ -375,32 +386,55 @@ /* If we are serving udp, set up the socket. */ if (udpflag) { - if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { - syslog(LOG_ERR, "can't create udp socket"); - exit(1); - } - inetaddr.sin_family = AF_INET; - inetaddr.sin_addr.s_addr = INADDR_ANY; - inetaddr.sin_port = htons(NFS_PORT); - inetaddr.sin_len = sizeof(inetaddr); - if (bind(sock, - (struct sockaddr *)&inetaddr, sizeof(inetaddr)) < 0) { - syslog(LOG_ERR, "can't bind udp addr"); - exit(1); + int i; + int *sock; + + get_ifaddrs(); + + sock = malloc(ifaddr_count * sizeof(*sock)); + + for (i = 0; i < ifaddr_count; i++) { + int on = 1; + + if ((sock[i] = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { + syslog(LOG_ERR, "can't create udp socket: %m"); + exit(1); + } + if (setsockopt(sock[i], SOL_SOCKET, SO_REUSEADDR, + &on, sizeof(on)) != 0) { + syslog(LOG_ERR, "setsockopt failed: %m"); + exit(1); + } + bzero(&inetaddr, sizeof(inetaddr)); + inetaddr.sin_family = AF_INET; + inetaddr.sin_addr.s_addr = ifaddr[i].s_addr; + inetaddr.sin_port = htons(NFS_PORT); + inetaddr.sin_len = sizeof(inetaddr); + if (bind(sock[i], (struct sockaddr *)&inetaddr, + sizeof(inetaddr)) < 0) { + syslog(LOG_ERR, "can't bind udp addr %s: %m", + inet_ntoa(ifaddr[i])); + exit(1); + } } if (!pmap_set(RPCPROG_NFS, 2, IPPROTO_UDP, NFS_PORT) || !pmap_set(RPCPROG_NFS, 3, IPPROTO_UDP, NFS_PORT)) { syslog(LOG_ERR, "can't register with udp portmap"); exit(1); } - nfsdargs.sock = sock; - nfsdargs.name = NULL; - nfsdargs.namelen = 0; - if (nfssvc(NFSSVC_ADDSOCK, &nfsdargs) < 0) { - syslog(LOG_ERR, "can't Add UDP socket"); - exit(1); + + for (i = 0; i < ifaddr_count; i++) { + nfsdargs.sock = sock[i]; + nfsdargs.name = NULL; + nfsdargs.namelen = 0; + if (nfssvc(NFSSVC_ADDSOCK, &nfsdargs) < 0) { + syslog(LOG_ERR, "can't Add UDP socket"); + exit(1); + } + (void)close(sock[i]); } - (void)close(sock); + + free(sock); } #ifdef ISO @@ -671,3 +705,87 @@ } #endif /* __FreeBSD__ */ #endif + + + +#define ROUNDUP(a) \ + ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long)) +#define ADVANCE(x, n) (x += ROUNDUP((n)->sa_len)) + +/* Get a list of local IP addresses, and store them in ifaddr[]. + * The first address will be INADDR_ANY as a fallback entry. + * The number of entries in the list is stored in ifaddr_count. + */ +void get_ifaddrs() { + register struct if_msghdr *ifm; + register struct ifa_msghdr *ifam; + size_t needed; + int mib[6], flags = 0; + char *buf, *lim, *next; + struct rt_addrinfo info; + + /* Create first address as wildcard */ + ifaddr = realloc(ifaddr, sizeof(*ifaddr) * (ifaddr_count + 1)); + ifaddr[ifaddr_count++].s_addr = INADDR_ANY; + + mib[0] = CTL_NET; + mib[1] = PF_ROUTE; + mib[2] = 0; + mib[3] = AF_INET; + mib[4] = NET_RT_IFLIST; + mib[5] = 0; + if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0) + errx(1, "route-sysctl-estimate"); + if ((buf = malloc(needed)) == NULL) + errx(1, "malloc"); + if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) + errx(1, "actual retrieval of interface table"); + lim = buf + needed; + + for (next = buf; next < lim; next += ifm->ifm_msglen) { + int i; + char *cp, *cplim; + struct sockaddr *saa[RTAX_MAX], *sa; + + ifm = (struct if_msghdr *)next; + if (ifm->ifm_type == RTM_IFINFO) { + flags = ifm->ifm_flags; + continue; + } + if ((flags & IFF_UP) == 0) + continue; + if (ifm->ifm_type != RTM_NEWADDR) + errx(1, "out of sync parsing NET_RT_IFLIST"); + ifam = (struct ifa_msghdr *)ifm; + info.rti_addrs = ifam->ifam_addrs; + cp = (char *)(ifam + 1); + cplim = ifam->ifam_msglen + (char *)ifam; + + bzero(saa, sizeof(saa)); + for (i = 0; i < RTAX_MAX && cp < cplim; i++) { + saa[i] = (struct sockaddr *)cp; + ADVANCE(cp, saa[i]); + } + + sa = saa[RTAX_GATEWAY]; + if (sa == 0 || sa->sa_family != AF_INET) + continue; + +#define SA2SIN(sa) (((struct sockaddr_in *)(sa))->sin_addr) + + /* Check if we've seen this address before */ + for (i = 0; i < ifaddr_count; i++) + if (ifaddr[i].s_addr == SA2SIN(sa).s_addr) { + sa = NULL; + break; + } + + if (sa == NULL) + continue; + + ifaddr = realloc(ifaddr, sizeof(*ifaddr) * (ifaddr_count + 1)); + ifaddr[ifaddr_count] = SA2SIN(sa); + ifaddr_count++; + } + free(buf); +} To Unsubscribe: send mail to majordomo@FreeBSD.org with "unsubscribe freebsd-current" in the body of the message