Date: Thu, 17 Mar 2005 01:10:38 +0900 From: Hajimu UMEMOTO <ume@freebsd.org> To: Mike Tancsa <mike@sentex.net> Cc: freebsd-stable@freebsd.org Subject: Re: RELENG_5 and FAST_IPSEC limits Message-ID: <ygek6o7inb5.wl%ume@mahoroba.org> In-Reply-To: <4237523B.7090005@errno.com> References: <6.2.1.2.0.20050315112131.054b56f8@64.7.153.2> <4237523B.7090005@errno.com>
next in thread | previous in thread | raw e-mail | index | archive | help
--Multipart_Thu_Mar_17_01:10:38_2005-1
Content-Type: text/plain; charset=US-ASCII
Hi,
>>>>> On Tue, 15 Mar 2005 13:23:07 -0800
>>>>> Sam Leffler <sam@errno.com> said:
> We are running into a case where there are too many SAs, and doing a
> setkey -D would fail with a
>
> "recv: Resource temporarily unavailable"
>
> after displaying most of the associations.
>
> Is there a way to get around this, or is there a hard limit ?
>
> # setkey -D | grep ^172 | wc
> 186 372 5096
>
> When the remotes are renegotiating, and there are a lot of tunnels in
> the state of mature and dying, this number can go up to 341, but not
> higher. This also seems to send racoon into a hung state that we then
> need to kill off and restart.
>
> It was suggested in a post that /usr/src/sys/net/raw_cb.h get changed from
>
>
> #define RAWSNDQ 8192
> #define RAWRCVQ 8192
>
> to something larger like
>
> #define RAWSNDQ 24576
> #define RAWRCVQ 24576
>
> If this is the underlying issue, will it work on its own, or are there
> other values that need to be tuned ? Will I need to recompile any
> userland apps (e.g. racoon, setkey) and are there any other values I
> would need to adjust
sam> Looks like you're hitting the limit on returning status information
sam> through a PF_KEY socket. FWIW this is not related to FAST_IPSEC; it's
sam> an issue with PF_KEY and is common to both IPsec implementations.
sam> Upping the raw socket buffer sizes should permit more information to be
sam> returned but you may always exceed this limit as you can create more
sam> SA's than can be reported in a single msg. Some groups have dealt with
sam> this by changing the PF_KEY api, e.g. to report an incomplete msg so the
sam> user-mode app can retrieve more data with additional reads. If upping
sam> the socket buffer limits doesn't help then you might search for patches.
KAME/NetBSD does it throuth sysctl. Since sysctl API is slightly
different between NetBSD and FreeBSD, I didn't merge it, yet.
Please try attached patch. This is mainly taken from NetBSD.
There is corresponding code in racoon, already. Please make sure to
copy sys/netkey/key_var.h into /usr/include/netkey/ before recompiling
racoon.
Sincerely,
--Multipart_Thu_Mar_17_01:10:38_2005-1
Content-Type: text/plain; type=patch; charset=US-ASCII
Content-Disposition: attachment; filename="keyctl_dump.diff"
Content-Transfer-Encoding: 7bit
Index: sys/netipsec/key.c
diff -u -p sys/netipsec/key.c.orig sys/netipsec/key.c
--- sys/netipsec/key.c.orig Mon Feb 28 05:34:43 2005
+++ sys/netipsec/key.c Thu Mar 17 00:44:24 2005
@@ -392,6 +392,7 @@ static int key_spdflush __P((struct sock
const struct sadb_msghdr *));
static int key_spddump __P((struct socket *, struct mbuf *,
const struct sadb_msghdr *));
+static struct mbuf *key_setspddump __P((int *));
static struct mbuf *key_setdumpsp __P((struct secpolicy *,
u_int8_t, u_int32_t, u_int32_t));
static u_int key_getspreqmsglen __P((struct secpolicy *));
@@ -483,6 +484,7 @@ static int key_flush __P((struct socket
const struct sadb_msghdr *));
static int key_dump __P((struct socket *, struct mbuf *,
const struct sadb_msghdr *));
+static struct mbuf *key_setdump __P((u_int8_t, int *));
static int key_promisc __P((struct socket *, struct mbuf *,
const struct sadb_msghdr *));
static int key_senderror __P((struct socket *, struct mbuf *, int));
@@ -2396,6 +2398,52 @@ key_spddump(so, m, mhp)
}
static struct mbuf *
+key_setspddump(errorp)
+ int *errorp;
+{
+ struct secpolicy *sp;
+ int cnt;
+ u_int dir;
+ struct mbuf *m, *n;
+
+ /* search SPD entry and get buffer size. */
+ cnt = 0;
+ for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ LIST_FOREACH(sp, &sptree[dir], chain) {
+ cnt++;
+ }
+ }
+
+ if (cnt == 0) {
+ *errorp = ENOENT;
+ return (NULL);
+ }
+
+ m = NULL;
+ for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ LIST_FOREACH(sp, &sptree[dir], chain) {
+ --cnt;
+ n = key_setdumpsp(sp, SADB_X_SPDDUMP, cnt, 0);
+
+ if (!n) {
+ *errorp = ENOBUFS;
+ m_freem(m);
+ return (NULL);
+ }
+ if (!m)
+ m = n;
+ else {
+ m->m_pkthdr.len += n->m_pkthdr.len;
+ m_cat(m, n);
+ }
+ }
+ }
+
+ *errorp = 0;
+ return (m);
+}
+
+static struct mbuf *
key_setdumpsp(sp, type, seq, pid)
struct secpolicy *sp;
u_int8_t type;
@@ -6502,6 +6550,98 @@ key_dump(so, m, mhp)
return 0;
}
+static struct mbuf *
+key_setdump(req_satype, errorp)
+ u_int8_t req_satype;
+ int *errorp;
+{
+ struct secashead *sah;
+ struct secasvar *sav;
+ u_int16_t proto;
+ u_int stateidx;
+ u_int8_t satype;
+ u_int8_t state;
+ int cnt;
+ struct mbuf *m, *n;
+
+ /* map satype to proto */
+ if ((proto = key_satype2proto(req_satype)) == 0) {
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ /* count sav entries to be sent to the userland. */
+ cnt = 0;
+ LIST_FOREACH(sah, &sahtree, chain) {
+ if (req_satype != SADB_SATYPE_UNSPEC &&
+ proto != sah->saidx.proto)
+ continue;
+
+ for (stateidx = 0;
+ stateidx < _ARRAYLEN(saorder_state_any);
+ stateidx++) {
+ state = saorder_state_any[stateidx];
+ LIST_FOREACH(sav, &sah->savtree[state], chain) {
+ cnt++;
+ }
+ }
+ }
+
+ if (cnt == 0) {
+ *errorp = ENOENT;
+ return (NULL);
+ }
+
+ /* send this to the userland, one at a time. */
+ m = NULL;
+ LIST_FOREACH(sah, &sahtree, chain) {
+ if (req_satype != SADB_SATYPE_UNSPEC &&
+ proto != sah->saidx.proto)
+ continue;
+
+ /* map proto to satype */
+ if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
+ m_freem(m);
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ for (stateidx = 0;
+ stateidx < _ARRAYLEN(saorder_state_any);
+ stateidx++) {
+ state = saorder_state_any[stateidx];
+ LIST_FOREACH(sav, &sah->savtree[state], chain) {
+ n = key_setdumpsa(sav, SADB_DUMP, satype,
+ --cnt, 0);
+ if (!n) {
+ m_freem(m);
+ *errorp = ENOBUFS;
+ return (NULL);
+ }
+
+ if (!m)
+ m = n;
+ else
+ m_cat(m, n);
+ }
+ }
+ }
+
+ if (!m) {
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ if ((m->m_flags & M_PKTHDR) != 0) {
+ m->m_pkthdr.len = 0;
+ for (n = m; n; n = n->m_next)
+ m->m_pkthdr.len += n->m_len;
+ }
+
+ *errorp = 0;
+ return (m);
+}
+
/*
* SADB_X_PROMISC processing
*
@@ -7175,6 +7315,75 @@ key_sa_stir_iv(sav)
IPSEC_ASSERT(sav->iv != NULL, ("null IV"));
key_randomfill(sav->iv, sav->ivlen);
}
+
+static int
+sysctl_net_key_dumpsa(SYSCTL_HANDLER_ARGS)
+{
+ int *name = (int *)arg1;
+ struct mbuf *m, *n;
+ int s, error;
+
+ if (req->newptr)
+ return (EPERM);
+ if (arg2 != 1)
+ return (EINVAL);
+
+ s = splnet();
+ m = key_setdump(name[0], &error);
+ splx(s);
+ if (!m)
+ return (error);
+ if (!req->oldptr)
+ req->oldidx = m->m_pkthdr.len;
+ else {
+ for (n = m; n; n = n->m_next) {
+ error = SYSCTL_OUT(req, mtod(n, const void *),
+ n->m_len);
+ if (error)
+ break;
+ }
+ }
+ m_freem(m);
+
+ return (error);
+}
+
+static int
+sysctl_net_key_dumpsp(SYSCTL_HANDLER_ARGS)
+{
+ struct mbuf *m, *n;
+ int s, error;
+
+ if (req->newptr)
+ return (EPERM);
+ if (arg2 != 0)
+ return (EINVAL);
+
+ s = splnet();
+ m = key_setspddump(&error);
+ splx(s);
+ if (!m)
+ return (error);
+ if (!req->oldptr)
+ req->oldidx = m->m_pkthdr.len;
+ else {
+ for (n = m; n; n = n->m_next) {
+ error = SYSCTL_OUT(req, mtod(n, const void *),
+ n->m_len);
+ if (error)
+ break;
+ }
+ }
+ m_freem(m);
+
+ return (error);
+}
+
+SYSCTL_NODE(_net_key, KEYCTL_DUMPSA, dumpsa, CTLFLAG_RD | CTLFLAG_SKIP,
+ sysctl_net_key_dumpsa, "");
+
+SYSCTL_NODE(_net_key, KEYCTL_DUMPSP, dumpsp, CTLFLAG_RD | CTLFLAG_SKIP,
+ sysctl_net_key_dumpsp, "");
/* XXX too much? */
static struct mbuf *
Index: sys/netipsec/key_var.h
diff -u sys/netipsec/key_var.h.orig sys/netipsec/key_var.h
--- sys/netipsec/key_var.h.orig Mon Feb 28 05:34:43 2005
+++ sys/netipsec/key_var.h Wed Mar 16 22:26:16 2005
@@ -46,7 +46,9 @@
#define KEYCTL_ESP_AUTH 10
#define KEYCTL_AH_KEYMIN 11
#define KEYCTL_PREFERED_OLDSA 12
-#define KEYCTL_MAXID 13
+#define KEYCTL_DUMPSA 13
+#define KEYCTL_DUMPSP 14
+#define KEYCTL_MAXID 15
#define KEYCTL_NAMES { \
{ 0, 0 }, \
Index: sys/netkey/key.c
diff -u -p sys/netkey/key.c.orig sys/netkey/key.c
--- sys/netkey/key.c.orig Mon Feb 28 05:34:46 2005
+++ sys/netkey/key.c Thu Mar 17 00:43:42 2005
@@ -383,6 +383,7 @@ static int key_spdflush(struct socket *,
const struct sadb_msghdr *);
static int key_spddump(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
+static struct mbuf *key_setspddump(int *);
static struct mbuf *key_setdumpsp(struct secpolicy *,
u_int8_t, u_int32_t, u_int32_t);
static u_int key_getspreqmsglen(struct secpolicy *);
@@ -478,6 +479,7 @@ static int key_flush(struct socket *, st
const struct sadb_msghdr *);
static int key_dump(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
+static struct mbuf *key_setdump(u_int8_t, int *);
static int key_promisc(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_senderror(struct socket *, struct mbuf *, int);
@@ -2480,6 +2482,52 @@ key_spddump(so, m, mhp)
}
static struct mbuf *
+key_setspddump(errorp)
+ int *errorp;
+{
+ struct secpolicy *sp;
+ int cnt;
+ u_int dir;
+ struct mbuf *m, *n;
+
+ /* search SPD entry and get buffer size. */
+ cnt = 0;
+ for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ LIST_FOREACH(sp, &sptree[dir], chain) {
+ cnt++;
+ }
+ }
+
+ if (cnt == 0) {
+ *errorp = ENOENT;
+ return (NULL);
+ }
+
+ m = NULL;
+ for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ LIST_FOREACH(sp, &sptree[dir], chain) {
+ --cnt;
+ n = key_setdumpsp(sp, SADB_X_SPDDUMP, cnt, 0);
+
+ if (!n) {
+ *errorp = ENOBUFS;
+ m_freem(m);
+ return (NULL);
+ }
+ if (!m)
+ m = n;
+ else {
+ m->m_pkthdr.len += n->m_pkthdr.len;
+ m_cat(m, n);
+ }
+ }
+ }
+
+ *errorp = 0;
+ return (m);
+}
+
+static struct mbuf *
key_setdumpsp(sp, type, seq, pid)
struct secpolicy *sp;
u_int8_t type;
@@ -6796,6 +6844,98 @@ key_dump(so, m, mhp)
return 0;
}
+static struct mbuf *
+key_setdump(req_satype, errorp)
+ u_int8_t req_satype;
+ int *errorp;
+{
+ struct secashead *sah;
+ struct secasvar *sav;
+ u_int16_t proto;
+ u_int stateidx;
+ u_int8_t satype;
+ u_int8_t state;
+ int cnt;
+ struct mbuf *m, *n;
+
+ /* map satype to proto */
+ if ((proto = key_satype2proto(req_satype)) == 0) {
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ /* count sav entries to be sent to the userland. */
+ cnt = 0;
+ LIST_FOREACH(sah, &sahtree, chain) {
+ if (req_satype != SADB_SATYPE_UNSPEC &&
+ proto != sah->saidx.proto)
+ continue;
+
+ for (stateidx = 0;
+ stateidx < _ARRAYLEN(saorder_state_any);
+ stateidx++) {
+ state = saorder_state_any[stateidx];
+ LIST_FOREACH(sav, &sah->savtree[state], chain) {
+ cnt++;
+ }
+ }
+ }
+
+ if (cnt == 0) {
+ *errorp = ENOENT;
+ return (NULL);
+ }
+
+ /* send this to the userland, one at a time. */
+ m = NULL;
+ LIST_FOREACH(sah, &sahtree, chain) {
+ if (req_satype != SADB_SATYPE_UNSPEC &&
+ proto != sah->saidx.proto)
+ continue;
+
+ /* map proto to satype */
+ if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
+ m_freem(m);
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ for (stateidx = 0;
+ stateidx < _ARRAYLEN(saorder_state_any);
+ stateidx++) {
+ state = saorder_state_any[stateidx];
+ LIST_FOREACH(sav, &sah->savtree[state], chain) {
+ n = key_setdumpsa(sav, SADB_DUMP, satype,
+ --cnt, 0);
+ if (!n) {
+ m_freem(m);
+ *errorp = ENOBUFS;
+ return (NULL);
+ }
+
+ if (!m)
+ m = n;
+ else
+ m_cat(m, n);
+ }
+ }
+ }
+
+ if (!m) {
+ *errorp = EINVAL;
+ return (NULL);
+ }
+
+ if ((m->m_flags & M_PKTHDR) != 0) {
+ m->m_pkthdr.len = 0;
+ for (n = m; n; n = n->m_next)
+ m->m_pkthdr.len += n->m_len;
+ }
+
+ *errorp = 0;
+ return (m);
+}
+
/*
* SADB_X_PROMISC processing
*
@@ -7593,6 +7733,75 @@ key_sp_unlink(sp)
key_freesp(sp);
}
}
+
+static int
+sysctl_net_key_dumpsa(SYSCTL_HANDLER_ARGS)
+{
+ int *name = (int *)arg1;
+ struct mbuf *m, *n;
+ int s, error;
+
+ if (req->newptr)
+ return (EPERM);
+ if (arg2 != 1)
+ return (EINVAL);
+
+ s = splnet();
+ m = key_setdump(name[0], &error);
+ splx(s);
+ if (!m)
+ return (error);
+ if (!req->oldptr)
+ req->oldidx = m->m_pkthdr.len;
+ else {
+ for (n = m; n; n = n->m_next) {
+ error = SYSCTL_OUT(req, mtod(n, const void *),
+ n->m_len);
+ if (error)
+ break;
+ }
+ }
+ m_freem(m);
+
+ return (error);
+}
+
+static int
+sysctl_net_key_dumpsp(SYSCTL_HANDLER_ARGS)
+{
+ struct mbuf *m, *n;
+ int s, error;
+
+ if (req->newptr)
+ return (EPERM);
+ if (arg2 != 0)
+ return (EINVAL);
+
+ s = splnet();
+ m = key_setspddump(&error);
+ splx(s);
+ if (!m)
+ return (error);
+ if (!req->oldptr)
+ req->oldidx = m->m_pkthdr.len;
+ else {
+ for (n = m; n; n = n->m_next) {
+ error = SYSCTL_OUT(req, mtod(n, const void *),
+ n->m_len);
+ if (error)
+ break;
+ }
+ }
+ m_freem(m);
+
+ return (error);
+}
+
+SYSCTL_NODE(_net_key, KEYCTL_DUMPSA, dumpsa, CTLFLAG_RD | CTLFLAG_SKIP,
+ sysctl_net_key_dumpsa, "");
+
+SYSCTL_NODE(_net_key, KEYCTL_DUMPSP, dumpsp, CTLFLAG_RD | CTLFLAG_SKIP,
+ sysctl_net_key_dumpsp, "");
/* XXX too much? */
static struct mbuf *
Index: sys/netkey/key_var.h
diff -u sys/netkey/key_var.h.orig sys/netkey/key_var.h
--- sys/netkey/key_var.h.orig Wed Mar 16 19:05:17 2005
+++ sys/netkey/key_var.h Wed Mar 16 19:05:26 2005
@@ -46,7 +46,9 @@
#define KEYCTL_ESP_AUTH 10
#define KEYCTL_AH_KEYMIN 11
#define KEYCTL_PREFERED_OLDSA 12
-#define KEYCTL_MAXID 13
+#define KEYCTL_DUMPSA 13
+#define KEYCTL_DUMPSP 14
+#define KEYCTL_MAXID 15
#ifdef _KERNEL
#define _ARRAYLEN(p) (sizeof(p)/sizeof(p[0]))
Index: usr.sbin/setkey/setkey.c
diff -u -p usr.sbin/setkey/setkey.c.orig usr.sbin/setkey/setkey.c
--- usr.sbin/setkey/setkey.c.orig Wed Nov 5 18:47:54 2003
+++ usr.sbin/setkey/setkey.c Thu Mar 17 00:39:46 2005
@@ -34,11 +34,13 @@
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/time.h>
+#include <sys/sysctl.h>
#include <err.h>
#include <net/route.h>
#include <netinet/in.h>
#include <net/pfkeyv2.h>
#include <netkey/keydb.h>
+#include <netkey/key_var.h>
#include <netkey/key_debug.h>
#include <netinet6/ipsec.h>
@@ -60,6 +62,7 @@ void sendkeyshort __P((u_int));
void promisc __P((void));
int sendkeymsg __P((char *, size_t));
int postproc __P((struct sadb_msg *, int));
+int sysctldump __P((u_int, u_int8_t));
const char *numstr __P((int));
void shortdump_hdr __P((void));
void shortdump __P((struct sadb_msg *));
@@ -106,6 +109,7 @@ main(ac, av)
{
FILE *fp = stdin;
int c;
+ int error;
if (ac == 1) {
usage();
@@ -166,6 +170,18 @@ main(ac, av)
switch (f_mode) {
case MODE_CMDDUMP:
+ error = sysctldump(f_policy ? SADB_X_SPDDUMP : SADB_DUMP,
+ SADB_SATYPE_UNSPEC);
+ if (error == 0)
+ break;
+ if (error < 0) {
+ if (errno == ENOENT) {
+ printf("No S%cD entries.\n",
+ f_policy ? 'P' : 'A');
+ break;
+ } else if (errno != 0)
+ err(1, "sysctl");
+ }
sendkeyshort(f_policy ? SADB_X_SPDDUMP: SADB_DUMP);
break;
case MODE_CMDFLUSH:
@@ -443,6 +459,49 @@ postproc(msg, len)
}
return(0);
+}
+
+int
+sysctldump(type, satype)
+ u_int type;
+ u_int8_t satype;
+{
+ int mib[] = { CTL_NET, PF_KEY, KEYCTL_DUMPSA, 0 };
+ size_t len, l;
+ char *buf, *p, *ep;
+ struct sadb_msg *msg;
+
+ if (type == SADB_DUMP) {
+ mib[2] = KEYCTL_DUMPSA;
+ mib[3] = satype;
+ l = 4;
+ } else if (type == SADB_X_SPDDUMP) {
+ mib[2] = KEYCTL_DUMPSP;
+ l = 3;
+ } else
+ return (EINVAL);
+
+ if (sysctl(mib, l, NULL, &len, NULL, 0) < 0)
+ return (-1);
+ buf = malloc(len);
+ if (!buf)
+ return (ENOBUFS);
+ if (sysctl(mib, l, buf, &len, NULL, 0) < 0) {
+ free(buf);
+ return (-1);
+ }
+
+ p = buf;
+ ep = buf + len;
+ while (p < ep) {
+ msg = (struct sadb_msg *)p;
+ l = PFKEY_UNUNIT64(msg->sadb_msg_len);
+ postproc(msg, l);
+ p += l;
+ }
+
+ free(buf);
+ return (0);
}
/*------------------------------------------------------------*/
--Multipart_Thu_Mar_17_01:10:38_2005-1
Content-Type: text/plain; charset=US-ASCII
--
Hajimu UMEMOTO @ Internet Mutual Aid Society Yokohama, Japan
ume@mahoroba.org ume@{,jp.}FreeBSD.org
http://www.imasy.org/~ume/
--Multipart_Thu_Mar_17_01:10:38_2005-1--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?ygek6o7inb5.wl%ume>