Date: Mon, 6 Aug 2007 12:32:08 +0400 (MSD) From: Alexei <alexei@files.42-club.ru> To: FreeBSD-gnats-submit@FreeBSD.org Subject: kern/115239: panic with 'kmem_map too small' using ipnat Message-ID: <200708060832.l768W8cw041223@sys.42-club.ru> Resent-Message-ID: <200708060850.l768o2kl022264@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 115239
>Category: kern
>Synopsis: panic with 'kmem_map too small' using ipnat
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Mon Aug 06 08:50:01 GMT 2007
>Closed-Date:
>Last-Modified:
>Originator: Alexei G <alexei@files.42-club.ru>
>Release: FreeBSD 6.2-RELEASE-p5 i386
>Organization:
Home, sweet home
>Environment:
System: FreeBSD sys.42-club.ru 6.2-RELEASE-p5 FreeBSD 6.2-RELEASE-p5 #2:
Mon Aug 6 10:07:58 MSD 2007 root@sys.42-club.ru:/usr/obj/usr/src/sys/sys i386
>Description:
I got my home router falling in panic with 'kmem_map too small' every 3-6 days.
So I started monitoring `vmstat -m` with mrtg and found 'inuse' of 'temp' value
rising up for about 200k per day. I applied patch to kern_malloc.c (attached),
recompiled kernel with some debug options (attached) and rebooted system.
After that, I did
`sysctl debug.ktr.alq_enable=1 && sleep 60 && sysctl debug.ktr.alq_enable=0`.
Then I found leaks by lf.pl (attached) script as `ktrdump -i /tmp/ktr.out | lf.pl`
and got the following:
0xc047b990 65
0xc04728ba 72
0xc0475915 65
0xc04704c7 41
After that I did
`addr2line -e /usr/obj/usr/src/sys/sys/kernel.debug 0xc047b990 0xc04728ba 0xc0475915 0xc04704c7`
and got:
/usr/src/sys/contrib/ipfilter/netinet/ip_proxy.c:416
/usr/src/sys/contrib/ipfilter/netinet/ip_nat.c:2212
/usr/src/sys/contrib/ipfilter/netinet/ip_ftp_pxy.c:132
/usr/src/sys/contrib/ipfilter/netinet/ip_nat.c:454
>How-To-Repeat:
System panics more often if p2p software is being used behind the nat.
--- ipnat.rules begins here ---
#map fxp1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32 proxy port ftp ftp/tcp
#map fxp1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32
#map ng1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32 proxy port ftp ftp/tcp
#map ng1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32
map fxp1 from 192.168.43.0/24 ! to 192.168.43.0/24 -> 0/32 proxy port ftp ftp/tcp
map fxp1 from 192.168.43.0/24 ! to 192.168.43.0/24 -> 0/32
map fxp1 from 192.168.44.0/24 ! to 192.168.44.0/24 -> 0/32 proxy port ftp ftp/tcp
map fxp1 from 192.168.44.0/24 ! to 192.168.44.0/24 -> 0/32
map fxp1 from 192.168.45.0/24 ! to 192.168.45.0/24 -> 0/32 proxy port ftp ftp/tcp
map fxp1 from 192.168.45.0/24 ! to 192.168.45.0/24 -> 0/32
rdr fxp1 81.88.212.138/32 port 4444 -> 192.168.43.2 port 4899
##map tun0 from 192.168.0.0/24 ! to 192.168.42.0/24 -> 0/32
#map tun0 from 192.168.12.0/24 ! to 192.168.42.0/24 -> 0/32
#rdr fxp1 81.88.212.138/32 port 9876 -> 192.168.42.2 port 4899
rdr fxp1 81.88.212.138/32 port 4662 -> 192.168.42.2 port 4662
rdr fxp1 81.88.212.138/32 port 4872 -> 192.168.42.2 port 4872
rdr fxp1 81.88.212.138/32 port 4711 -> 192.168.42.2 port 4711
rdr fxp1 81.88.212.138/32 port 6881 -> 192.168.42.2 port 6881
#rdr fxp0 0.0.0.0/0 port 80 -> 81.88.212.138 port 2345
#map tun0 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32 proxy port ftp ftp/tcp
#map tun0 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32
map fxp1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32 proxy port ftp ftp/tcp
map fxp1 from 192.168.42.0/24 ! to 192.168.42.0/24 -> 0/32
rdr fxp1 81.88.212.138/32 port 4221 -> 192.168.43.2 port 21
rdr fxp1 81.88.212.138/32 port 6000 -> 192.168.43.2 port 6000
rdr fxp1 81.88.212.138/32 port 6001 -> 192.168.43.2 port 6001
rdr fxp1 81.88.212.138/32 port 6002 -> 192.168.43.2 port 6002
rdr fxp1 81.88.212.138/32 port 6003 -> 192.168.43.2 port 6003
rdr fxp1 81.88.212.138/32 port 6004 -> 192.168.43.2 port 6004
rdr fxp1 81.88.212.138/32 port 6005 -> 192.168.43.2 port 6005
rdr fxp1 81.88.212.138/32 port 6006 -> 192.168.43.2 port 6006
rdr fxp1 81.88.212.138/32 port 6007 -> 192.168.43.2 port 6007
rdr fxp1 81.88.212.138/32 port 6008 -> 192.168.43.2 port 6008
rdr fxp1 81.88.212.138/32 port 6009 -> 192.168.43.2 port 6009
rdr fxp1 81.88.212.138/32 port 6010 -> 192.168.43.2 port 6010
rdr fxp1 81.88.212.138/32 port 25 -> 192.168.44.2 port 25
rdr fxp1 81.88.212.138/32 port 110 -> 192.168.44.2 port 110
--- ipnat.rules ends here ---
--- lf.pl begins here ---
#!/usr/bin/perl -w
%vas = ();
%callers = ();
while (<>) {
chomp;
my @a = split;
if ($a[1] eq 'malloc') {
$vas{$a[3]} = $a[5];
} elsif ($a[1] eq 'free') {
delete $vas{$a[3]};
}
}
foreach my $va (keys %vas) {
if (defined $callers{$vas{$va}}) {
$callers{$vas{$va}}++;
} else {
$callers{$vas{$va}} = 1;
}
}
foreach my $caller (keys %callers) {
print "$caller $callers{$caller}\n";
}
--- lf.pl ends here ---
--- kern_malloc.c.patch begins here ---
--- kern_malloc.c.orig Mon Mar 6 11:42:07 2006
+++ kern_malloc.c Sun Aug 5 10:41:49 2007
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD: src/sys/kern/kern_ma
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/time.h>
+#include <sys/ktr.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -338,6 +339,14 @@ malloc(unsigned long size, struct malloc
memset(va, 0x70, osize);
}
#endif
+
+ register_t ebp;
+
+ __asm __volatile("movl %%ebp,%0" : "=r" (ebp));
+ if (mtp == M_TEMP)
+ CTR3(KTR_MALLOC, "malloc va %p caller %p size %d",
+ va, ((register_t *)(ebp))[1], size);
+
return ((void *) va);
}
@@ -365,6 +374,13 @@ free(void *addr, struct malloc_type *mtp
return;
}
#endif
+
+ register_t ebp;
+
+ __asm __volatile("movl %%ebp,%0" : "=r" (ebp));
+ if (mtp == M_TEMP)
+ CTR2(KTR_MALLOC, "free va %p caller %p",
+ addr, ((register_t *)(ebp))[1]);
size = 0;
--- kern_malloc.c.patch ends here ---
--- sys begins here ---
#
# GENERIC -- Generic kernel configuration file for FreeBSD/i386
#
# For more information on this file, please read the handbook section on
# Kernel Configuration Files:
#
# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
#
# The handbook is also available locally in /usr/share/doc/handbook
# if you've installed the doc distribution, otherwise always see the
# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
# latest information.
#
# An exhaustive list of options and more detailed explanations of the
# device lines is also present in the ../../conf/NOTES and NOTES files.
# If you are in doubt as to the purpose or necessity of a line, check first
# in NOTES.
#
# $FreeBSD: src/sys/i386/conf/GENERIC,v 1.429.2.3.2.1 2005/10/28 19:22:41 jhb Exp $
machine i386
cpu I486_CPU
cpu I586_CPU
cpu I686_CPU
ident sys
# To statically compile in device wiring instead of /boot/device.hints
#hints "GENERIC.hints" # Default places to look for devices.
makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
#options SCHED_ULE # ULE scheduler
options SCHED_4BSD # 4BSD scheduler
options PREEMPTION # Enable kernel thread preemption
options INET # InterNETworking
options INET6 # IPv6 communications protocols
options FFS # Berkeley Fast Filesystem
options SOFTUPDATES # Enable FFS soft updates support
options UFS_ACL # Support for access control lists
options UFS_DIRHASH # Improve performance on big directories
options MD_ROOT # MD is a potential root device
options NFSCLIENT # Network Filesystem Client
options NFSSERVER # Network Filesystem Server
options NFS_ROOT # NFS usable as /, requires NFSCLIENT
options MSDOSFS # MSDOS Filesystem
options CD9660 # ISO 9660 Filesystem
options PROCFS # Process filesystem (requires PSEUDOFS)
options PSEUDOFS # Pseudo-filesystem framework
options GEOM_GPT # GUID Partition Tables.
options COMPAT_43 # Compatible with BSD 4.3 [KEEP THIS!]
options COMPAT_FREEBSD4 # Compatible with FreeBSD4
options COMPAT_FREEBSD5 # Compatible with FreeBSD5
options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
options KTRACE # ktrace(1) support
options SYSVSHM # SYSV-style shared memory
options SYSVMSG # SYSV-style message queues
options SYSVSEM # SYSV-style semaphores
options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~128k to driver.
options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~215k to driver.
options ADAPTIVE_GIANT # Giant mutex is adaptive.
device apic # I/O APIC
# Bus support.
device eisa
device pci
# Floppy drives
device fdc
# ATA and ATAPI devices
device ata
device atadisk # ATA disk drives
device ataraid # ATA RAID drives
device atapicd # ATAPI CDROM drives
device atapifd # ATAPI floppy drives
device atapist # ATAPI tape drives
options ATA_STATIC_ID # Static device numbering
# SCSI Controllers
device ahb # EISA AHA1742 family
device ahc # AHA2940 and onboard AIC7xxx devices
device ahd # AHA39320/29320 and onboard AIC79xx devices
device amd # AMD 53C974 (Tekram DC-390(T))
device isp # Qlogic family
#device ispfw # Firmware for QLogic HBAs- normally a module
device mpt # LSI-Logic MPT-Fusion
#device ncr # NCR/Symbios Logic
device sym # NCR/Symbios Logic (newer chipsets + those of `ncr')
device trm # Tekram DC395U/UW/F DC315U adapters
device adv # Advansys SCSI adapters
device adw # Advansys wide SCSI adapters
device aha # Adaptec 154x SCSI adapters
device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60.
device bt # Buslogic/Mylex MultiMaster SCSI adapters
device ncv # NCR 53C500
device nsp # Workbit Ninja SCSI-3
device stg # TMC 18C30/18C50
# SCSI peripherals
device scbus # SCSI bus (required for SCSI)
device ch # SCSI media changers
device da # Direct Access (disks)
device sa # Sequential Access (tape etc)
device cd # CD
device pass # Passthrough device (direct SCSI access)
device ses # SCSI Environmental Services (and SAF-TE)
# RAID controllers interfaced to the SCSI subsystem
device amr # AMI MegaRAID
device arcmsr # Areca SATA II RAID
device asr # DPT SmartRAID V, VI and Adaptec SCSI RAID
device ciss # Compaq Smart RAID 5*
device dpt # DPT Smartcache III, IV - See NOTES for options
device hptmv # Highpoint RocketRAID 182x
device iir # Intel Integrated RAID
device ips # IBM (Adaptec) ServeRAID
device mly # Mylex AcceleRAID/eXtremeRAID
device twa # 3ware 9000 series PATA/SATA RAID
# RAID controllers
device aac # Adaptec FSA RAID
device aacp # SCSI passthrough for aac (requires CAM)
device ida # Compaq Smart RAID
device mlx # Mylex DAC960 family
device pst # Promise Supertrak SX6000
device twe # 3ware ATA RAID
# atkbdc0 controls both the keyboard and the PS/2 mouse
device atkbdc # AT keyboard controller
device atkbd # AT keyboard
device psm # PS/2 mouse
device vga # VGA video card driver
device splash # Splash screen and screen saver support
# syscons is the default console driver, resembling an SCO console
device sc
# Enable this for the pcvt (VT220 compatible) console driver
#device vt
#options XSERVER # support for X server on a vt console
#options FAT_CURSOR # start with block cursor
device agp # support several AGP chipsets
# Power management support (see NOTES for more options)
#device apm
# Add suspend/resume support for the i8254.
device pmtimer
# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
device cbb # cardbus (yenta) bridge
device pccard # PC Card (16-bit) bus
device cardbus # CardBus (32-bit) bus
# Serial (COM) ports
device sio # 8250, 16[45]50 based serial ports
# Parallel port
device ppc
device ppbus # Parallel port bus (required)
device lpt # Printer
device plip # TCP/IP over parallel
device ppi # Parallel port interface device
#device vpo # Requires scbus and da
# If you've got a "dumb" serial or parallel PCI card that is
# supported by the puc(4) glue driver, uncomment the following
# line to enable it (connects to the sio and/or ppc drivers):
#device puc
# PCI Ethernet NICs.
device de # DEC/Intel DC21x4x (``Tulip'')
device em # Intel PRO/1000 adapter Gigabit Ethernet Card
device ixgb # Intel PRO/10GbE Ethernet Card
device txp # 3Com 3cR990 (``Typhoon'')
device vx # 3Com 3c590, 3c595 (``Vortex'')
# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device miibus # MII bus support
device bfe # Broadcom BCM440x 10/100 Ethernet
device bge # Broadcom BCM570xx Gigabit Ethernet
device dc # DEC/Intel 21143 and various workalikes
device fxp # Intel EtherExpress PRO/100B (82557, 82558)
device lge # Level 1 LXT1001 gigabit Ethernet
device nge # NatSemi DP83820 gigabit Ethernet
device nve # nVidia nForce MCP on-board Ethernet Networking
device pcn # AMD Am79C97x PCI 10/100(precedence over 'lnc')
device re # RealTek 8139C+/8169/8169S/8110S
device rl # RealTek 8129/8139
device sf # Adaptec AIC-6915 (``Starfire'')
device sis # Silicon Integrated Systems SiS 900/SiS 7016
device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet
device ste # Sundance ST201 (D-Link DFE-550TX)
device ti # Alteon Networks Tigon I/II gigabit Ethernet
device tl # Texas Instruments ThunderLAN
device tx # SMC EtherPower II (83c170 ``EPIC'')
device vge # VIA VT612x gigabit Ethernet
device vr # VIA Rhine, Rhine II
device wb # Winbond W89C840F
device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'')
# ISA Ethernet NICs. pccard NICs included.
#device cs # Crystal Semiconductor CS89x0 NIC
# 'device ed' requires 'device miibus'
#device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards
#device ex # Intel EtherExpress Pro/10 and Pro/10+
#device ep # Etherlink III based cards
#device fe # Fujitsu MB8696x based cards
#device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc.
#device lnc # NE2100, NE32-VL Lance Ethernet cards
#device sn # SMC's 9000 series of Ethernet chips
#device xe # Xircom pccard Ethernet
# ISA devices that use the old ISA shims
#device le
# Wireless NIC cards
#device wlan # 802.11 support
#device an # Aironet 4500/4800 802.11 wireless NICs.
#device awi # BayStack 660 and others
#device ral # Ralink Technology RT2500 wireless NICs.
#device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs.
#device wl # Older non 802.11 Wavelan wireless NIC.
# Pseudo devices.
device loop # Network loopback
device random # Entropy device
device ether # Ethernet support
device sl # Kernel SLIP
device ppp # Kernel PPP
device tun # Packet tunnel.
device pty # Pseudo-ttys (telnet etc)
device md # Memory "disks"
device gif # IPv6 and IPv4 tunneling
device faith # IPv6-to-IPv4 relaying (translation)
# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device bpf # Berkeley packet filter
# USB support
#device uhci # UHCI PCI->USB interface
#device ohci # OHCI PCI->USB interface
#device ehci # EHCI PCI->USB interface (USB 2.0)
#device usb # USB Bus (required)
#device udbp # USB Double Bulk Pipe devices
#device ugen # Generic
#device uhid # "Human Interface Devices"
#device ukbd # Keyboard
#device ulpt # Printer
#device umass # Disks/Mass storage - Requires scbus and da
#device ums # Mouse
#device ural # Ralink Technology RT2500USB wireless NICs
#device urio # Diamond Rio 500 MP3 player
#device uscanner # Scanners
# USB Ethernet, requires miibus
#device aue # ADMtek USB Ethernet
#device axe # ASIX Electronics USB Ethernet
#device cdce # Generic USB over Ethernet
#device cue # CATC USB Ethernet
#device kue # Kawasaki LSI USB Ethernet
#device rue # RealTek RTL8150 USB Ethernet
# FireWire support
#device firewire # FireWire bus code
#device sbp # SCSI over FireWire (Requires scbus and da)
#device fwe # Ethernet over FireWire (non-standard!)
options IPDIVERT
options DUMMYNET
options IPFIREWALL
options IPFIREWALL_VERBOSE
options IPFIREWALL_VERBOSE_LIMIT=100
#options QUOTA
#options SUIDDIR
options IPFILTER
options IPFILTER_LOG
options TCP_DROP_SYNFIN
#options MAXUSERS=512
#options MAXFILES=65000
options DEVICE_POLLING
options HZ=1000
options IPSEC
options IPSEC_ESP
options NETGRAPH
options MROUTING
options PIM
options KDB
options DDB
options KTR
options ALQ
options KTR_ALQ
options KTR_COMPILE=(KTR_MALLOC)
options KTR_CPUMASK=0x15
options KTR_ENTRIES=8192
options KTR_MASK=(KTR_MALLOC)
--- sys ends here ---
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200708060832.l768W8cw041223>