Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 16 Jun 2008 11:10:34 +0000 (UTC)
From:      Stef Walter <stef-list@memberwebs.com>
Cc:        freebsd-hackers@freebsd.org, freebsd-stable@freebsd.org
Subject:   Re: FreeBSD 6.3 deadlock (interrupted pmap_remove_pages) with DDB output
Message-ID:  <20080616111032.E194BF181D4@mx.npubs.com>
References:  <20080615112318.146C1F18512@mx.npubs.com>

next in thread | previous in thread | raw e-mail | index | archive | help
This is a multi-part message in MIME format.
--------------010905080203030203030807
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: 7bit

Stef Walter wrote:
> I've been trying to track down a deadlock on some newish production
> servers running FreeBSD 6.3-RELEASE-p2. The deadlock occurs on a
> specific (although mundane) hardware configuration, and each of several
> servers running this hardware deadlock about once per week.
> 
> Although I suspect that this is not hardware related, from a (naive)
> perusal of the attached stack traces.

Here's another similar deadlock that occurred on an similar machine.
Last time pmap_enter was preempted by an interrupt, this time
pmap_remove_pages was preempted by a timer.

I tried changing machdep.cpu_idle_hlt to zero, based on a hint [1], but
that didn't seem to make a difference.

Again in this case the processes are waiting on the page queue mutex:
 python (in trap > trap_pfault > vm_fault)
 rsync (in ... cluster_read > getblk > getnewbuf > vfs_vmio_release)

And another rsync process is holding the vm page queue mutex:
 rsync (in exit1 > vmspace_exit > pmap_remove_pages)

The rsync process in pmap_remove_pages was interrupted by a timer while
holding the vm page queue lock and then was not scheduled again.

Kernel config and full details attched.

I may try building the kernel without PREEMPTION to see if it fixes this
problem.

Again, any and all advice or hints are welcome. I really appreciate any
time spent on this.

Stef Walter

[1]
http://unix.derkeiler.com/Mailing-Lists/FreeBSD/current/2005-02/0508.html

--------------010905080203030203030807
Content-Type: text/plain;
 name="RACK1DBG"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="RACK1DBG"

#
# RACK1
#

machine		i386
cpu		I686_CPU
ident		RACK1

# To statically compile in device wiring instead of /boot/device.hints
#hints		"GENERIC.hints"		# Default places to look for devices.

makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols

options 	SCHED_4BSD		# 4BSD scheduler
options 	PREEMPTION		# Enable kernel thread preemption
options 	INET			# InterNETworking
options 	INET6			# IPv6 communications protocols
options 	FFS			# Berkeley Fast Filesystem
options 	SOFTUPDATES		# Enable FFS soft updates support
options 	UFS_ACL			# Support for access control lists
options 	UFS_DIRHASH		# Improve performance on big directories
options 	MD_ROOT			# MD is a potential root device
options 	NFSCLIENT		# Network Filesystem Client
options 	NFSSERVER		# Network Filesystem Server
options 	NFS_ROOT		# NFS usable as /, requires NFSCLIENT
options 	MSDOSFS			# MSDOS Filesystem
options 	CD9660			# ISO 9660 Filesystem
options 	PROCFS			# Process filesystem (requires PSEUDOFS)
options 	PSEUDOFS		# Pseudo-filesystem framework
options 	GEOM_GPT		# GUID Partition Tables.
options 	COMPAT_43		# Compatible with BSD 4.3 [KEEP THIS!]
options 	COMPAT_FREEBSD4		# Compatible with FreeBSD4
options 	COMPAT_FREEBSD5		# Compatible with FreeBSD5
options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
options 	KTRACE			# ktrace(1) support
options 	SYSVSHM			# SYSV-style shared memory
options 	SYSVMSG			# SYSV-style message queues
options 	SYSVSEM			# SYSV-style semaphores
options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.

device		apic			# I/O APIC

options         SMP

options         QUOTA
options         FAST_IPSEC
options         IPFIREWALL
options         IPFIREWALL_DEFAULT_TO_ACCEPT
options         IPFIREWALL_VERBOSE
options         IPFIREWALL_FORWARD
options         IPDIVERT
options         DUMMYNET

options         ALTQ
options         ALTQ_CBQ
options         ALTQ_RED
options         ALTQ_RIO
options         ALTQ_HFSC
options         ALTQ_CDNR
options         ALTQ_PRIQ


options KDB
options KDB_TRACE
options KDB_UNATTENDED
options DDB
options INVARIANTS
options INVARIANT_SUPPORT
options WITNESS
options WITNESS_SKIPSPIN
options DIAGNOSTIC
options BREAK_TO_DEBUGGER
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS


device 		crypto

# Bus support.
device		eisa
device		pci

# Floppy drives
device		fdc

# ATA and ATAPI devices
device		ata
device		atadisk		# ATA disk drives
device		ataraid		# ATA RAID drives
device		atapicd		# ATAPI CDROM drives
#device		atapifd		# ATAPI floppy drives
#device		atapist		# ATAPI tape drives
options 	ATA_STATIC_ID	# Static device numbering

# SCSI Controllers
#device		ahb		# EISA AHA1742 family
#device		ahc		# AHA2940 and onboard AIC7xxx devices
#options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~128k to driver.
#device		ahd		# AHA39320/29320 and onboard AIC79xx devices
#options 	AHD_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~215k to driver.
#device		amd		# AMD 53C974 (Tekram DC-390(T))
#device		isp		# Qlogic family
#device 	ispfw		# Firmware for QLogic HBAs- normally a module
#device		mpt		# LSI-Logic MPT-Fusion
#device		ncr		# NCR/Symbios Logic
#device		sym		# NCR/Symbios Logic (newer chipsets + those of `ncr')
#device		trm		# Tekram DC395U/UW/F DC315U adapters

#device		adv		# Advansys SCSI adapters
#device		adw		# Advansys wide SCSI adapters
#device		aha		# Adaptec 154x SCSI adapters
#device		aic		# Adaptec 15[012]x SCSI adapters, AIC-6[23]60.
#device		bt		# Buslogic/Mylex MultiMaster SCSI adapters

#device		ncv		# NCR 53C500
#device		nsp		# Workbit Ninja SCSI-3
#device		stg		# TMC 18C30/18C50

# SCSI peripherals
device		scbus		# SCSI bus (required for SCSI)
#device		ch		# SCSI media changers
device		da		# Direct Access (disks)
#device		sa		# Sequential Access (tape etc)
#device		cd		# CD
#device		pass		# Passthrough device (direct SCSI access)
#device		ses		# SCSI Environmental Services (and SAF-TE)

# RAID controllers interfaced to the SCSI subsystem
#device		amr		# AMI MegaRAID
#device		arcmsr		# Areca SATA II RAID
#device		asr		# DPT SmartRAID V, VI and Adaptec SCSI RAID
#device		ciss		# Compaq Smart RAID 5*
#device		dpt		# DPT Smartcache III, IV - See NOTES for options
#device		hptmv		# Highpoint RocketRAID 182x
#device		hptrr		# Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx
#device		rr232x		# Highpoint RocketRAID 232x
device		iir		# Intel Integrated RAID
#device		ips		# IBM (Adaptec) ServeRAID
#device		mly		# Mylex AcceleRAID/eXtremeRAID
#device		twa		# 3ware 9000 series PATA/SATA RAID

# RAID controllers
#device		aac		# Adaptec FSA RAID
#device		aacp		# SCSI passthrough for aac (requires CAM)
#device		ida		# Compaq Smart RAID
#device		mfi		# LSI MegaRAID SAS
#device		mlx		# Mylex DAC960 family
#device		pst		# Promise Supertrak SX6000
#device		twe		# 3ware ATA RAID

# atkbdc0 controls both the keyboard and the PS/2 mouse
device		atkbdc		# AT keyboard controller
device		atkbd		# AT keyboard
device		psm		# PS/2 mouse

device		kbdmux		# keyboard multiplexer

device		vga		# VGA video card driver

device		splash		# Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device		sc

# Enable this for the pcvt (VT220 compatible) console driver
#device		vt
#options 	XSERVER		# support for X server on a vt console
#options 	FAT_CURSOR	# start with block cursor

device		agp		# support several AGP chipsets

# Power management support (see NOTES for more options)
#device		apm
# Add suspend/resume support for the i8254.
device		pmtimer

# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
#device		cbb		# cardbus (yenta) bridge
#device		pccard		# PC Card (16-bit) bus
#device		cardbus		# CardBus (32-bit) bus

# Serial (COM) ports
device		sio		# 8250, 16[45]50 based serial ports

# Parallel port
#device		ppc
#device		ppbus		# Parallel port bus (required)
#device		lpt		# Printer
#device		plip		# TCP/IP over parallel
#device		ppi		# Parallel port interface device
#device		vpo		# Requires scbus and da

# If you've got a "dumb" serial or parallel PCI card that is
# supported by the puc(4) glue driver, uncomment the following
# line to enable it (connects to the sio and/or ppc drivers):
#device		puc

# PCI Ethernet NICs.
device		de		# DEC/Intel DC21x4x (``Tulip'')
device		em		# Intel PRO/1000 adapter Gigabit Ethernet Card
#device		ixgb		# Intel PRO/10GbE Ethernet Card
#device		txp		# 3Com 3cR990 (``Typhoon'')
#device		vx		# 3Com 3c590, 3c595 (``Vortex'')

# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device		miibus		# MII bus support
#device		bce		# Broadcom BCM5706/BCM5708 Gigabit Ethernet
#device		bfe		# Broadcom BCM440x 10/100 Ethernet
#device		bge		# Broadcom BCM570xx Gigabit Ethernet
device		dc		# DEC/Intel 21143 and various workalikes
device		fxp		# Intel EtherExpress PRO/100B (82557, 82558)
#device		lge		# Level 1 LXT1001 gigabit Ethernet
#device		msk		# Marvell/SysKonnect Yukon II Gigabit Ethernet
#device		nge		# NatSemi DP83820 gigabit Ethernet
#device		nve		# nVidia nForce MCP on-board Ethernet Networking
#device		pcn		# AMD Am79C97x PCI 10/100(precedence over 'lnc')
#device		re		# RealTek 8139C+/8169/8169S/8110S
#device		rl		# RealTek 8129/8139
#device		sf		# Adaptec AIC-6915 (``Starfire'')
#device		sis		# Silicon Integrated Systems SiS 900/SiS 7016
#device		sk		# SysKonnect SK-984x & SK-982x gigabit Ethernet
#device		ste		# Sundance ST201 (D-Link DFE-550TX)
#device		stge		# Sundance/Tamarack TC9021 gigabit Ethernet
#device		ti		# Alteon Networks Tigon I/II gigabit Ethernet
#device		tl		# Texas Instruments ThunderLAN
#device		tx		# SMC EtherPower II (83c170 ``EPIC'')
#device		vge		# VIA VT612x gigabit Ethernet
#device		vr		# VIA Rhine, Rhine II
#device		wb		# Winbond W89C840F
#device		xl		# 3Com 3c90x (``Boomerang'', ``Cyclone'')

# ISA Ethernet NICs.  pccard NICs included.
#device		cs		# Crystal Semiconductor CS89x0 NIC
# 'device ed' requires 'device miibus'
#device		ed		# NE[12]000, SMC Ultra, 3c503, DS8390 cards
#device		ex		# Intel EtherExpress Pro/10 and Pro/10+
#device		ep		# Etherlink III based cards
#device		fe		# Fujitsu MB8696x based cards
#device		ie		# EtherExpress 8/16, 3C507, StarLAN 10 etc.
#device		lnc		# NE2100, NE32-VL Lance Ethernet cards
#device		sn		# SMC's 9000 series of Ethernet chips
#device		xe		# Xircom pccard Ethernet

# Wireless NIC cards
#device		wlan		# 802.11 support
#device		wlan_wep	# 802.11 WEP support
#device		wlan_ccmp	# 802.11 CCMP support
#device		wlan_tkip	# 802.11 TKIP support
#device		an		# Aironet 4500/4800 802.11 wireless NICs.
#device		ath		# Atheros pci/cardbus NIC's
#device		ath_hal		# Atheros HAL (Hardware Access Layer)
#device		ath_rate_sample	# SampleRate tx rate control for ath
#device		awi		# BayStack 660 and others
#device		ral		# Ralink Technology RT2500 wireless NICs.
#device		wi		# WaveLAN/Intersil/Symbol 802.11 wireless NICs.
#device		wl		# Older non 802.11 Wavelan wireless NIC.

# Pseudo devices.
device		loop		# Network loopback
device		random		# Entropy device
device		ether		# Ethernet support
#device		sl		# Kernel SLIP
device		ppp		# Kernel PPP
device		tun		# Packet tunnel.
device		pty		# Pseudo-ttys (telnet etc)
device		md		# Memory "disks"
device		gif		# IPv6 and IPv4 tunneling
device		faith		# IPv6-to-IPv4 relaying (translation)

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device		bpf		# Berkeley packet filter

# USB support
#device		uhci		# UHCI PCI->USB interface
#device		ohci		# OHCI PCI->USB interface
#device		ehci		# EHCI PCI->USB interface (USB 2.0)
#device		usb		# USB Bus (required)
#device		udbp		# USB Double Bulk Pipe devices
#device		ugen		# Generic
#device		uhid		# "Human Interface Devices"
#device		ukbd		# Keyboard
#device		ulpt		# Printer
#device		umass		# Disks/Mass storage - Requires scbus and da
#device		ums		# Mouse
#device		ural		# Ralink Technology RT2500USB wireless NICs
#device		urio		# Diamond Rio 500 MP3 player
#device		uscanner	# Scanners
# USB Ethernet, requires miibus
#device		aue		# ADMtek USB Ethernet
#device		axe		# ASIX Electronics USB Ethernet
#device		cdce		# Generic USB over Ethernet
#device		cue		# CATC USB Ethernet
#device		kue		# Kawasaki LSI USB Ethernet
#device		rue		# RealTek RTL8150 USB Ethernet

# FireWire support
#device		firewire	# FireWire bus code
#device		sbp		# SCSI over FireWire (Requires scbus and da)
#device		fwe		# Ethernet over FireWire (non-standard!)

--------------010905080203030203030807
Content-Type: text/plain;
 name="pmap_remove_pages-deadlock-information.txt"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
	filename="pmap_remove_pages-deadlock-information.txt"


# --------------------------------------------------------------------
# FREEBSD BUILD 

FreeBSD m1.npubs.com 6.3-RELEASE-p2 FreeBSD 6.3-RELEASE-p2 #0: Tue Jun 10 02:07:06 UTC 2008     nate@m1.npubs.com:/usr/obj/usr/src/sys/RACK1DBG  i386

# --------------------------------------------------------------------
# PROCESS LIST

db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
69677 69671  4145 65534  LLJ    *vm page  0xcf2cc140 python
69674 69654 69486     0  RE                          rsync
69671 69670  4145 65534  SJ      piperd   0xca44a660 python
69670  4145  4145 65534  SJ      wait     0xcf673430 sh
69669  4684  4684   125  SJ      select   0xc07b7ce4 smtpd
69668 69664 69664     0  S       select   0xc07b7ce4 rsync
69664  5663 69664     0  Ls     *vm page  0xcf2cc140 rsync
69654 69511 69486     0  S       wait     0xca033c90 sh
69561 45988 45982    70  SJ      sbwait   0xcabb4bc8 postgres
69557  4141  4141   125  SJ      select   0xc07b7ce4 smtp
69541  4141  4141   125  SJ      select   0xc07b7ce4 cleanup
69540  4141  4141   125  SJ      select   0xc07b7ce4 smtpd
69535  4141  4141   125  SJ      select   0xc07b7ce4 trivial-rewrite
69533  4141  4141   125  SJ      select   0xc07b7ce4 anvil
69523  4141  4141   125  SJ      select   0xc07b7ce4 proxymap
69511 69498 69486     0  S       wait     0xc92e7430 sh
69510  4141  4141   125  SJ      select   0xc07b7ce4 smtpd
69504 69495 69495     0  SJ      sbwait   0xcf2b1638 python
69498 69486 69486     0  S       wait     0xca67b648 sh
69495 69492 69495     0  SsJ     wait     0xcf6aa218 sh
69492  1374  1374     0  SJ      piperd   0xc91c8198 cron
69486 69484 69486     0  Ss      wait     0xc92e4c90 sh
69484   691   691     0  S       piperd   0xc901b990 cron
69455     1 69455     0  Ls     *Giant    0xcf52b6c0 jail-measure
69434  1940  1940   125  SJ      kqread   0xc9987a00 trivial-rewrite
69415  1940  1940     0  SJ      kqread   0xc9b21700 local
69414  1940  1940   125  SJ      kqread   0xca77a600 smtpd
69413  1940  1940   125  SJ      kqread   0xc988ee00 smtp
69412  1940  1940   125  SJ      kqread   0xcf87b900 cleanup
69400  4684  4684     0  SJ      select   0xc07b7ce4 virtual
69379 45988 45982    70  SJ      sbwait   0xd0b004d4 postgres
69374 45988 45982    70  SJ      sbwait   0xc8f7579c postgres
69372 45988 45982    70  SJ      sbwait   0xd08a2d2c postgres
69361  2260  2260    80  SJ      accept   0xc97fe5ca httpd
69352  1940  1940   125  SJ      lockf    0xc94529c0 smtpd
69335 45988 45982    70  SJ      sbwait   0xcf2d5900 postgres
69327  1940  1940   125  SJ      lockf    0xc9452740 bounce
69326  1940  1940   125  SJ      kqread   0xd0080100 bounce
69276  1940  1940   125  SJ      lockf    0xcf5158c0 smtp
69274  1940  1940   125  SJ      kqread   0xca333800 smtp
69256 45988 45982    70  SJ      sbwait   0xc8ce5638 postgres
69245 45988 45982    70  SJ      sbwait   0xc9e23bc8 postgres
69240 45988 45982    70  SJ      sbwait   0xd0ad379c postgres
69208  4684  4684   125  SJ      select   0xc07b7ce4 smtp
69184  4684  4684   125  SJ      select   0xc07b7ce4 smtpd
69183  4684  4684   125  SJ      select   0xc07b7ce4 smtp
69182  4684  4684   125  SJ      select   0xc07b7ce4 cleanup
69175  1940  1940   125  SJ      kqread   0xc993ba00 smtpd
69174  4684  4684   125  SJ      select   0xc07b7ce4 trivial-rewrite
69141 45988 45982    70  SJ      sbwait   0xd0b030a8 postgres
68996 45988 45982    70  SJ      sbwait   0xcf2b1e90 postgres
68790  2260  2260    80  SJ      accept   0xc97fe5ca httpd
68693  1940  1940   125  SJ      kqread   0xcbf11a00 anvil
68338  4557  4557   125  SJ      select   0xc07b7ce4 imapd
68262 45988 45982    70  SJ      sbwait   0xc9f54900 postgres
68199  4557  4557   125  SJ      select   0xc07b7ce4 imapd
68189 45988 45982    70  SJ      sbwait   0xd0afd0a8 postgres
67971  4684  4684   125  SJ      select   0xc07b7ce4 proxymap
67135  4684  4684   125  SJ      select   0xc07b7ce4 pickup
67085  4557  4557   125  SJ      select   0xc07b7ce4 imapd
66421 45988 45982    70  SJ      sbwait   0xd0ac9900 postgres
64620  1940  1940   125  SJ      kqread   0xcbfa9400 pickup
63268 45988 45982    70  SJ      sbwait   0xd0af9d2c postgres
62647 45988 45982    70  SJ      sbwait   0xc9f23370 postgres
62453 45988 45982    70  SJ      sbwait   0xc8d6379c postgres
62324  4141  4141   125  SJ      select   0xc07b7ce4 pickup
59910  2260  2260    80  SJ      accept   0xc97fe5ca httpd
58656  3815  3815    80  SJ      accept   0xc986d892 httpd
58655  3815  3815    80  SJ      accept   0xc986d892 httpd
58641  3815  3815    80  SJ      accept   0xc986d892 httpd
58523  2260  2260    80  SJ      accept   0xc97fe5ca httpd
58520  2260  2260    80  SJ      accept   0xc97fe5ca httpd
58398  2260  2260    80  SJ      accept   0xc97fe5ca httpd
57936  2260  2260    80  SJ      accept   0xc97fe5ca httpd
55518  2260  2260    80  SJ      accept   0xc97fe5ca httpd
49914  2260  2260    80  SJ      accept   0xc97fe5ca httpd
46007     1 46007     0  SsJ     nanslp   0xc076a96c cron
45999     1 45999     0  SsJ     select   0xc07b7ce4 inetd
45993 45992 45982    70  S+J     select   0xc07b7ce4 postgres
45992 45988 45982    70  S+J     select   0xc07b7ce4 postgres
45991 45988 45982    70  S+J     select   0xc07b7ce4 postgres
45989 45988 45982    70  S+J     select   0xc07b7ce4 postgres
45988     1 45982    70  S+J     select   0xc07b7ce4 postgres
45983 45958 45957    88  S+J     (threaded)          mysqld
100467                   S       kserel   0xcf638694 mysqld
100473                   S       kserel   0xcf638694 mysqld
100358                   S       kserel   0xcf638694 mysqld
100362                   S       kserel   0xcf638694 mysqld
100355                   S       select   0xc07b7ce4 mysqld
100948                   S       kserel   0xd092cc94 mysqld
100949                   S       sigwait  0xec446c14 mysqld
100938                   S       ksesigwa 0xcf6aabb0 mysqld
45958     1 45957    88  S+J     wait     0xcf644000 sh
45942     1 45942     0  SsJ     select   0xc07b7ce4 sshd
45876     1 45876     0  SsJ     select   0xc07b7ce4 syslogd
33626  2260  2260    80  SJ      accept   0xc97fe5ca httpd
23070  1336  1336    80  SJ      lockf    0xd0401e40 httpd
23069  1336  1336    80  SJ      kqread   0xca0be100 httpd
23057  1336  1336    80  SJ      lockf    0xcf515680 httpd
22000  1336  1336    80  SJ      lockf    0xc9c80380 httpd
21992  1336  1336    80  SJ      lockf    0xcf515dc0 httpd
20343  1336  1336    80  SJ      lockf    0xc91dd200 httpd
20342  1336  1336    80  SJ      lockf    0xd0458ac0 httpd
20341  3815  3815    80  SJ      accept   0xc986d892 httpd
20340  1336  1336    80  SJ      lockf    0xd0401140 httpd
20339  3815  3815    80  SJ      accept   0xc986d892 httpd
20337  1336  1336    80  SJ      lockf    0xd0458800 httpd
20336  3815  3815    80  SJ      accept   0xc986d892 httpd
20335  3815  3815    80  SJ      accept   0xc986d892 httpd
20334  1336  1336    80  SJ      lockf    0xcf4eb280 httpd
20333  3815  3815    80  SJ      accept   0xc986d892 httpd
20324 20323 20323     0  SJ      piperd   0xd02164c8 sockin
20323  4456 20323     0  SsJ     wait     0xd092e430 sh
78897     0     0     0  SL      ipmireq  0xcf93bec0 [ipmi0: kcs]
65042 64222 64222    70  SJ      sbwait   0xcf0f14d4 postgres
64780 64222 64222    70  SJ      sbwait   0xcabb4900 postgres
64778 64222 64222    70  SJ      sbwait   0xd0af10a8 postgres
64777 64222 64222    70  SJ      sbwait   0xc8f96370 postgres
64769 64222 64222    70  SJ      sbwait   0xca43e20c postgres
64768 64222 64222    70  SJ      sbwait   0xc9a26900 postgres
64767 64222 64222    70  SJ      sbwait   0xd0afd4d4 postgres
64742 64222 64222    70  SJ      sbwait   0xd0afb638 postgres
64666 64222 64222    70  SJ      sbwait   0xccb3579c postgres
64665 64222 64222    70  SJ      sbwait   0xd0afc20c postgres
64664 64222 64222    70  SJ      sbwait   0xc9e4979c postgres
64663 64222 64222    70  SJ      sbwait   0xd05c0e90 postgres
64662 64222 64222    70  SJ      sbwait   0xd0ad3d2c postgres
64661 64222 64222    70  SJ      sbwait   0xca39a79c postgres
64484 64222 64222    70  SJ      sbwait   0xc8d6a370 postgres
64414 64222 64222    70  SJ      sbwait   0xc9f2379c postgres
64361 64222 64222    70  SJ      sbwait   0xd08a1370 postgres
64349 64222 64222    70  SJ      sbwait   0xd05c8370 postgres
64348 64222 64222    70  SJ      sbwait   0xcc9b70a8 postgres
64332 64222 64222    70  SJ      sbwait   0xd0509a64 postgres
64331 64222 64222    70  SJ      sbwait   0xc9a2679c postgres
64227 64226 64222    70  SJ      select   0xc07b7ce4 postgres
64226 64222 64222    70  SJ      select   0xc07b7ce4 postgres
64225 64222 64222    70  SJ      select   0xc07b7ce4 postgres
64223 64222 64222    70  SJ      select   0xc07b7ce4 postgres
64222     1 64222    70  SsJ     select   0xc07b7ce4 postgres
43040  1114 43040    70  SsJ     sbwait   0xca50b370 postgres
75907  4167  4167  1003  RJ      CPU 0               stunnel
46911     1 46911     0  SsJ     nanslp   0xc076a96c cron
46904     1 46904     0  SsJ     select   0xc07b7ce4 sshd
46880     1 46880     0  SsJ     (threaded)          pdns_server
100752                   S       kserel   0xcf45e0f4 pdns_server
100748                   S       kserel   0xcf45e0f4 pdns_server
100695                   S       kserel   0xcf45e0f4 pdns_server
100171                   S       kserel   0xcf45e0f4 pdns_server
100464                   S       sbwait   0xcf7be638 pdns_server
100572                   S       sbwait   0xca872370 pdns_server
100836                   S       accept   0xcac13e22 pdns_server
100834                   S       select   0xc07b7ce4 pdns_server
100784                   S       ksesigwa 0xcf564bb0 pdns_server
46845     1 46845   389  SsJ     (threaded)          slapd
100729                   S       kserel   0xcf638454 slapd
100750                   S       kserel   0xcf638454 slapd
100747                   S       select   0xc07b7ce4 slapd
100630                   S       kserel   0xcf638454 slapd
100512                   S       kserel   0xcf638454 slapd
100733                   S       ksesigwa 0xcf6ab998 slapd
46825     1 46825     0  SsJ     select   0xc07b7ce4 syslogd
68366  5772  5772    70  SJ      sbwait   0xcabb379c postgres
57454  5772  5772    70  SJ      sbwait   0xca4124d4 postgres
94638  5772  5772    70  SJ      sbwait   0xc9f22bc8 postgres
94626  5772  5772    70  SJ      sbwait   0xd052ee90 postgres
85402  5800  5800    80  SJ      accept   0xc9cf4892 httpd
85398  5772  5772    70  SJ      sbwait   0xceb45e90 postgres
85396  5772  5772    70  SJ      sbwait   0xca4b4638 postgres
71130  1114 71130    70  SsJ     sbwait   0xc9eeaa64 postgres
71129  1114 71129    70  SsJ     sbwait   0xcf2b0bc8 postgres
71128  1114 71128    70  SsJ     sbwait   0xc9e484d4 postgres
71127  1114 71127    70  SsJ     sbwait   0xcf7bd20c postgres
71126  1114 71126    70  SsJ     sbwait   0xc8f76900 postgres
71115  1114 71115    70  SsJ     sbwait   0xca50ce90 postgres
71081 71080 71080    89  SJ      (threaded)          java
100360                   S       kserel   0xcf5dbe74 java
100255                   S       kserel   0xcf5dbe74 java
100722                   S       select   0xc07b7ce4 java
100746                   S       kserel   0xcf5dbe74 java
100505                   S       kserel   0xcf5dbe74 java
100409                   S       sbwait   0xceb88bc8 java
100718                   S       sbwait   0xca609900 java
100163                   S       sbwait   0xcf7be79c java
100518                   S       sbwait   0xca6094d4 java
100778                   S       sbwait   0xca609370 java
100258                   S       sbwait   0xc93454d4 java
100506                   S       ksesigwa 0xcf6ad780 java
71080     1 71080     0  SsJ     select   0xc07b7ce4 udb-server
18181  2388  2388    80  SJ      accept   0xc986d19e httpd
16803  2388  2388    80  SJ      accept   0xc986d19e httpd
16802  2388  2388    80  SJ      accept   0xc986d19e httpd
16801  2388  2388    80  SJ      accept   0xc986d19e httpd
16795  2388  2388    80  SJ      accept   0xc986d19e httpd
 7788  5772  5772    70  SJ      sbwait   0xc9e240a8 postgres
 7723  5772  5772    70  SJ      sbwait   0xcabb620c postgres
 7705  5772  5772    70  SJ      sbwait   0xca8dc370 postgres
 7703  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 7700  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 7699  5772  5772    70  SJ      sbwait   0xca39aa64 postgres
 7696  5772  5772    70  SJ      sbwait   0xc9cf3638 postgres
 7694  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 6376  4141  4141   125  SJ      select   0xc07b7ce4 tlsmgr
 5969  3438  3438    80  SJ      lockf    0xce4036c0 httpd
 5968  3438  3438    80  SJ      lockf    0xcaf5dcc0 httpd
 5966  3438  3438    80  SJ      select   0xc07b7ce4 httpd
 5962  3438  3438    80  SJ      lockf    0xc94524c0 httpd
 5854  5772  5772    70  SJ      sbwait   0xca314638 postgres
 5852  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 5851  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 5850  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 5849  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 5848  5800  5800    80  SJ      accept   0xc9cf4892 httpd
 5827  5826  5826  1020  SJ      (threaded)          java
100390                   S       kserel   0xc8e0b034 java
100475                   S       kserel   0xc8e0b034 java
100418                   S       kserel   0xc8e0b034 java
100389                   S       select   0xc07b7ce4 java
100180                   S       kserel   0xc8e0b034 java
100400                   S       sbwait   0xc9e49a64 java
100812                   S       sbwait   0xc9f5420c java
100382                   S       sbwait   0xc9f54e90 java
100394                   S       sbwait   0xc9cf3900 java
100388                   S       sbwait   0xca314370 java
100398                   S       sbwait   0xca02f370 java
100386                   S       ksesigwa 0xc8e0d138 java
 5826     1  5826     0  SsJ     select   0xc07b7ce4 udb-server
 5819     1  5819     0  SsJ     nanslp   0xc076a96c cron
 5812     1  5812     0  SsJ     select   0xc07b7ce4 sshd
 5809  5805  5800     0  SJ      piperd   0xc9cfb7f8 cronolog
 5808  5803  5800     0  SJ      piperd   0xc9e44b28 cronolog
 5805  5800  5800     0  SJ      wait     0xc969aa78 sh
 5803  5800  5800     0  SJ      wait     0xca033648 sh
 5800     1  5800     0  SsJ     nanslp   0xc076a96c httpd
 5788  5787  5772    70  SJ      select   0xc07b7ce4 postgres
 5787  5772  5772    70  SJ      select   0xc07b7ce4 postgres
 5786  5772  5772    70  SJ      select   0xc07b7ce4 postgres
 5772     1  5772    70  SsJ     select   0xc07b7ce4 postgres
 5713     1  5713     0  SsJ     select   0xc07b7ce4 syslogd
 5704     1  5704     0  Ss+     ttyin    0xc8a61410 getty
 5703     1  5703     0  Ss+     ttyin    0xc8a69c10 getty
 5702     1  5702     0  Ss+     ttyin    0xc8a6c010 getty
 5701     1  5701     0  Ss+     ttyin    0xc8a6b010 getty
 5700     1  5700     0  Ss+     ttyin    0xc8a6a410 getty
 5699     1  5699     0  Ss+     ttyin    0xc8a6a810 getty
 5698     1  5698     0  Ss+     ttyin    0xc8a60810 getty
 5697     1  5697     0  Ss+     ttyin    0xc8a6c410 getty
 5696     1  5696     0  Ss+     ttyin    0xc8a6b410 getty
 5675     1  5675     0  Ss      select   0xc07b7ce4 bsnmpd
 5663     1  5663     0  Ss      select   0xc07b7ce4 inetd
 5443  5442  5442  1020  SJ      (threaded)          java
100223                   S       kserel   0xc998b874 java
100095                   S       kserel   0xc998b874 java
100385                   S       sbwait   0xd0509978 java
100414                   S       sbwait   0xd089e814 java
100699                   S       kserel   0xc998b874 java
100187                   S       kserel   0xc998b874 java
100229                   S       select   0xc07b7ce4 java
100736                   S       sbwait   0xd025379c java
100157                   S       sbwait   0xc9644f08 java
100338                   S       ksesigwa 0xc998abb0 java
 5442     1  5442     0  SsJ     select   0xc07b7ce4 udb-server
 5435     1  5435     0  SsJ     nanslp   0xc076a96c cron
 5419     1  5419     0  SsJ     select   0xc07b7ce4 sshd
 5265     1  5265     0  SsJ     select   0xc07b7ce4 syslogd
 4960     1  4960     0  SsJ     nanslp   0xc076a96c cron
 4953     1  4953     0  SsJ     select   0xc07b7ce4 sshd
 4906     1  4906    53  SsJ     select   0xc07b7ce4 named
 4891     1  4891     0  SsJ     select   0xc07b7ce4 syslogd
 4834  4684  4684   125  SJ      select   0xc07b7ce4 anvil
 4789  4684  4684   125  SJ      select   0xc07b7ce4 tlsmgr
 4709     1  4709     0  SsJ     nanslp   0xc076a96c cron
 4699     1  4699     0  SsJ     accept   0xc9dfae22 saslauthd1
 4691  4684  4684   125  SJ      select   0xc07b7ce4 qmgr
 4684     1  4684     0  SsJ     select   0xc07b7ce4 master
 4598     1  4598     0  SsJ     select   0xc07b7ce4 rpc.dracd
 4585  4584  4585     0  S+J     select   0xc07b7ce4 couriertcpd
 4584     1  4584     0  S+J     piperd   0xc9cb6b28 courierlogger
 4581  4532  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4579  4532  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4578  4532  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4577  4532  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4576  4532  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4569  4568  4569     0  S+J     select   0xc07b7ce4 couriertcpd
 4568     1  4568     0  S+J     piperd   0xc8e04330 courierlogger
 4557  4556  4557     0  S+J     select   0xc07b7ce4 couriertcpd
 4556     1  4556     0  S+J     piperd   0xc8f6e000 courierlogger
 4544  4543  4544     0  S+J     select   0xc07b7ce4 couriertcpd
 4543     1  4543     0  S+J     piperd   0xc9cb64c8 courierlogger
 4532  4531  4531     0  S+J     select   0xc07b7ce4 authdaemond
 4531     1  4531     0  S+J     piperd   0xc94917f8 courierlogger
 4517     1  4517     0  SsJ     select   0xc07b7ce4 sshd
 4500     1  4500     0  SsJ     select   0xc07b7ce4 bsnmpd
 4467     1  4467     0  SsJ     select   0xc07b7ce4 rpcbind
 4456     1  4456     0  SsJ     select   0xc07b7ce4 syslogd
 4180     1  4180     0  SsJ     nanslp   0xc076a96c cron
 4172     1  4172     0  SsJ     select   0xc07b7ce4 inetd
 4167     1  4167  1003  SsJ     (threaded)          stunnel
100735                   S       kserel   0xc8e0b454 stunnel
100532                   S       kserel   0xc8e0b454 stunnel
100189                   S       kserel   0xc8e0b454 stunnel
100749                   S       kserel   0xc8e0b454 stunnel
100412                   S       select   0xc07b7ce4 stunnel
100080                   S       ksesigwa 0xc8e05780 stunnel
 4160     1  4160     0  SsJ     accept   0xc986d302 saslauthd1
 4149  4141  4141   125  SJ      select   0xc07b7ce4 qmgr
 4145     1  4145 65534  SsJ     (threaded)          proxsmtpd
100762                   S       kserel   0xc8e0b9f4 proxsmtpd
100383                   S       kserel   0xc8e0b9f4 proxsmtpd
100761                   S       kserel   0xc8e0b9f4 proxsmtpd
100765                   S       kserel   0xc8e0b9f4 proxsmtpd
100740                   S       select   0xc07b7ce4 proxsmtpd
100783                   S       accept   0xc9a97892 proxsmtpd
100516                   S       ksesigwa 0xc9b99dc8 proxsmtpd
 4141     1  4141     0  SsJ     select   0xc07b7ce4 master
 4077     1  4077     0  SsJ     select   0xc07b7ce4 sshd
 4021     1  4021     0  SsJ     select   0xc07b7ce4 syslogd
 3844     1  3844     0  SsJ     select   0xc07b7ce4 inetd
 3830     1  3830     0  SsJ     nanslp   0xc076a96c cron
 3823     1  3823     0  SsJ     select   0xc07b7ce4 sshd
 3815     1  3815     0  SsJ     nanslp   0xc076a96c httpd
 3680     1  3680     0  SsJ     select   0xc07b7ce4 syslogd
 3538  3534  3438     0  SJ      piperd   0xc8f69660 cronolog
 3537  3531  3438     0  SJ      piperd   0xc9311cc0 cronolog
 3536  3532  3438     0  SJ      piperd   0xc9491b28 cronolog
 3535  3438  3438    80  SJ      lockf    0xcdedd2c0 httpd
 3534  3438  3438     0  SJ      wait     0xc8d95a78 sh
 3532  3438  3438     0  SJ      wait     0xc969a648 sh
 3531  3438  3438     0  SJ      wait     0xc9828430 sh
 3486  3450  3449    88  S+J     (threaded)          mysqld
100165                   S       kserel   0xc894d0f4 mysqld
100256                   S       kserel   0xc894d0f4 mysqld
100176                   S       kserel   0xc894d0f4 mysqld
100181                   S       select   0xc07b7ce4 mysqld
100166                   S       kserel   0xc894d0f4 mysqld
100252                   S       kserel   0xc894ea54 mysqld
100253                   S       sigwait  0xec476c14 mysqld
100172                   S       ksesigwa 0xc8a4d780 mysqld
 3479     1  3479     0  SsJ     nanslp   0xc076a96c cron
 3460     1  3460     0  SsJ     select   0xc07b7ce4 inetd
 3450     1  3449    88  S+J     wait     0xc92e4000 sh
 3448  3438  3438    80  SJ      select   0xc07b7ce4 httpd
 3438     1  3438     0  SsJ     select   0xc07b7ce4 httpd
 3326     1  3326     0  SsJ     select   0xc07b7ce4 sshd
 3265     1  3265     0  SsJ     select   0xc07b7ce4 syslogd
 2983     1  2983     0  SsJ     nanslp   0xc076a96c cron
 2976     1  2976     0  SsJ     select   0xc07b7ce4 sshd
 2968     1  2968     0  SsJ     (threaded)          httpauthd
100179                   S       kserel   0xc93a0454 httpauthd
100741                   S       kserel   0xc93a0454 httpauthd
100760                   S       kserel   0xc93a0454 httpauthd
100730                   S       kserel   0xc93a0454 httpauthd
100185                   S       sbwait   0xca02fd2c httpauthd
100593                   S       sbwait   0xceb5c79c httpauthd
100404                   S       sbwait   0xd05c8a64 httpauthd
100191                   S       sbwait   0xca43fd2c httpauthd
100393                   S       sbwait   0xca8dc20c httpauthd
100417                   S       sbwait   0xca41c638 httpauthd
100359                   S       sbwait   0xd05c04d4 httpauthd
100468                   S       sbwait   0xc9f224d4 httpauthd
100273                   S       sbwait   0xca50b638 httpauthd
100408                   S       sbwait   0xca50b0a8 httpauthd
100357                   S       sbwait   0xcabb7a64 httpauthd
100254                   S       sbwait   0xcf2b3900 httpauthd
100173                   S       sbwait   0xcf0f2900 httpauthd
100251                   S       sbwait   0xca50cbc8 httpauthd
100226                   S       sbwait   0xd0af00a8 httpauthd
100221                   S       sbwait   0xc9eeabc8 httpauthd
100391                   S       sbwait   0xca41c900 httpauthd
100162                   S       sbwait   0xcca28a64 httpauthd
100352                   S       sbwait   0xd0b01900 httpauthd
100421                   S       sbwait   0xc97a64d4 httpauthd
100411                   S       sbwait   0xd0b0120c httpauthd
100479                   S       accept   0xc9800892 httpauthd
100169                   S       sbwait   0xca41c370 httpauthd
100396                   S       sbwait   0xca4110a8 httpauthd
100265                   S       sbwait   0xcf2d5d2c httpauthd
100402                   S       sbwait   0xca411e90 httpauthd
100781                   S       sbwait   0xcf0f2bc8 httpauthd
100183                   S       sbwait   0xc95b7bc8 httpauthd
100356                   S       sbwait   0xd0afc638 httpauthd
100502                   S       sbwait   0xc9df8370 httpauthd
100751                   S       sbwait   0xc97ff0a8 httpauthd
100500                   S       sbwait   0xc900b0a8 httpauthd
100513                   S       sbwait   0xc9a794d4 httpauthd
100170                   S       sbwait   0xcabaf20c httpauthd
100476                   S       sbwait   0xc8d6a900 httpauthd
100851                   S       sbwait   0xca39b79c httpauthd
100274                   S       sbwait   0xceb44900 httpauthd
100510                   S       sbwait   0xca8dc79c httpauthd
100936                   S       sbwait   0xc986d4d4 httpauthd
100419                   S       sbwait   0xcabaf900 httpauthd
100711                   S       sbwait   0xd05aca64 httpauthd
100533                   S       sbwait   0xcefa7a64 httpauthd
100250                   S       sbwait   0xd0aef0a8 httpauthd
100422                   S       sbwait   0xcabb3a64 httpauthd
100405                   S       sbwait   0xc91dfd2c httpauthd
100514                   S       sbwait   0xd0ad34d4 httpauthd
100482                   S       sbwait   0xc9f23bc8 httpauthd
100470                   S       sbwait   0xc9e4879c httpauthd
100410                   S       sbwait   0xc96af4d4 httpauthd
100614                   S       sbwait   0xca50c20c httpauthd
100801                   S       sbwait   0xc9eea370 httpauthd
100175                   S       sbwait   0xd0ac9638 httpauthd
100737                   S       sbwait   0xd089ee90 httpauthd
100403                   S       ksesigwa 0xc93a1bb0 httpauthd
 2920     1  2920   389  SsJ     (threaded)          slapd
100186                   S       kserel   0xc8ddedb4 slapd
100497                   S       kserel   0xc8ddedb4 slapd
100763                   S       kserel   0xc8ddedb4 slapd
100469                   S       kserel   0xc8ddedb4 slapd
100471                   S       select   0xc07b7ce4 slapd
100136                   S       ksesigwa 0xc8e0d998 slapd
 2900     1  2900     0  SsJ     select   0xc07b7ce4 syslogd
 2668     1  2668     0  SsJ     select   0xc07b7ce4 slapd
 2661     1  2661     0  SsJ     nanslp   0xc076a96c cron
 2654     1  2654     0  SsJ     select   0xc07b7ce4 sshd
 2599     1  2599     0  SsJ     select   0xc07b7ce4 syslogd
 2547  2388  2388    80  SJ      accept   0xc986d19e httpd
 2546  2388  2388    80  SJ      accept   0xc986d19e httpd
 2545  2388  2388    80  SJ      accept   0xc986d19e httpd
 2544  2388  2388    80  SJ      accept   0xc986d19e httpd
 2543  2388  2388    80  SJ      accept   0xc986d19e httpd
 2403     1  2403     0  SsJ     nanslp   0xc076a96c cron
 2396     1  2396     0  SsJ     select   0xc07b7ce4 sshd
 2388     1  2388     0  SsJ     nanslp   0xc076a96c httpd
 2289     1  2289     0  SsJ     select   0xc07b7ce4 inetd
 2275     1  2275     0  SsJ     nanslp   0xc076a96c cron
 2268     1  2268     0  SsJ     select   0xc07b7ce4 sshd
 2260     1  2260     0  SsJ     nanslp   0xc076a96c httpd
 2206     1  2206     0  SsJ     select   0xc07b7ce4 syslogd
 2006  1951  1950    88  S+J     (threaded)          mysqld
100161                   S       kserel   0xc93a0514 mysqld
100038                   S       kserel   0xc93a0514 mysqld
100167                   S       kserel   0xc93a0514 mysqld
100225                   S       select   0xc07b7ce4 mysqld
100091                   S       kserel   0xc93a0514 mysqld
100222                   S       sigwait  0xec3e1c14 mysqld
100219                   S       ksesigwa 0xc93a1780 mysqld
 1951     1  1950    88  S+J     wait     0xc939f430 sh
 1949  1940  1940   125  SJ      kqread   0xc97a9b00 qmgr
 1940     1  1940     0  SsJ     kqread   0xc969eb00 master
 1800  1795  1800    70  SsJ     select   0xc07b7ce4 postgres
 1799  1795  1799    70  SsJ     nanslp   0xc076a96c postgres
 1798  1795  1798    70  SsJ     nanslp   0xc076a96c postgres
 1797  1795  1797    70  SsJ     nanslp   0xc076a96c postgres
 1795     1  1791    70  S+J     select   0xc07b7ce4 postgres
 1765  1760  1760     0  SJ      lockf    0xcca4ed00 saslauthd
 1763  1760  1760     0  SJ      lockf    0xca55b240 saslauthd
 1762  1760  1760     0  SJ      lockf    0xcf50de40 saslauthd
 1761  1760  1760     0  SJ      accept   0xc8f76cbe saslauthd
 1760     1  1760     0  SsJ     lockf    0xcff97c00 saslauthd
 1731     1  1731     0  SsJ     select   0xc07b7ce4 syslogd
 1374     1  1374     0  SsJ     nanslp   0xc076a96c cron
 1364     1  1364     0  SsJ     select   0xc07b7ce4 sshd
 1336     1  1336     0  SsJ     nanslp   0xc076a96c httpd
 1117  1114  1117    70  SsJ     select   0xc07b7ce4 postgres
 1116  1114  1116    70  SsJ     select   0xc07b7ce4 postgres
 1114     1  1114    70  SsJ     select   0xc07b7ce4 postgres
 1057     1  1057     0  SsJ     select   0xc07b7ce4 syslogd
  691     1   691     0  Ss      nanslp   0xc076a96c cron
  685     1   685    25  Ss      pause    0xc8cf0894 sendmail
  681     1   681     0  Ss      select   0xc07b7ce4 sendmail
  675     1   675     0  Ss      select   0xc07b7ce4 sshd
  655     1   654     0  S       nanslp   0xc076a96c python
  615     1   615     0  Ss      select   0xc07b7ce4 ntpd
  525     1   525     0  Ss      select   0xc07b7ce4 syslogd
  470     1   470     0  Ss      select   0xc07b7ce4 devd
   41     0     0     0  SL      -        0xe9c77d04 [schedcpu]
   40     0     0     0  SL      sdflush  0xc07c9294 [softdepflush]
   39     0     0     0  SL      syncer   0xc076a6dc [syncer]
   38     0     0     0  SL      vlruwt   0xc8a07c90 [vnlru]
   37     0     0     0  SL      psleep   0xc07b8260 [bufdaemon]
   36     0     0     0  SL      pgzero   0xc07ca264 [pagezero]
   35     0     0     0  SL      psleep   0xc07c9d6c [vmdaemon]
   34     0     0     0  SL      psleep   0xc07c9d28 [pagedaemon]
   33     0     0     0  SL      -        0xc8bee580 [dummynet]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq1: atkbd0]
   30     0     0     0  WL                          [irq19: atapci1]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  LL     *Giant    0xcf52b6c0 [irq17: em1]
   26     0     0     0  LL     *Giant    0xcf52b6c0 [irq16: em0]
   25     0     0     0  WL                          [irq9: acpi0]
   24     0     0     0  SL      -        0xc89a2c00 [thread taskq]
   23     0     0     0  SL      -        0xc89a2c80 [acpi_task_2]
   22     0     0     0  SL      -        0xc89a2c80 [acpi_task_1]
    9     0     0     0  SL      -        0xc89a2c80 [acpi_task_0]
   21     0     0     0  WL                          [swi6: Giant taskq]
   20     0     0     0  WL                          [swi6: task queue]
   19     0     0     0  WL                          [swi2: cambio]
    8     0     0     0  SL      ccb_scan 0xc0752ae4 [xpt_thrd]
    7     0     0     0  SL      -        0xc89ef180 [kqueue taskq]
   18     0     0     0  WL                          [swi5: +]
   17     0     0     0  SL      -        0xc0767320 [yarrow]
    6     0     0     0  SL      crypto_r 0xc07c8ec4 [crypto returns]
    5     0     0     0  SL      crypto_w 0xc07c8e84 [crypto]
    4     0     0     0  SL      -        0xc0767e28 [g_down]
    3     0     0     0  SL      -        0xc0767e24 [g_up]
    2     0     0     0  SL      -        0xc0767e1c [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  LL     *Giant    0xcf52b6c0 [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL      CPU 1               [idle: cpu1]
   11     0     0     0  RL      CPU 2               [idle: cpu2]
   10     0     0     0  RL      CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc894f000 [init]
    0     0     0     0  WLs                         [swapper]


# --------------------------------------------------------------------
# CPU INFO

db> show pcpu
cpuid        = 1
curthread    = 0xc894a900: pid 12 "idle: cpu1"
curpcb       = 0xe81f2d90
fpcurthread  = none
idlethread   = 0xc894a900: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:


db> show allpcpu 
Current CPU: 1

cpuid        = 0
curthread    = 0xcf2ba780: pid 75907 "stunnel"
curpcb       = 0xed1b1d90
fpcurthread  = none
idlethread   = 0xc894aa80: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x58
spin locks held:

cpuid        = 1
curthread    = 0xc894a900: pid 12 "idle: cpu1"
curpcb       = 0xe81f2d90
fpcurthread  = none
idlethread   = 0xc894a900: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc894a780: pid 11 "idle: cpu2"
curpcb       = 0xe81efd90
fpcurthread  = none
idlethread   = 0xc894a780: pid 11 "idle: cpu2"
APIC ID      = 2
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc894a600: pid 10 "idle: cpu3"
curpcb       = 0xe81ecd90
fpcurthread  = none
idlethread   = 0xc894a600: pid 10 "idle: cpu3"
APIC ID      = 3
currentldt   = 0x50
spin locks held:


# --------------------------------------------------------------------
# LOCK INFO

db> show locks

db> show alllocks
Process 69677 (python) thread 0xca239900 (100432)
exclusive sleep mutex vm object (standard object) r = 0 (0xce1f3420) locked @ /usr/src/sys/vm/vm_fault.c:688
exclusive sx user map r = 0 (0xcf67f4e4) locked @ /usr/src/sys/vm/vm_map.c:3105
Process 69674 (rsync) thread 0xc8cf5a80 (100132)
exclusive sleep mutex pmap r = 0 (0xcf5b87b0) locked @ /usr/src/sys/i386/i386/pmap.c:2749
exclusive sleep mutex vm page queue mutex r = 0 (0xc07c9ce0) locked @ /usr/src/sys/i386/i386/pmap.c:2748
Process 69664 (rsync) thread 0xca239480 (100485)
exclusive sleep mutex vm object (standard object) r = 0 (0xcc3ad528) locked @ /usr/src/sys/kern/vfs_bio.c:1480
exclusive sleep mutex Giant r = 0 (0xc076a080) locked @ /usr/src/sys/kern/vfs_vnops.c:492


db> show lockedvnods
Locked vnodes

0xccc2e414: tag ufs, type VREG
    usecount 1, writecount 0, refcount 5965 mountedhere 0
    flags ()
    v_object 0xcc3ad528 ref 0 pages 81936
     lock type ufs: SHARED (count 1)#0 0xc04fa690 at lockmgr+0x160
#1 0xc0649d8a at ffs_lock+0x76
#2 0xc06bf8bb at VOP_LOCK_APV+0x87
#3 0xc056a59c at vn_lock+0xac
#4 0xc0569c72 at vn_read+0x132
#5 0xc052c741 at dofileread+0x85
#6 0xc052c5da at kern_readv+0x36
#7 0xc052c505 at read+0x45
#8 0xc06acec3 at syscall+0x25b
#9 0xc06984df at Xint0x80_syscall+0x1f

	ino 13262597, on dev ad8s1e


# --------------------------------------------------------------------
# PROCESS STACK TRACES

# PYTHON
db> trace 69677
Tracing pid 69677 tid 100432 td 0xca239900
sched_switch(ca239900,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c07c9ce0,c8cf5a80,0,c07c9ce0,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c07c9ce0,ca239900,0,c07075e5,15a) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c07c9ce0,0,c07075e5,15a,c06eba6d,...) at _mtx_lock_flags+0xa2
vm_fault(cf67f4a0,8147000,2,8,ca239900,...) at vm_fault+0x311
trap_pfault(ec755d38,1,8147f20,8147f20,0,...) at trap_pfault+0xce
trap(812003b,811003b,bfbf003b,812511c,8147f20,...) at trap+0x1eb
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8073f3c, esp = 0xbfbfda00, ebp = 0xbfbfda28 ---

# RSYNC 
db> trace 69664
Tracing pid 69664 tid 100485 td 0xca239480
sched_switch(ca239480,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c07c9ce0,c8cf5a80,0,c07c9ce0,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c07c9ce0,ca239480,0,c06f2142,5c9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c07c9ce0,0,c06f2142,5c9,cc3ad528,...) at _mtx_lock_flags+0xa2
vfs_vmio_release(dcc09838) at vfs_vmio_release+0x35
getnewbuf(0,0,4000,4000) at getnewbuf+0x276
getblk(ccc2e414,236e,0,4000,0,...) at getblk+0x3b3
cluster_read(ccc2e414,14002000,0,236e,0,...) at cluster_read+0xde
ffs_read(ec74cbf4) at ffs_read+0x2d6
VOP_READ_APV(c0741260,ec74cbf4) at VOP_READ_APV+0x9b
vn_read(cb7676c0,ec74ccbc,c94d6a00,0,ca239480) at vn_read+0x1fb
dofileread(ca239480,5,cb7676c0,ec74ccbc,ffffffff,...) at dofileread+0x85
kern_readv(ca239480,5,ec74ccbc,85063e8,42d10,...) at kern_readv+0x36
read(ca239480,ec74cd04) at read+0x45
syscall(3b,3b,3b,0,430f8,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28185a63, esp = 0xbfbf987c, ebp = 0xbfbf98c8 ---

# RSYNC
db> trace 69674
Tracing pid 69674 tid 100132 td 0xc8cf5a80
sched_switch(c8cf5a80,0,2) at sched_switch+0x177
mi_switch(2,0,c076a040,0,c06eba6d,...) at mi_switch+0x270
critical_exit(c5e1bd38,e9f25c30,c0698a70,0,e9f20008,...) at critical_exit+0x8b
lapic_handle_timer(0) at lapic_handle_timer+0xe1
Xtimerint(c5e1bd38,cf5b87b0,28120000,e9f25c48) at Xtimerint+0x30
pmap_remove_pages(cf5b87b0,0,bfc00000) at pmap_remove_pages+0x1c2
vmspace_exit(c8cf5a80,c076a240,0,c06e6c7d,125,...) at vmspace_exit+0x97
exit1(c8cf5a80,0,e9f25d30,c06acec3,c8cf5a80,...) at exit1+0x496
exit1(c8cf5a80,e9f25d04) at exit1
syscall(3b,3b,3b,8088eda,407,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (1, FreeBSD ELF32, sys_exit), eip = 0x2818520b, esp = 0xbfbfd82c, ebp = 0xbfbfd848 ---

# JAIL-MEASURE
db> trace 69455
Tracing pid 69455 tid 100839 td 0xc9a5d780
sched_switch(c9a5d780,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c9a5d780,0,c06f3435,c1) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06f3435,c1) at _mtx_lock_flags+0xa2
namei(ece75ba0) at namei+0x227
kern_lstat(c9a5d780,80637a8,0,ece75c74) at kern_lstat+0x47
lstat(c9a5d780,ece75d04) at lstat+0x1b
syscall(805003b,805003b,bfbf003b,8063748,8063700,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (190, FreeBSD ELF32, lstat), eip = 0x28159b47, esp = 0xbfbfeafc, ebp = 0xbfbfeb98 ---

# EM1
db> trace 27
Tracing pid 27 tid 100017 td 0xc894c900
sched_switch(c894c900,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c894c900,0,c06e6f85,2a9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06e6f85,2a9) at _mtx_lock_flags+0xa2
ithread_execute_handlers(c89a3648,c89a0480) at ithread_execute_handlers+0xde
ithread_loop(c8a2cae0,e8210d38,c8a2cae0,c04f290c,0,...) at ithread_loop+0x67
fork_exit(c04f290c,c8a2cae0,e8210d38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe8210d6c, ebp = 0 ---

# EM0
db> trace 26
Tracing pid 26 tid 100018 td 0xc894c780
sched_switch(c894c780,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c894c780,0,c06e6f85,2a9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06e6f85,2a9) at _mtx_lock_flags+0xa2
ithread_execute_handlers(c89a3860,c89a0500) at ithread_execute_handlers+0xde
ithread_loop(c8a2cc40,e820dd38,c8a2cc40,c04f290c,0,...) at ithread_loop+0x67
fork_exit(c04f290c,c8a2cc40,e820dd38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe820dd6c, ebp = 0 ---


# --------------------------------------------------------------------
# OTHER INFO

db> x cpu_idle_hlt
cpu_idle_hlt:   0               



--------------010905080203030203030807--




Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20080616111032.E194BF181D4>