Date: Thu, 9 Sep 2004 18:40:22 -0400 (EDT) From: Andrew Gallatin <gallatin@cs.duke.edu> To: Julian Elischer <julian@elischer.org> Cc: freebsd-threads@freebsd.org Subject: Re: Unkillable KSE threaded proc Message-ID: <16704.56278.102480.817628@grasshopper.cs.duke.edu> In-Reply-To: <4140C04D.1060906@elischer.org> References: <16703.11479.679335.588170@grasshopper.cs.duke.edu> <16703.12410.319869.29996@grasshopper.cs.duke.edu> <413F55B8.50003@elischer.org> <16703.28031.454342.774229@grasshopper.cs.duke.edu> <413F8DBB.5040502@elischer.org> <16704.40876.708925.425911@grasshopper.cs.duke.edu> <4140AA2A.90605@elischer.org> <16704.45327.42494.922427@grasshopper.cs.duke.edu> <4140C04D.1060906@elischer.org>
next in thread | previous in thread | raw e-mail | index | archive | help
Here it is with WITNESS, after hacking it to enter my driver's spinlocks into the order list, and to expand by 4x this size of the witness pool.. db> ps pid proc uarea uid ppid pgrp flag stat wmesg wchan cmd 595 c16478c0 e52e2000 1387 1 593 000c482 (threaded) mx_pingpong thread 0xc35ff960 ksegrp 0xc15ba7e0 [CPU 0] thread 0xc35ffaf0 ksegrp 0xc15ba7e0 [SUSP] thread 0xc35ffc80 ksegrp 0xc15ba7e0 [RUNQ] thread 0xc35ffe10 ksegrp 0xc1880e70 [CPU 1] db> sho pcpu cpuid = 0 curthread = 0xc35ff960: pid 595 "mx_pingpong" curpcb = 0xe782dda0 fpcurthread = none idlethread = 0xc1561640: pid 12 "idle: cpu0" APIC ID = 0 currentldt = 0x30 spin locks held: exclusive spin mutex sio r = 0 (0xc0702220) locked @ dev/sio/sio.c:1709 db> sho locks exclusive sleep mutex process lock r = 0 (0xc164792c) locked @ kern/kern_exit.c:136 exclusive spin mutex sio r = 0 (0xc0702220) locked @ dev/sio/sio.c:1709 db> sho pcpu 1 cpuid = 1 curthread = 0xc35ffe10: pid 595 "mx_pingpong" curpcb = 0xe7836da0 fpcurthread = none idlethread = 0xc15614b0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x30 spin locks held: db> call db_trace_thread(0xc35ffaf0, -1) sched_switch(c35ffaf0,c35ff960,0,11d,f187c1ca) at sched_switch+0xfd mi_switch(1,c35ff960,c065f8ac,335,c164792c) at mi_switch+0x2a0 thread_single(1,0,c065c2a3,88,e7830c68) at thread_single+0x1d7 exit1(c35ffaf0,9,c065eebc,996,1) at exit1+0xd5 expand_name(c35ffaf0,9,c065eebc,928,0) at expand_name postsig(9,0,c0661a77,100,1020800) at postsig+0x1e0 ast(e7830d48) at ast+0x48a doreti_ast() at doreti_ast+0x17 0 db> call db_trace_thread(0xc35ffc80, -1) sched_switch(c35ffc80,0,0,117,7b0cc0ca) at sched_switch+0xfd mi_switch(2,0,c0661a77,f5,1010000) at mi_switch+0x2a0 ast(e7833d48) at ast+0x3dd doreti_ast() at doreti_ast+0x17 0 db> call db_trace_thread(0xc35ffe10, -1) sched_switch(18e,bb9,c1880e70,1e,0) at sched_switch+0xfd __func__.0() at __func__.0+0xac79 0 db> Show witness is kinda long. Sleep locks: 0 mxsleep(0,2): es->wait_sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,2): es->cmd_sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,1): es->wait_sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,1): es->cmd_sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,0): mapper sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,-1): route update sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mxsleep(0,-1): route cmd sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 0 mx(-1,0): mx mapper mapbuf -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_msgbuf.c:84 0 mx(-1,0): mx mapper msgbuf -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_msgbuf.c:84 0 ATAPI CD bioqueue lock -- last acquired @ dev/ata/atapi-cd.c:1100 0 g_xdown -- last acquired @ geom/geom_io.c:392 4 ATA queue lock -- last acquired @ dev/ata/ata-queue.c:172 4 bio queue -- last acquired @ geom/geom_io.c:65 4 ATA disk bioqueue lock -- last acquired @ dev/ata/ata-disk.c:236 12 system map -- last acquired @ vm/vm_map.c:2313 13 kmem object -- last acquired @ vm/vm_kern.c:398 14 vm page queue mutex -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:842 15 vnode interlock -- last acquired @ kern/vfs_subr.c:2099 16 spechash -- last acquired @ kern/vfs_subr.c:1903 16 Syncer mtx -- last acquired @ kern/vfs_subr.c:1504 16 vnode_free_list -- last acquired @ kern/vfs_subr.c:3217 16 cdev -- last acquired @ kern/kern_conf.c:81 17 Malloc Stats -- last acquired @ kern/kern_malloc.c:185 17 UMA pcpu -- last acquired @ vm/uma_core.c:1800 18 KMAP ENTRY -- last acquired @ vm/uma_core.c:2224 19 UMA zone -- last acquired @ vm/uma_core.c:1817 16 Name Cache -- last acquired @ kern/vfs_cache.c:352 15 CMAPCADDR12 -- last acquired @ i386/i386/pmap.c:2488 15 pmap -- last acquired @ i386/i386/pmap.c:847 16 uma object -- last acquired @ vm/uma_core.c:963 17 UMA pcpu -- (already displayed) 13 kernel object -- last acquired @ vm/vm_object.c:454 14 vm page queue mutex -- (already displayed) 0 g_xup -- last acquired @ geom/geom_io.c:449 1 g_disk_done -- last acquired @ geom/geom_disk.c:196 4 bio queue -- (already displayed) 17 UMA pcpu -- (already displayed) 3 Giant -- last acquired @ kern/kern_timeout.c:247 4 kobj -- last acquired @ kern/subr_kobj.c:298 4 struct mount mtx -- last acquired @ kern/vfs_subr.c:901 15 vnode interlock -- (already displayed) 4 bounce pages lock -- last acquired @ i386/i386/busdma_machdep.c:860 4 vm86 lock -- last acquired @ i386/i386/vm86.c:582 4 standard object -- last acquired @ vm/vm_object.c:454 5 dev_pager list -- last acquired @ vm/device_pager.c:163 5 vm object_list -- last acquired @ vm/vm_object.c:643 14 vm page queue mutex -- (already displayed) 4 udp -- last acquired @ netinet/udp_usrreq.c:263 5 UMA lock -- last acquired @ vm/uma_core.c:1466 12 system map -- (already displayed) 5 udpinp -- last acquired @ netinet/udp_usrreq.c:401 6 arc4_mtx -- last acquired @ libkern/arc4random.c:137 6 accept -- last acquired @ kern/uipc_socket.c:334 7 so_snd -- last acquired @ kern/uipc_socket.c:2091 8 so_rcv -- last acquired @ kern/uipc_socket.c:2092 9 radix node head -- last acquired @ net/route.c:148 10 ifnet -- last acquired @ net/if.c:697 10 rtentry -- last acquired @ netinet/ip_output.c:824 11 rts_inq -- last acquired @ net/netisr.c:231 11 network driver -- last acquired @ dev/fxp/if_fxp.c:1225 12 bpf interface lock -- last acquired @ net/bpf.c:1155 13 bpf cdev lock -- last acquired @ net/bpf.c:1157 14 sellck -- last acquired @ kern/sys_generic.c:726 12 if send queue -- last acquired @ dev/fxp/if_fxp.c:1267 12 knlist lock for lockless objects -- last acquired @ kern/kern_event.c:1598 12 system map -- (already displayed) 11 ifaddr -- last acquired @ net/if.c:594 9 process lock -- last acquired @ kern/kern_exit.c:136 10 ktrace -- last acquired @ kern/kern_exit.c:347 10 sigacts -- last acquired @ kern/subr_trap.c:256 10 struct pargs.ref -- last acquired @ kern/kern_proc.c:1100 10 session -- last acquired @ kern/kern_proc.c:464 15 vnode interlock -- (already displayed) 11 tty -- last acquired @ kern/tty.c:2746 11 uidinfo hash -- last acquired @ kern/kern_resource.c:1004 12 sleep mtxpool -- last acquired @ kern/kern_descrip.c:1896 12 uidinfo struct -- last acquired @ order list:0 13 allprison -- last acquired @ kern/kern_jail.c:460 4 GEOM orphanage -- last acquired @ geom/geom_event.c:170 4 ATA disk bioqueue lock -- (already displayed) 4 ithread -- last acquired @ kern/kern_intr.c:276 4 kernel linker -- last acquired @ kern/kern_linker.c:461 4 protect sysfilt_ops -- last acquired @ kern/kern_event.c:667 4 TID lock -- last acquired @ kern/kern_thread.c:206 4 rman head -- last acquired @ kern/subr_rman.c:111 4 rman -- last acquired @ kern/subr_rman.c:448 12 system map -- (already displayed) 4 bio queue -- (already displayed) 4 taskqueue list -- last acquired @ kern/subr_taskqueue.c:85 4 sf_buf -- last acquired @ i386/i386/vm_machdep.c:674 4 domain list -- last acquired @ kern/uipc_domain.c:110 4 buffer daemon lock -- last acquired @ kern/vfs_bio.c:401 4 ttylist -- last acquired @ kern/tty.c:2745 11 tty -- (already displayed) 4 if_cloners lock -- last acquired @ net/if_clone.c:199 4 pseudofs -- last acquired @ fs/pseudofs/pseudofs_fileno.c:86 4 pbuf mutex -- last acquired @ vm/vm_pager.c:414 4 accounting -- last acquired @ kern/kern_acct.c:232 4 if_clone lock -- last acquired @ net/if_clone.c:304 4 pfil_head_mtx -- last acquired @ net/pfil.c:166 5 pfil_head_list lock -- last acquired @ net/pfil.c:172 4 bdone lock -- last acquired @ kern/vfs_bio.c:3759 4 nfsd_mtx -- last acquired @ nfsserver/nfs_srvsock.c:811 4 lo_mtx -- last acquired @ net/if_loop.c:154 4 fdesc -- last acquired @ kern/kern_descrip.c:1614 5 filedesc structure -- last acquired @ kern/kern_descrip.c:1926 6 accept -- (already displayed) 6 devd -- last acquired @ kern/subr_bus.c:497 14 sellck -- (already displayed) 6 pipe mutex -- last acquired @ kern/sys_pipe.c:1520 14 sellck -- (already displayed) 7 sigio lock -- last acquired @ kern/kern_descrip.c:729 8 process group -- last acquired @ kern/kern_proc.c:458 9 process lock -- (already displayed) 4 mntid -- last acquired @ kern/vfs_subr.c:407 5 mountlist -- last acquired @ kern/vfs_subr.c:3464 4 pseudofs_vncache -- last acquired @ fs/pseudofs/pseudofs_vncache.c:239 4 ATA queue lock -- (already displayed) 4 taskqueue -- last acquired @ kern/subr_taskqueue.c:193 4 devstat -- last acquired @ kern/subr_devstat.c:190 4 buf queue lock -- last acquired @ kern/vfs_bio.c:1505 15 vnode interlock -- (already displayed) 4 ufs ihash -- last acquired @ ufs/ufs/ufs_ihash.c:120 15 vnode interlock -- (already displayed) 4 dirhash list -- last acquired @ ufs/ufs/ufs_dirhash.c:348 5 dirhash -- last acquired @ ufs/ufs/ufs_dirhash.c:349 4 needsbuffer lock -- last acquired @ kern/vfs_bio.c:296 4 runningbufspace lock -- last acquired @ kern/vfs_bio.c:314 4 eventhandler -- last acquired @ kern/subr_eventhandler.c:213 5 eventhandler list -- last acquired @ kern/kern_exit.c:199 17 Malloc Stats -- (already displayed) 17 UMA pcpu -- (already displayed) 4 rtsock route_cb lock -- last acquired @ net/rtsock.c:234 4 tcp -- last acquired @ netinet/tcp_timer.c:138 5 tcpinp -- last acquired @ netinet/tcp_input.c:737 6 tcp_hc_entry -- last acquired @ netinet/tcp_hostcache.c:287 12 system map -- (already displayed) 6 random reseed -- last acquired @ dev/random/yarrow.c:193 6 arc4_mtx -- (already displayed) 6 so_glabel -- last acquired @ kern/uipc_socket.c:282 6 accept -- (already displayed) 4 malloc -- last acquired @ kern/kern_malloc.c:518 17 Malloc Stats -- (already displayed) 4 mx(0,2): es->sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:539 4 bpf global lock -- last acquired @ net/bpf.c:1445 4 if_afdata -- last acquired @ net/if.c:487 4 unp -- last acquired @ kern/uipc_usrreq.c:247 6 accept -- (already displayed) 4 mxsleep(0,-1): is->sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:774 4 mx(-1,-1): mx_global_mutex -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_instance.c:1578 5 mx(0,-1): is->sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_common.c:952 4 mx(0,1): es->sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/mx.c:539 4 mx(-1,0): peer sync -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_peer.c:248 0 arp_inq -- last acquired @ net/netisr.c:231 0 igmp_mtx -- last acquired @ netinet/igmp.c:431 0 ip_inq -- last acquired @ net/netisr.c:231 0 ipqlock -- last acquired @ netinet/ip_input.c:1092 17 UMA pcpu -- (already displayed) 0 sem -- last acquired @ kern/sysv_sem.c:1174 0 GEOM topology -- last acquired @ geom/geom_event.c:202 4 bdone lock -- (already displayed) 4 GEOM orphanage -- (already displayed) 4 devstat -- (already displayed) 1 fdc lock -- last acquired @ dev/fdc/fdc.c:748 2 callout_wait_lock -- last acquired @ kern/kern_timeout.c:289 1 swapdev -- last acquired @ vm/swap_pager.c:2124 12 system map -- (already displayed) 4 ATA queue lock -- (already displayed) 4 bio queue -- (already displayed) 0 p_peers -- last acquired @ kern/kern_exit.c:243 0 module subsystem sx lock -- last acquired @ kern/kern_module.c:113 0 rawcb -- last acquired @ net/raw_usrreq.c:80 8 so_rcv -- (already displayed) 0 sysctl lock -- last acquired @ kern/kern_sysctl.c:1315 1 rip -- last acquired @ netinet/raw_ip.c:195 17 UMA pcpu -- (already displayed) 1 filelist lock -- last acquired @ kern/kern_descrip.c:1388 5 filedesc structure -- (already displayed) 1 allproc -- last acquired @ kern/kern_exit.c:690 2 user map -- last acquired @ vm/vm_map.c:301 3 Giant -- (already displayed) 0 kernel environment -- last acquired @ kern/kern_environment.c:285 0 dev_pager create -- last acquired @ vm/device_pager.c:150 4 standard object -- (already displayed) 0 ddp_list_mtx -- last acquired @ order list:0 1 ddp_mtx -- last acquired @ order list:0 0 slip_mtx -- last acquired @ order list:0 1 slip sc_mtx -- last acquired @ order list:0 0 proctree -- last acquired @ kern/kern_exit.c:583 1 allproc -- (already displayed) Spin locks: 0 ap boot -- last acquired @ i386/i386/mp_machdep.c:517 1 sio -- last acquired @ dev/sio/sio.c:1709 2 cy -- last acquired @ order list:0 3 uart_hwmtx -- last acquired @ order list:0 4 sabtty -- last acquired @ order list:0 5 zstty -- last acquired @ order list:0 6 ng_node -- last acquired @ order list:0 7 ng_worklist -- last acquired @ order list:0 8 taskqueue_fast -- last acquired @ order list:0 9 intr table -- last acquired @ i386/i386/intr_machdep.c:89 10 ithread table lock -- last acquired @ order list:0 11 sleepq chain -- last acquired @ kern/subr_sleepqueue.c:223 12 sched lock -- last acquired @ kern/kern_clock.c:382 13 turnstile chain -- last acquired @ kern/subr_turnstile.c:411 14 td_contested -- last acquired @ kern/subr_turnstile.c:712 15 callout -- last acquired @ kern/kern_clock.c:227 16 entropy harvest -- last acquired @ dev/random/randomdev_soft.c:248 17 entropy harvest buffers -- last acquired @ dev/random/randomdev_soft.c:270 18 allpmaps -- last acquired @ i386/i386/pmap.c:1146 19 vm page queue free mutex -- last acquired @ vm/vm_page.c:1076 20 icu -- last acquired @ order list:0 21 smp rendezvous -- last acquired @ i386/i386/pmap.c:663 22 tlb -- last acquired @ order list:0 23 clk -- last acquired @ i386/isa/clock.c:404 24 mutex profiling lock -- last acquired @ order list:0 25 kse zombie lock -- last acquired @ kern/kern_thread.c:383 26 ALD Queue -- last acquired @ order list:0 27 pcicfg -- last acquired @ i386/pci/pci_cfgreg.c:205 28 kreqq spinlock -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mi_common.c:444 29 is->cmdq.spinlock -- last acquired @ /home/gallatin/mx/tiki/driver/freebsd/../common/mx_lanai_command.c:245 Locks which were never acquired: <...> No dump... it failed in witness.. Drew
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?16704.56278.102480.817628>