Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 31 Aug 2004 08:37:58 -0400 (EDT)
From:      Andrew Gallatin <gallatin@cs.duke.edu>
To:        freebsd-current@freebsd.org
Subject:   RELENG_5 deadlocks on quad amd64
Message-ID:  <16692.28966.142456.796633@grasshopper.cs.duke.edu>

next in thread | raw e-mail | index | archive | help

I'm running RELENG_5 (from Friday, just after ALC's mfc of the IPI
deadlock fix). Older dmesg at
http://people.freebsd.org/~gallatin/quartet/dmesg

Occasionally, it will deadlock.  Usually while building something
(kernel, world, our product, it seems pretty random).  When it is in
this state, its unpingable and sometimes (but not always) responds to
breaks on the serial port.  Enabling debugging options like WITNESS is
enough to prevent the deadlock.

It locked up this morning after a cvsup and a make depend in my kernel
build directory.  I've appended some diagnostic info.  I'll leave it
in this state for a while, in case anybody wants more info.

Thanks,

Drew


[halt sent]
KDB: enter: Line break on console
[thread 100056]
db> tr
kdb_enter() at kdb_enter+0x2f
siointr1() at siointr1+0x3f6
siointr() at siointr+0x5b
intr_execute_handlers() at intr_execute_handlers+0x112
lapic_handle_intr() at lapic_handle_intr+0x21
Xapic_isr1() at Xapic_isr1+0x7d
--- interrupt, rip = 0xffffffff8025f184, rsp = 0xffffffffb30c1b40, rbp = 0xffffffffb30c1b70 ---
_mtx_lock_sleep() at _mtx_lock_sleep+0x114
softclock() at softclock+0x3c2
ithread_loop() at ithread_loop+0xde
fork_exit() at fork_exit+0x8f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffffffb30c1d00, rbp = 0 ---
db> show pcpu
cpuid        = 3
curthread    = 0xffffff00d8a184a0: pid 54 "swi5: clock sio"
curpcb       = 0xffffffffb30c1d10
fpcurthread  = none
idlethread   = 0xffffff00d8a526f0: pid 11 "idle: cpu3"
db> show pcpu 0
cpuid        = 0
curthread    = 0xffffff00d8a35000: pid 14 "idle: cpu0"
curpcb       = 0xffffffffb2f9ad10
fpcurthread  = none
idlethread   = 0xffffff00d8a35000: pid 14 "idle: cpu0"
db> show pcpu 1
cpuid        = 1
curthread    = 0xffffff00d8a52b90: pid 13 "idle: cpu1"
curpcb       = 0xffffffffb2f95d10
fpcurthread  = none
idlethread   = 0xffffff00d8a52b90: pid 13 "idle: cpu1"
db> show pcpu 2
cpuid        = 2
curthread    = 0xffffff00aa4426f0: pid 3328 "cc1"
curpcb       = 0xffffffffb6096d10
fpcurthread  = 0xffffff00aa4426f0: pid 3328 "cc1"
idlethread   = 0xffffff00d8a52940: pid 12 "idle: cpu2"
db> tr 3328
fork_trampoline() at fork_trampoline
db> ps         
  pid   proc     uarea   uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
 3328 ffffff00a8cfc000 ffffffffb6109000    0  2634  2616 0004002 [CPU 2] cc1
 2634 ffffff00a47e12e0 ffffffffb61a0000    0  2633  2616 0004002 [SLPQ wait 0xffffff00a47e12e0][SLP] cc
 2633 ffffff006b13a2e0 ffffffffb5e80000    0  2630  2616 0004002 [SLPQ wait 0xffffff006b13a2e0][SLP] sh
 2630 ffffff00a9387b80 ffffffffb6194000    0  2628  2616 0004002 [SLPQ wait 0xffffff00a9387b80][SLP] xargs
 2628 ffffff006d7135c0 ffffffffb5f49000    0  2622  2616 0004002 [SLPQ wait 0xffffff006d7135c0][SLP] sh
 2622 ffffff00d85c52e0 ffffffffb3ac9000    0  2616  2616 0004002 [SLPQ wait 0xffffff00d85c52e0][SLP] make
 2616 ffffff00a87aa000 ffffffffb619a000    0  2591  2616 0004002 [SLPQ wait 0xffffff00a87aa000][SLP] make
 2591 ffffff006d64a8a0 ffffffffb5f40000 8107  2590  2591 0004002 [SLPQ pause 0xffffff006d64a910][SLP] csh
 2590 ffffff0016a27000 ffffffffb5e2f000 8107  2587  2587 0000100 [SLPQ select 0xffffffff805d1550][SLP] sshd
 2587 ffffff00a93872e0 ffffffffb6191000    0   466  2587 0000100 [SLPQ sbwait 0xffffff00ab466568][SLP] sshd
  528 ffffff00d8a21b80 ffffffffb316a000    0     1   528 0004002 [SLPQ ttyin 0xffffff0000e29c10][SLP] getty
  527 ffffff00abed28a0 ffffffffb6017000    0     1   527 0004002 [SLPQ ttyin 0xffffff0000bb1c10][SLP] getty
  526 ffffff00abcb35c0 ffffffffb601b000    0     1   526 0004002 [SLPQ ttyin 0xffffff0000bb2010][SLP] getty
  525 ffffff00abcb38a0 ffffffffb601c000    0     1   525 0004002 [SLPQ ttyin 0xffffff0000bb2410][SLP] getty
  524 ffffff00abeed5c0 ffffffffb5fb7000    0     1   524 0004002 [SLPQ ttyin 0xffffff0000b73810][SLP] getty
  523 ffffff00abeed8a0 ffffffffb6012000    0     1   523 0004002 [SLPQ ttyin 0xffffff0000af6010][SLP] getty
  522 ffffff00abeedb80 ffffffffb6013000    0     1   522 0004002 [SLPQ ttyin 0xffffff0000b73c10][SLP] getty
  521 ffffff00abeed2e0 ffffffffb5fb6000    0     1   521 0004002 [SLPQ ttyin 0xffffff0000b72c10][SLP] getty
  520 ffffff00abeed000 ffffffffb5fb5000    0     1   520 0004002 [SLPQ ttyin 0xffffff0000b73010][SLP] getty
  489 ffffff00abed2000 ffffffffb6014000    0     1   489 0000000 [SLPQ nanslp 0xffffffff805c4cc0][SLP] cron
  476 ffffff00abed25c0 ffffffffb6016000   25     1   476 0000100 [SLPQ pause 0xffffff00abed2630][SLP] sendmail
  472 ffffff00abed2b80 ffffffffb6018000    0     1   472 0000100 [SLPQ select 0xffffffff805d1550][SLP] sendmail
  466 ffffff00abcb3000 ffffffffb6019000    0     1   466 0000100 [SLPQ select 0xffffffff805d1550][SLP] sshd
  452 ffffff00abbd4000 ffffffffb601e000    0     1   452 0000000 [SLPQ select 0xffffffff805d1550][SLP] ntpd
  379 ffffff006d64a5c0 ffffffffb5f3f000    0     1   379 0000000 [SLPQ select 0xffffffff805d1550][SLP] amd
  350 ffffff006d72e000 ffffffffb5f42000    0     1   350 0000000 [SLPQ select 0xffffffff805d1550][SLP] ypbind
  347 ffffff00acb925c0 ffffffffb5fb2000    0     1   347 0000000 [SLPQ select 0xffffffff805d1550][SLP] rpcbind
  310 ffffff00d819e000 ffffffffb3a69000    0     1   310 0000000 [SLPQ select 0xffffffff805d1550][SLP] syslogd
  290 ffffff006d64a2e0 ffffffffb5ee4000    0     1   290 0000000 [SLPQ select 0xffffffff805d1550][SLP] devd
  260 ffffff006d72e5c0 ffffffffb5f44000    0     1   260 0000000 [SLPQ select 0xffffffff805d1550][SLP] dhclient
  207 ffffff00d85c5000 ffffffffb3ac8000    0     1   207 0000000 [SLPQ pause 0xffffff00d85c5070][SLP] adjkerntz
   77 ffffff00d85c5b80 ffffffffb3acc000    0     0     0 0000204 [SLPQ - 0xffffffff805dad18][SLP] nfsiod 3
   76 ffffff00d8a038a0 ffffffffb311e000    0     0     0 0000204 [SLPQ - 0xffffffff805dad10][SLP] nfsiod 2
   75 ffffff00d8a03b80 ffffffffb311f000    0     0     0 0000204 [SLPQ - 0xffffffff805dad08][SLP] nfsiod 1
   74 ffffff00d8a1d000 ffffffffb3120000    0     0     0 0000204 [SLPQ - 0xffffffff805dad00][SLP] nfsiod 0
   73 ffffff00d8a1d2e0 ffffffffb3121000    0     0     0 0000204 [SLPQ vlruwt 0xffffff00d8a1d2e0][SLP] vnlru
   72 ffffff00d8a1d5c0 ffffffffb3122000    0     0     0 0000204 [SLPQ syncer 0xffffffff805c49c0][SLP] syncer
   71 ffffff00d8a1d8a0 ffffffffb3123000    0     0     0 0000204 [SLPQ psleep 0xffffffff805d1e7c][SLP] bufdaemon
   70 ffffff00d8a1db80 ffffffffb3124000    0     0     0 000020c [SLPQ pgzero 0xffffffff805e36b0][SLP] pagezero
   69 ffffff00d8a5e000 ffffffffb3125000    0     0     0 0000204 [SLPQ psleep 0xffffffff805e371c][SLP] vmdaemon
   68 ffffff00d8a5e2e0 ffffffffb3162000    0     0     0 0000204 [SLPQ psleep 0xffffffff805e36cc][SLP] pagedaemon
   67 ffffff00d8a5e5c0 ffffffffb3163000    0     0     0 0000204 [IWAIT] swi0: sio
   66 ffffff00d8a5e8a0 ffffffffb3164000    0     0     0 0000204 [SLPQ usbevt 0xffffffff80823420][SLP] usb1
   65 ffffff00d8a5eb80 ffffffffb3165000    0     0     0 0000204 [SLPQ usbtsk 0xffffffff805b81d0][SLP] usbtask
   64 ffffff00d8a21000 ffffffffb3166000    0     0     0 0000204 [SLPQ usbevt 0xffffffff80821420][SLP] usb0
    9 ffffff00d8a592e0 ffffffffb30d6000    0     0     0 0000204 [SLPQ actask 0xffffffff805b57c0][SLP] acpi_task2
    8 ffffff00d8a595c0 ffffffffb30d7000    0     0     0 0000204 [SLPQ actask 0xffffffff805b57c0][SLP] acpi_task1
    7 ffffff00d8a598a0 ffffffffb30d8000    0     0     0 0000204 [SLPQ actask 0xffffffff805b57c0][SLP] acpi_task0
   63 ffffff00d8a59b80 ffffffffb30d9000    0     0     0 0000204 [IWAIT] swi6: acpitaskq
   62 ffffff00d8a5f000 ffffffffb30da000    0     0     0 0000204 [IWAIT] swi3: cambio
   61 ffffff00d8a5f2e0 ffffffffb30db000    0     0     0 0000204 [IWAIT] swi2: camnet
   60 ffffff00d8a5f5c0 ffffffffb30dc000    0     0     0 0000204 [IWAIT] swi6: task queue
   59 ffffff00d8a5f8a0 ffffffffb3119000    0     0     0 0000204 [IWAIT] swi6:+
    6 ffffff00d8a5fb80 ffffffffb311a000    0     0     0 0000204 [SLPQ - 0xffffff00009aa280][SLP] thread taskq
   58 ffffff00d8a03000 ffffffffb311b000    0     0     0 0000204 [IWAIT] swi6:+
    5 ffffff00d8a032e0 ffffffffb311c000    0     0     0 0000204 [SLPQ - 0xffffff00009aa400][SLP] kqueue taskq
   57 ffffff00d8a035c0 ffffffffb311d000    0     0     0 0000204 [SLPQ - 0xffffffff805b5f40][SLP] yarrow
    4 ffffff00d8a15000 ffffffffb308f000    0     0     0 0000204 [SLPQ - 0xffffffff805bbce8][SLP] g_down
    3 ffffff00d8a152e0 ffffffffb3090000    0     0     0 0000204 [SLPQ - 0xffffffff805bbce0][SLP] g_up
    2 ffffff00d8a155c0 ffffffffb3091000    0     0     0 0000204 [SLPQ - 0xffffffff805bbcd0][SLP] g_event
   56 ffffff00d8a158a0 ffffffffb3092000    0     0     0 0000204 [IWAIT] swi1: net
   55 ffffff00d8a15b80 ffffffffb3093000    0     0     0 0000204 [IWAIT] swi4: vm
   54 ffffff00d8a56000 ffffffffb3094000    0     0     0 000020c [CPU 3] swi5: clock sio
   53 ffffff00d8a562e0 ffffffffb30d1000    0     0     0 0000204 [IWAIT] irq39:
   52 ffffff00d8a565c0 ffffffffb30d2000    0     0     0 0000204 [IWAIT] irq38:
   51 ffffff00d8a568a0 ffffffffb30d3000    0     0     0 0000204 [IWAIT] irq37:
   50 ffffff00d8a56b80 ffffffffb30d4000    0     0     0 0000204 [IWAIT] irq36:
   49 ffffff00d8a59000 ffffffffb30d5000    0     0     0 0000204 [IWAIT] irq35:
   48 ffffff00d8a23000 ffffffffb3049000    0     0     0 0000204 [IWAIT] irq34:
   47 ffffff00d8a232e0 ffffffffb304a000    0     0     0 0000204 [IWAIT] irq33:
   46 ffffff00d8a235c0 ffffffffb304b000    0     0     0 0000204 [IWAIT] irq32:
   45 ffffff00d8a238a0 ffffffffb304c000    0     0     0 0000204 [RUNQ] irq31: bge0
   44 ffffff00d8a23b80 ffffffffb304d000    0     0     0 0000204 [IWAIT] irq30: mpt1
   43 ffffff00d8a27000 ffffffffb308a000    0     0     0 0000204 [IWAIT] irq29: mpt0
   42 ffffff00d8a272e0 ffffffffb308b000    0     0     0 0000204 [IWAIT] irq28: bge1
   41 ffffff00d8a275c0 ffffffffb308c000    0     0     0 0000204 [IWAIT] irq27:
   40 ffffff00d8a278a0 ffffffffb308d000    0     0     0 0000204 [IWAIT] irq26:
   39 ffffff00d8a27b80 ffffffffb308e000    0     0     0 0000204 [IWAIT] irq25:
   38 ffffff00d8a0b2e0 ffffffffb3004000    0     0     0 0000204 [IWAIT] irq24:
   37 ffffff00d8a0b5c0 ffffffffb3005000    0     0     0 0000204 [IWAIT] irq23:
   36 ffffff00d8a0b8a0 ffffffffb3006000    0     0     0 0000204 [IWAIT] irq22:
   35 ffffff00d8a0bb80 ffffffffb3007000    0     0     0 0000204 [IWAIT] irq21:
   34 ffffff00d8a37000 ffffffffb3044000    0     0     0 0000204 [IWAIT] irq20:
   33 ffffff00d8a372e0 ffffffffb3045000    0     0     0 0000204 [RUNQ] irq19: ohci0 ohci1
   32 ffffff00d8a375c0 ffffffffb3046000    0     0     0 0000204 [IWAIT] irq18:
   31 ffffff00d8a378a0 ffffffffb3047000    0     0     0 0000204 [IWAIT] irq17:
   30 ffffff00d8a37b80 ffffffffb3048000    0     0     0 0000204 [IWAIT] irq16:
   29 ffffff00d8a208a0 ffffffffb2fde000    0     0     0 0000204 [IWAIT] irq15: ata1
   28 ffffff00d8a20b80 ffffffffb2fdf000    0     0     0 0000204 [IWAIT] irq14: ata0
   27 ffffff00d8a28000 ffffffffb2fe0000    0     0     0 0000204 [IWAIT] irq13:
   26 ffffff00d8a282e0 ffffffffb2fff000    0     0     0 0000204 [IWAIT] irq12:
   25 ffffff00d8a285c0 ffffffffb3000000    0     0     0 0000204 [IWAIT] irq11:
   24 ffffff00d8a288a0 ffffffffb3001000    0     0     0 0000204 [IWAIT] irq10:
   23 ffffff00d8a28b80 ffffffffb3002000    0     0     0 0000204 [IWAIT] irq9: acpi0
   22 ffffff00d8a0b000 ffffffffb3003000    0     0     0 0000204 [IWAIT] irq8: rtc
   21 ffffff00d8a5b2e0 ffffffffb2fb9000    0     0     0 0000204 [IWAIT] irq7:
   20 ffffff00d8a5b5c0 ffffffffb2fba000    0     0     0 0000204 [IWAIT] irq6: fdc0
   19 ffffff00d8a5b8a0 ffffffffb2fd9000    0     0     0 0000204 [IWAIT] irq5:
   18 ffffff00d8a5bb80 ffffffffb2fda000    0     0     0 0000204 [IWAIT] irq4: sio0
   17 ffffff00d8a20000 ffffffffb2fdb000    0     0     0 0000204 [IWAIT] irq3:
   16 ffffff00d8a202e0 ffffffffb2fdc000    0     0     0 0000204 [IWAIT] irq0: clk
   15 ffffff00d8a205c0 ffffffffb2fdd000    0     0     0 0000204 [IWAIT] irq1:
   14 ffffff00d8a51000 ffffffffb2f77000    0     0     0 000020c [CPU 0] idle: cpu0
   13 ffffff00d8a512e0 ffffffffb2fb4000    0     0     0 000020c [CPU 1] idle: cpu1
   12 ffffff00d8a515c0 ffffffffb2fb5000    0     0     0 000020c [Can run] idle: cpu2
   11 ffffff00d8a518a0 ffffffffb2fb6000    0     0     0 000020c [Can run] idle: cpu3
    1 ffffff00d8a51b80 ffffffffb2fb7000    0     0     1 0004200 [SLPQ wait 0xffffff00d8a51b80][SLP] init
   10 ffffff00d8a5b000 ffffffffb2fb8000    0     0     0 0000204 [SLPQ ktrace 0xffffffff805c1610][SLP] ktrace
    0 ffffffff805bbe60 ffffffff806b2000    0     0     0 0000200 [SLPQ sched 0xffffffff805bbe60][SLP] swapper




Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?16692.28966.142456.796633>