Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 23 Feb 2006 15:53:48 +0300
From:      Denis Shaposhnikov <dsh@vlink.ru>
To:        freebsd-net@freebsd.org
Subject:   network lock on CURRENT, em0
Message-ID:  <874q2qs0bn.fsf@neva.vlink.ru>

next in thread | raw e-mail | index | archive | help
Hi!

I have too bad situation. My SMP system

FreeBSD sagitta.internal.vlink.ru 7.0-CURRENT FreeBSD 7.0-CURRENT #3: Thu Feb 23 12:38:04 MSK 2006     dsh@build.rigel.internal.vlink.ru:/usr/obj/usr/src/sys/MYSMPIC  i386

with todays CURRENT can't live longer than several minutes with Squid
running (I mean with my full network load). It locks and do not
respond to pings. I've got DDB and called doadump for two such locks,
so I have two vmcore files and debug kernel. Here an info from DDB for
both:

~KDB: enter: Line break on console
[thread pid 21 tid 100023 ]
Stopped at      kdb_enter+0x30: leave   
db> wh
Tracing pid 21 tid 100023 td 0xc60754e0
kdb_enter(c067dbd9,e65deac8,0,c60754e0,c60cdc00) at kdb_enter+0x30
siointr1(c60cdc00,4,e65deb04,c05f0009,c0c6bb40) at siointr1+0xd1
siointr(c60cdc00,cb21120c,e65deb1c,0,c60754e0) at siointr+0x76
intr_execute_handlers(c5fcaca0,e65deb30,e65deb90,c0632773,37) at intr_execute_handlers+0xfd
lapic_handle_intr(37) at lapic_handle_intr+0x3a
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc05f06e5, esp = 0xe65deb70, ebp = 0xe65deb90 ---
uma_zfree_internal(c0c5ed20,cb20ba00,0,2,0) at uma_zfree_internal+0x1ec
uma_zalloc_bucket(c0c5ed20,1,0,1,2db5e800) at uma_zalloc_bucket+0x261
uma_zalloc_arg(c0c5ed20,e65dec20,1,e65dec14,c60754e0) at uma_zalloc_arg+0x30a
em_get_buf(0,c60754e0,c60a2800,e65dec60,c04e5a00) at em_get_buf+0x146
em_rxeof(c60754e0,e65decb8,c60754e0,c609a9cc,1) at em_rxeof+0x19e
em_handle_rxtx(c609a800,1,0,ffffffff,0) at em_handle_rxtx+0x83
taskqueue_run(c60a2800,c60a281c,c066891d,0,c0506697) at taskqueue_run+0x17f
taskqueue_thread_loop(c609a9dc,e65ded38,ffffffff,c609a9dc,0) at taskqueue_thread_loop+0xc0
fork_exit(c0506697,c609a9dc,e65ded38) at fork_exit+0x7e
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe65ded6c, ebp = 0 ---



~KDB: enter: Line break on console
[thread pid 21 tid 100023 ]
Stopped at      kdb_enter+0x30: leave   
db> wh
Tracing pid 21 tid 100023 td 0xc60754e0
kdb_enter(c067dbd9,c60a2800,e0,c60754e0,c60cdc00) at kdb_enter+0x30
siointr1(c60cdc00,c60754e0,e65deac4,46,c609a9cc) at siointr1+0xd1
siointr(c60cdc00,e65deb04,c05f0009,0,c60754e0) at siointr+0x76
intr_execute_handlers(c5fcaca0,e65deb10,e65deb70,c0632773,37) at intr_execute_handlers+0xfd
lapic_handle_intr(37) at lapic_handle_intr+0x3a
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc05f067f, esp = 0xe65deb50, ebp = 0xe65deb70 ---
uma_zfree_internal(c0c6bb40,cb27da3c,0,0,2) at uma_zfree_internal+0x186
bucket_free(cb27da3c,cb26ce00,0,2,0) at bucket_free+0x3c
uma_zalloc_bucket(c0c5ed20,1,0,1,36dae800) at uma_zalloc_bucket+0x2fb
uma_zalloc_arg(c0c5ed20,e65dec20,1,e65dec14,c60754e0) at uma_zalloc_arg+0x30a
em_get_buf(0,c60754e0,c60a2800,e65dec60,c04e5a00) at em_get_buf+0x146
em_rxeof(c60754e0,e65decb8,c60754e0,c609a9cc,1) at em_rxeof+0x19e
em_handle_rxtx(c609a800,1,0,ffffffff,0) at em_handle_rxtx+0x83
taskqueue_run(c60a2800,c60a281c,c066891d,0,c0506697) at taskqueue_run+0x17f
taskqueue_thread_loop(c609a9dc,e65ded38,ffffffff,c609a9dc,0) at taskqueue_thread_loop+0xc0
fork_exit(c0506697,c609a9dc,e65ded38) at fork_exit+0x7e
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe65ded6c, ebp = 0 ---
db> show intr
irq1: atkbd0 (pid 27)
irq3: sio1 (no thread)
irq4: sio0 (no thread)
irq9: acpi0 (pid 20)
irq14: ata0 (pid 23) {ENTROPY}
irq15: ata1 (pid 24) {ENTROPY}
irq18: atapci1 (pid 25) {ENTROPY}
irq26: em0 (no thread)
irq27: em1 (no thread)
swi1: net (pid 12) {SOFT}
swi4: clock sio (pid 13) {SOFT}
swi3: vm (pid 14) {SOFT}
swi5: + (pid 16) {SOFT}
swi2: cambio (pid 17) {SOFT}
swi6: task queue (pid 18) {SOFT}
swi6: Giant taskq (pid 19) {SOFT}
swi0: sio (pid 26) {SOFT}
db> show uma
              Zone       Allocs        Frees         Used    Cache
           NFSNODE            1            0            1       17
          NFSMOUNT            1            0            1       15
       FFS2 dinode        10865          489        10376       49
       FFS1 dinode            0            0            0        0
         FFS inode        10865          489        10376       93
          SWAPMETA            0            0            0        0
            pfosfp          188            0          188      193
          pfospfen          345            0          345       51
         pfiaddrpl            0            0            0        0
      pfstatescrub            0            0            0        0
          pffrcent            0            0            0        0
         pffrcache            0            0            0        0
            pffrag            0            0            0        0
           pffrent            0            0            0        0
        pfrkentry2            0            0            0        0
         pfrkentry            4            0            4       46
         pfrktable            4            2            2        4
      pfpooladdrpl           10            0           10      102
          pfaltqpl            0            0            0        0
         pfstatepl         6604         5667          937     2123
          pfrulepl           17            0           17        7
         pfsrctrpl            0            0            0        0
           rtentry           37            0           37       59
             ripcb            0            0            0        0
          sackhole         1396         1396            0      507
          tcpreass         7381         7353           28      479
         hostcache          250            0          250       50
          syncache         6000         6000            0      273
             tcptw         2522         2522            0      858
             tcpcb        11864        10190         1674      174
             inpcb        11864        10190         1674      741
             udpcb          460          433           27       65
               ipq         1353         1353            0      339
             unpcb          124           63           61       51
            socket        12450        10688         1762       98
             KNOTE         1911         1895           16      208
            itimer            0            0            0        0
          ksiginfo          133            0          133       59
              PIPE          623          580           43       47
           DIRHASH         2407          641         1766       74
       L VFS Cache           23           14            9       30
       S VFS Cache        30193        20274         9919       49
             NAMEI       111764       111764            0       80
         VNODEPOLL            4            0            4      114
             VNODE        48747        38273        10474       41
     ata_composite            0            0            0        0
       ata_request       156399       156399            0      630
             g_bio       632655       632655            0     2291
      ACL UMA zone            0            0            0        0
         mtag_vlan            0            0            0        0
   mbuf_ext_refcnt            0            0            0        0
    mbuf_jumbo_16k            0            0            0        0
     mbuf_jumbo_9k            0            0            0        0
mbuf_jumbo_pagesize            0            0            0        0
      mbuf_cluster        25747          152        25595        5
              mbuf       318137       318052           85      510
       mbuf_packet      6801246      6775720        25526       69
           VMSPACE         1765         1685           80       40
            UPCALL            3            0            3      153
            KSEGRP          185            2          183       57
            THREAD        12924        12713          211       50
              PROC         1802         1685          117       44
             Files        65839        64471         1368      752
              4096        22181        22031          150       43
              2048         6218         6016          202      178
              1024         9402         9326           76       84
               512         1563          752          811       13
               256         6706         6200          506      439
               128        31905        30229         1676     1234
                64       203263       199233         4030     5528
                32       100623        98096         2527      750
                16       133108       129965         3143      511
           mt_zone          159            0          159      113
         DP fakepg            0            0            0        0
          PV ENTRY      1147925       955580       192345     9640
         MAP ENTRY        66307        61621         4686      522
        KMAP ENTRY       258716       258701           15      321
               MAP            7            0            7       16
         VM OBJECT        39420        29887         9533       99
        128 Bucket     19600430     19600247          183        5
         64 Bucket          126           62           64       20
         32 Bucket           74           42           32       24
         16 Bucket           79           31           48       52
          UMA Hash            6            2            4       26
     UMA RCntSlabs        12800            0        12800        2
         UMA Slabs       103933       102836         1097       24
         UMA Zones           86            0           86        2
          UMA Kegs           86            0           86        4
db> show intrcnt
irq4: sio0              764
irq13: npx0             1
irq14: ata0             33
irq18: atapci1          156381
irq26: em0              312271
cpu0: timer             1892870
cpu1: timer             1891948
db> show malloc
              Type       Allocs        Frees         Used
      nullfs_mount            9            0            9
       nullfs_hash            1            0            1
       nullfs_node        37816        37782           34
           rpcclnt            0            0            0
             idmap            0            0            0
          nfs4_dev            0            0            0
     nfsclient_req            2            2            0
   nfsclient_bigfh            0            0            0
  nfsclient_diroff            0            0            0
    nfsclient_hash            0            0            0
nfsclient_directio            0            0            0
 nfsclient_srvsock            0            0            0
    nfsclient_lock            0            0            0
 nfsclient_nlminfo            0            0            0
               gre            1            0            1
        madt_table            0            0            0
         acpicmbat            0            0            0
          pci_link           16            0           16
           acpisem           20            0           20
           acpipwr            0            0            0
           acpidev           57            0           57
          acpitask            1            1            0
            acpica        29883        27761         2122
         acpi_perf            0            0            0
        acd_driver            1            0            1
            devbuf          231            4          227
             cache            0            0            0
              $PIR            0            0            0
             lockf         2284         2271           13
     CAM dev queue            1            0            1
          nexusdev            3            0            3
           mptable            0            0            0
            linker          175          123           52
           memdesc            1            0            1
         legacydrv            0            0            0
         CAM queue            3            0            3
           io_apic            2            0            2
               LED            0            0            0
            KTRACE          100            0          100
            prison            5            0            5
           ata_pci            0            0            0
          atkbddev            2            0            2
           ithread           87            1           86
            zombie         1685         1685            0
         vm_pgdata            2            0            2
         proc-args         1185         1116           69
            kqueue          288          272           16
              kenv          110            1          109
             sigio            2            0            2
filedesc_to_leader            0            0            0
           UMAHash           13           11            2
         ufs_mount           15            0           15
         ufs_quota            0            0            0
       ufs_dirhash         2484          500         1984
          savedino         1093         1093            0
         newdirblk            0            0            0
            dirrem          559          559            0
             mkdir           74           74            0
            diradd         1257         1257            0
          freefile          472          472            0
          freeblks         1262         1262            0
          freefrag         7568         7568            0
        allocindir        91890        91888            2
          indirdep          184          183            1
       allocdirect         8317         8317            0
         bmsafemap          384          383            1
            newblk       100208       100207            1
          inodedep         2567         2566            1
           pagedep          489          488            1
          p1003.1b            1            0            1
      ip6_moptions            0            0            0
              mf6c            0            0            0
         in6_multi           40            0           40
          fragment            0            0            0
          syncache            1            0            1
          filedesc         1835         1697          138
         hostcache            1            0            1
       ip_moptions            0            0            0
 encap_export_host            2            1            1
          in_multi            2            0            2
              igmp            0            0            0
          routetbl          207          137           70
           ata_dma            4            0            4
         ad_driver            2            0            2
              cdev           22            0           22
                lo            1            0            1
            arpcom            2            0            2
             clone            3            0            3
       ether_multi           46            5           41
            ifaddr           59            0           59
             ifnet            6            0            6
               BPF           10            1            9
        CAM periph            1            0            1
  subr_export_host            0            0            0
       vnodemarker          266          266            0
             mount          306           37          269
            vnodes            3            0            3
          vfs_hash            1            0            1
       export_host            0            0            0
cluster_save_buffer         1506         1506            0
          vfscache            1            0            1
            biobuf          779          674          105
           CAM XPT            9            3            6
               pcb         6631         6109          522
            soname        33456        33413           43
          mbuf_tag        85082        85054           28
              accf            0            0            0
           CAM SIM            1            0            1
              ptys            0            0            0
              ptys            1            0            1
              ttys         1518         1342          176
               shm            1            0            1
               sem            4            0            4
               msg            4            0            4
               iov         9249         9249            0
            select            0            0            0
          ioctlops         2324         2324            0
            Unitno        25434        25426            8
        turnstiles          262            0          262
         taskqueue           13            0           13
             stack            0            0            0
       ata_generic           25           22            3
             pf_if           28            4           24
        sleepqueue          262            0          262
              sbuf          302          302            0
              rman          517          357          160
            isadev           20            0           20
             pflog            1            0            1
              GEOM          536          408          128
              kobj           76           10           66
             DEVFS           85            1           84
      eventhandler           50            2           48
           devstat           10            0           10
        DEVFS_RULE           65           31           34
            bus-sc         1011          967           44
               bus         2712         2167          545
              SWAP            2            0            2
              umtx          261            0          261
         sysctltmp          339          339            0
         sysctloid         2081            0         2081
            sysctl          290          290            0
            DEVFS1           93            0           93
           uidinfo           61           45           16
            plimit          326          305           21
            DEVFS3          720            1          719
              cred         7817         7684          133
           subproc         1965         1685          280
              proc            2            0            2
           session           68           20           48
              pgrp          100           48           52
            DEVFS2          550          457           93
          mtx_pool            1            0            1
            module          124            0          124
           entropy         1024            0         1024
              free            0            0            0
     CAM ccb queue            0            0            0
            ip6ndp            9            1            8
            ip6opt            1            0            1
              temp       134444       134433           11
db> show irqs
irq0: (no thread)
irq1: atkbd0 (pid 27)
irq3: sio1 (no thread)
irq4: sio0 (no thread)
irq5: (no thread)
irq6: (no thread)
irq7: (no thread)
irq8: (no thread)
irq9: acpi0 (pid 20)
irq10: (no thread)
irq11: (no thread)
irq12: (no thread)
irq13: npx0 (no thread)
irq14: ata0 (pid 23) {ENTROPY}
irq15: ata1 (pid 24) {ENTROPY}
irq16: (no thread)
irq17: (no thread)
irq18: atapci1 (pid 25) {ENTROPY}
irq19: (no thread)
irq20: (no thread)
irq21: (no thread)
irq22: (no thread)
irq23: (no thread)
irq24: (no thread)
irq25: (no thread)
irq26: em0 (no thread)
irq27: em1 (no thread)
irq28: (no thread)
irq29: (no thread)
irq30: (no thread)
irq31: (no thread)
irq32: (no thread)
irq33: (no thread)
irq34: (no thread)
irq35: (no thread)
irq36: (no thread)
irq37: (no thread)
irq38: (no thread)
irq39: (no thread)
irq40: (no thread)
irq41: (no thread)
irq42: (no thread)
irq43: (no thread)
irq44: (no thread)
irq45: (no thread)
irq46: (no thread)
irq47: (no thread)
db> show apic
vec 0x30 -> IRQ 13
vec 0x31 -> IRQ 9
vec 0x32 -> IRQ 26
vec 0x33 -> IRQ 27
vec 0x34 -> IRQ 14
vec 0x35 -> IRQ 15
vec 0x36 -> IRQ 18
vec 0x37 -> IRQ 4
vec 0x38 -> IRQ 3
vec 0x39 -> IRQ 1
vec 0xef -> lapic timer
db> show lockedbufs
buf at 0xda196cd8
b_flags = 0x20000000<vmio>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc670c9d8), b_data = 0xddf79000, b_blkno = 89960896
b_npages = 4, pages(OBJ, IDX, PA): (0xc0c5c2d0, 0xab9638, 0x68417000),(0xc0c5c2d0, 0xab9639, 0x6d058000),(0xc0c5c2d0, 0xab963a, 0x68579000),(0xc0c5c2d0, 0xab963b, 0x6961a000)

-- 
DSS5-RIPE DSS-RIPN 2:550/5068@fidonet 2:550/5069@fidonet
mailto:dsh@vlink.ru http://neva.vlink.ru/~dsh/



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?874q2qs0bn.fsf>