Date: Tue, 7 Dec 2010 17:13:06 GMT From: Artem Kim <artem_kim@inbox.ru> To: freebsd-gnats-submit@FreeBSD.org Subject: misc/152893: 8.2-PRERELEASE panic in NETGRAP Message-ID: <201012071713.oB7HD6PG075589@red.freebsd.org> Resent-Message-ID: <201012071720.oB7HK7V2084512@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 152893
>Category: misc
>Synopsis: 8.2-PRERELEASE panic in NETGRAP
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Tue Dec 07 17:20:06 UTC 2010
>Closed-Date:
>Last-Modified:
>Originator: Artem Kim
>Release: 8.2-PRERELEASE
>Organization:
>Environment:
FreeBSD nas4 8.2-PRERELEASE FreeBSD 8.2-PRERELEASE #0: Thu Dec 2 19:07:46 MSK 2010 xxx@nas4:/usr/obj/usr/src/sys/router i386
>Description:
I have problem on one of my pppoe routers with 8.2-PRERELEASE.
panic occur at intervals ranging from several hours to several days.
A similar problem: kern/137881
environment:
pppoe router,
CPU Xeon X5472,
network adapter 82575EB,
mpd5,
FreeBSD nas4 8.2-PRERELEASE FreeBSD 8.2-PRERELEASE #0: Thu Dec 2 19:07:46 MSK 2010 xxx@nas4:/usr/obj/usr/src/sys/router i386
extra kernel config:
options KVA_PAGES=512
sysctl:
kern.ipc.maxsockbuf=524288
kern.ipc.nmbclusters=65535
net.graph.recvspace=40960
net.graph.maxdgram=40960
net.graph.maxdata=1024
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
vm.kmem_size=1512M
vm.kmem_size_max=1512M
Values of members callout structure's seems strange:
$10 = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40, c_flags = 13, c_cpu = 64}
c_arg = c_lock = 0x40
This value is close to the fault virtual address - 0x44.
Fatal trap 12: page fault while in kernel mode
cpuid = 3; apic id = 03
fault virtual address = 0x44
fault code = supervisor read, page not present
instruction pointer = 0x20:0x805dfb56
stack pointer = 0x28:0xfbbab944
frame pointer = 0x28:0xfbbab970
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 1944 (mpd5)
trap number = 12
panic: page fault
cpuid = 2
KDB: stack backtrace:
#0 0x805fce6d at kdb_backtrace+0x48
#1 0x805cdb9c at panic+0x108
#2 0x8079bbd2 at trap_fatal+0x24c
#3 0x8079bf8e at trap_pfault+0x270
#4 0x8079c3db at trap+0x371
#5 0x807842dc at calltrap+0x6
#6 0x8068c2ae at ng_uncallout+0x1b
#7 0x8069c454 at ng_pppoe_disconnect+0xf8
#8 0x8068d5cc at ng_destroy_hook+0xe0
#9 0x8068e5e9 at ng_apply_item+0x903
#10 0x8068cea7 at ng_snd_item+0x2e9
#11 0x806a04f8 at ngc_send+0x1d3
#12 0x8062e01a at sosend_generic+0x2aa
#13 0x80631df0 at kern_sendit+0xfc
#14 0x8063203f at sendit+0xcd
#15 0x80632122 at sendto+0x48
#16 0x80608641 at syscallenter+0x28d
#17 0x8079bfef at syscall+0x2e
Uptime: 7h53m5s
Physical memory: 2038 MB
Dumping 253 MB: 238 222 206 190 174 158 142 126 110 94 78 62 46 30 14
#0 doadump () at pcpu.h:231
231 __asm("movl %%fs:0,%0" : "=r" (td));
(kgdb) f 6
#6 0x807842dc in calltrap () at /usr/src/sys/i386/i386/exception.s:166
166 call trap
Current language: auto; currently asm
(kgdb) up
#7 0x805dfb56 in _callout_stop_safe (c=0x8c29f008, safe=0) at /usr/src/sys/kern/kern_timeout.c:683
683 if (c->c_lock == &Giant.lock_object)
Current language: auto; currently c
(kgdb) l
678 /*
679 * Some old subsystems don't hold Giant while running a callout_stop(),
680 * so just discard this check for the moment.
681 */
682 if (!safe && c->c_lock != NULL) {
683 if (c->c_lock == &Giant.lock_object)
684 use_lock = mtx_owned(&Giant);
685 else {
686 use_lock = 1;
687 class = LOCK_CLASS(c->c_lock);
(kgdb) p c
$1 = (struct callout *) 0x8c29f008
(kgdb) p c->c_lock
$2 = (struct lock_object *) 0x40
(kgdb) p *c
$3 = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40, c_flags = 13, c_cpu = 64}
(kgdb) p *c->c_lock
Cannot access memory at address 0x40
(kgdb) up
#8 0x8068c2ae in ng_uncallout (c=0x8c29f008, node=0x874abb00) at /usr/src/sys/netgraph/ng_base.c:3732
3732 rval = callout_stop(c);
(kgdb) l
3727 int rval;
3728
3729 KASSERT(c != NULL, ("ng_uncallout: NULL callout"));
3730 KASSERT(node != NULL, ("ng_uncallout: NULL node"));
3731
3732 rval = callout_stop(c);
3733 item = c->c_arg;
3734 /* Do an extra check */
3735 if ((rval > 0) && (c->c_func == &ng_callout_trampoline) &&
3736 (NGI_NODE(item) == node)) {
(kgdb) p c
$4 = (struct callout *) 0x8c29f008
(kgdb) p *c
$5 = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40, c_flags = 13, c_cpu = 64}
(kgdb)
$6 = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40, c_flags = 13, c_cpu = 64}
(kgdb) up
#9 0x8069c454 in ng_pppoe_disconnect (hook=0x88bcb880) at /usr/src/sys/netgraph/ng_pppoe.c:1791
1791 ng_uncallout(&sp->neg->handle, node);
(kgdb) l
1786 /*
1787 * As long as we have somewhere to store the timeout handle,
1788 * we may have a timeout pending.. get rid of it.
1789 */
1790 if (sp->neg) {
1791 ng_uncallout(&sp->neg->handle, node);
1792 if (sp->neg->m)
1793 m_freem(sp->neg->m);
1794 free(sp->neg, M_NETGRAPH_PPPOE);
1795 }
(kgdb) p &sp
$7 = (sessp *) 0xfbbab9b0
(kgdb) p sp
$8 = 0x89391180
(kgdb) p &sp->neg->handle
$9 = (struct callout *) 0x8c29f008
(kgdb) p sp->neg->handle
$10 = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40, c_flags = 13, c_cpu = 64}
(kgdb) p sp->neg
$11 = 0x8c29f000
(kgdb) p *sp->neg
$12 = {m = 0x7, pkt = 0x40, handle = {c_links = {sle = {sle_next = 0x9}, tqe = {tqe_next = 0x9, tqe_prev = 0x40}}, c_time = 10, c_arg = 0x40, c_func = 0xc, c_lock = 0x40,
c_flags = 13, c_cpu = 64}, timeout = 39, numtags = 64, tags = {0x28, 0x40, 0x29, 0x40, 0x2a, 0x40, 0x2b, 0x40, 0x2c, 0x40, 0x24, 0x40, 0x23, 0x40, 0x18, 0x40, 0x1c, 0x40,
0x1b, 0x40}, service_len = 23, ac_name_len = 64, service = {hdr = {tag_type = 26, tag_len = 0},
data = "@\000\000\000\031\000\000\000@\000\000\000\"\000\000\000@\000\000\000\036\000\000\000@\000\000\000!\000\000\000@\000\000\000\026\000\000\000@\000\000\000\023\000\000\000@\000\000\000\035\000\000\000@\000\000\000%\000\000"}, ac_name = {hdr = {tag_type = 64, tag_len = 0},
data = " \000\000\000@\000\000\000\037\000\000\000@\000\000\000&\000\000\000@\000\000\000\024\000\000\000@\000\000\000\021\000\000\000@\000\000\000\025\000\000\000@\000\000\000\006\000\000\000@\000\000\000\000\000\000\000\000\000\000"}}
(kgdb) p sp
$13 = 0x89391180
(kgdb) p *sp
$14 = {hook = 0x88bcb880, Session_ID = 0, state = PPPOE_SOFFER, creator = 47, pkt_hdr = {eh = {ether_dhost = "\000\000\000\000\000", ether_shost = "\000\000\000\000\000",
ether_type = 0}, ph = {ver = 0 '\0', type = 0 '\0', code = 0 '\0', sid = 0, length = 0}}, neg = 0x8c29f000, sessions = {le_next = 0x0, le_prev = 0x0}}
(kgdb)
>How-To-Repeat:
Highly loaded router with pppoe mpd5.
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201012071713.oB7HD6PG075589>