Date: Sun, 28 Apr 2013 00:02:56 -0400 From: Glen Barber <gjb@FreeBSD.org> To: Ian FREISLICH <ianf@clue.co.za> Cc: freebsd-current@freebsd.org Subject: Re: panic: in_pcblookup_local (?) Message-ID: <20130428040256.GK1611@glenbarber.us> In-Reply-To: <E1UW0K5-000P7H-36@clue.co.za> References: <E1UW0K5-000P7H-36@clue.co.za>
next in thread | previous in thread | raw e-mail | index | archive | help
--BFVE2HhgxTpCzM8t
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
On Sat, Apr 27, 2013 at 10:17:32AM +0200, Ian FREISLICH wrote:
> Hi
>=20
> I've been getting the following panic on recent current r249717.
> Sadly the crashdump is useless.
>=20
I just saw similar panic on 10-CURRENT r249588.
> Fatal trap 9: general protection fault while in kernel mode
> cpuid =3D 15; apic id =3D 0f
> instruction pointer =3D 0x20:0xffffffff80546fbc
> stack pointer =3D 0x28:0xffffff846b677770
> frame pointer =3D 0x28:0xffffff846b6777b0
> code segment =3D base 0x0, limit 0xfffff, type 0x1b
> =3D DPL 0, pres 1, long 1, def32 0, gran 1
> processor eflags =3D interrupt enabled, resume, IOPL =3D 0
> current process =3D 4361 (zabbix_agentd)
Hmm.. This is interests me. In my case, cf-agent was the current
process.
Backtrace of my panic follows. Any pointers on how to debug this
further would be appreciated.
Glen
Script started on Sat Apr 27 23:53:53 2013
root@orion:/usr/obj/usr/src/sys/ORION # kgdb ./kernel.debug /var/crash/vmco=
re.4
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
Fatal trap 9: general protection fault while in kernel mode
cpuid =3D 1; apic id =3D 01
instruction pointer =3D 0x20:0xffffffff80736cec
stack pointer =3D 0x28:0xffffff81aad4e760
frame pointer =3D 0x28:0xffffff81aad4e7a0
code segment =3D base 0x0, limit 0xfffff, type 0x1b
=3D DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags =3D interrupt enabled, resume, IOPL =3D 0
current process =3D 78664 (cf-agent)
trap number =3D 9
panic: general protection fault
cpuid =3D 1
KDB: stack backtrace:
#0 0xffffffff80642a56 at kdb_backtrace+0x66
#1 0xffffffff80606eeb at panic+0x13b
#2 0xffffffff808e3b10 at trap_fatal+0x290
#3 0xffffffff808e4331 at trap+0x241
#4 0xffffffff808cdbb3 at calltrap+0x8
#5 0xffffffff807371d8 at in_pcb_lport+0x128
#6 0xffffffff8073745a at in_pcbbind_setup+0x16a
#7 0xffffffff80737d8e at in_pcbconnect_setup+0x71e
#8 0xffffffff80737df9 at in_pcbconnect_mbuf+0x59
#9 0xffffffff807bf29f at udp_connect+0x11f
#10 0xffffffff80680615 at kern_connectat+0x275
#11 0xffffffff80680731 at sys_connect+0x41
#12 0xffffffff808e32cb at amd64_syscall+0x63b
#13 0xffffffff808cde97 at Xfast_syscall+0xf7
Uptime: 3d19h38m52s
(ada0:ahcich0:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada0:ahcich0:0:0:0): CAM status: CCB request is in progress
(ada0:ahcich0:0:0:0): Error 5, Retries exhausted
(ada0:ahcich0:0:0:0): Synchronize cache failed
(ada1:ahcich1:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada1:ahcich1:0:0:0): CAM status: CCB request is in progress
(ada1:ahcich1:0:0:0): Error 5, Retries exhausted
(ada1:ahcich1:0:0:0): Synchronize cache failed
(ada2:ahcich4:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada2:ahcich4:0:0:0): CAM status: CCB request is in progress
(ada2:ahcich4:0:0:0): Error 5, Retries exhausted
(ada2:ahcich4:0:0:0): Synchronize cache failed
(ada3:ahcich5:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada3:ahcich5:0:0:0): CAM status: CCB request is in progress
(ada3:ahcich5:0:0:0): Error 5, Retries exhausted
(ada3:ahcich5:0:0:0): Synchronize cache failed
Dumping 1014 out of 6049 MB:..2%..12%..21%..32%..42%..51%..62%..71%..81%..9=
2%
Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:231
231 __asm("movq %%gs:%1,%0" : "=3Dr" (td)
(kgdb) bt
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:231
#1 0xffffffff80606a56 in kern_reboot (howto=3D260) at /usr/src/sys/kern/ke=
rn_shutdown.c:447
#2 0xffffffff80606ed5 in panic (fmt=3D<value optimized out>)
at /usr/src/sys/kern/kern_shutdown.c:754
#3 0xffffffff808e3b10 in trap_fatal (frame=3D0x9, eva=3D<value optimized o=
ut>)
at /usr/src/sys/amd64/amd64/trap.c:872
#4 0xffffffff808e4331 in trap (frame=3D0xffffff81aad4e6b0)
at /usr/src/sys/amd64/amd64/trap.c:605
#5 0xffffffff808cdbb3 in calltrap () at /usr/src/sys/amd64/amd64/exception=
=2ES:228
#6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=3D0xffffffff80dc9180,=
laddr=3D
{s_addr =3D 50374848}, lport=3D339, lookupflags=3D1, cred=3D0xfffffe0=
16cdad100)
at /usr/src/sys/netinet/in_pcb.c:1438
#7 0xffffffff807371d8 in in_pcb_lport (inp=3D0xfffffe016c2fb7a8, laddrp=3D=
0xffffff81aad4e860,=20
lportp=3D0xffffff81aad4e86e, cred=3D0xfffffe016cdad100, lookupflags=3D1)
at /usr/src/sys/netinet/in_pcb.c:457
#8 0xffffffff8073745a in in_pcbbind_setup (inp=3D0xfffffe016c2fb7a8, nam=
=3D0x0,=20
laddrp=3D0xffffff81aad4e900, lportp=3D0xffffff81aad4e90e, cred=3D0xffff=
fe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:615
#9 0xffffffff80737d8e in in_pcbconnect_setup (inp=3D0xfffffe016c2fb7a8,=20
nam=3D<value optimized out>, laddrp=3D0xffffff81aad4e9b8, lportp=3D0xff=
ffff81aad4e9be,=20
faddrp=3D0xffffff81aad4e9b4, fportp=3D0xffffff81aad4e9bc, oinpp=3D0x0, =
cred=3D0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:1019
#10 0xffffffff80737df9 in in_pcbconnect_mbuf (inp=3D0xfffffe016c2fb7a8,=20
nam=3D<value optimized out>, cred=3D<value optimized out>, m=3D0x0)
at /usr/src/sys/netinet/in_pcb.c:645
#11 0xffffffff807bf29f in udp_connect (so=3D0xfffffe016ce4e2a8, nam=3D0xfff=
ffe0017c43740,=20
td=3D0xfffffe00ae986000) at /usr/src/sys/netinet/udp_usrreq.c:1530
#12 0xffffffff80680615 in kern_connectat (td=3D0xfffffe00ae986000, dirfd=3D=
-100,=20
fd=3D<value optimized out>, sa=3D0xfffffe0017c43740) at /usr/src/sys/ke=
rn/uipc_syscalls.c:596
#13 0xffffffff80680731 in sys_connect (td=3D0xfffffe00ae986000, uap=3D0xfff=
fff81aad4eb70)
at /usr/src/sys/kern/uipc_syscalls.c:562
#14 0xffffffff808e32cb in amd64_syscall (td=3D0xfffffe00ae986000, traced=3D=
0)
at subr_syscall.c:134
#15 0xffffffff808cde97 in Xfast_syscall () at /usr/src/sys/amd64/amd64/exce=
ption.S:387
#16 0x0000000801b3d8da in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) frame 6
#6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=3D0xffffffff80dc9180,=
laddr=3D
{s_addr =3D 50374848}, lport=3D339, lookupflags=3D1, cred=3D0xfffffe0=
16cdad100)
at /usr/src/sys/netinet/in_pcb.c:1438
1438 LIST_FOREACH(phd, porthash, phd_hash) {
(kgdb) list *0xffffffff80736cec
0xffffffff80736cec is in in_pcblookup_local (/usr/src/sys/netinet/in_pcb.c:=
1439).
1434 * port hash list.
1435 */
1436 porthash =3D &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1437 pcbinfo->ipi_porthashmask)];
1438 LIST_FOREACH(phd, porthash, phd_hash) {
1439 if (phd->phd_port =3D=3D lport)
1440 break;
1441 }
1442 if (phd !=3D NULL) {
1443 /*
(kgdb) p *pcbinfo
$1 =3D {ipi_lock =3D {lock_object =3D {lo_name =3D 0xffffffff809d4d82 "udp"=
, lo_flags =3D 69926912,=20
lo_data =3D 0, lo_witness =3D 0x0}, rw_lock =3D 1}, ipi_listhead =3D =
0xffffffff80dc9108,=20
ipi_count =3D 28, ipi_gencnt =3D 535501, ipi_lastport =3D 21249, ipi_last=
low =3D 0,=20
ipi_lasthi =3D 0, ipi_zone =3D 0xfffffe0017b60380, ipi_pcbgroups =3D 0x0,=
ipi_npcbgroups =3D 0,=20
ipi_hashfields =3D 0, ipi_hash_lock =3D {lock_object =3D {
lo_name =3D 0xffffffff80a03d80 "pcbinfohash", lo_flags =3D 69402624, =
lo_data =3D 0,=20
lo_witness =3D 0x0}, rw_lock =3D 18446741877615517696}, ipi_hashbase =
=3D 0xfffffe00120f6000,=20
ipi_hashmask =3D 127, ipi_porthashbase =3D 0xfffffe00120f5c04, ipi_portha=
shmask =3D 127,=20
ipi_wildbase =3D 0x0, ipi_wildmask =3D 0, ipi_vnet =3D 0x0, ipi_pspare =
=3D {0x0, 0x0}}
(kgdb) up
#7 0xffffffff807371d8 in in_pcb_lport (inp=3D0xfffffe016c2fb7a8, laddrp=3D=
0xffffff81aad4e860,=20
lportp=3D0xffffff81aad4e86e, cred=3D0xfffffe016cdad100, lookupflags=3D1)
at /usr/src/sys/netinet/in_pcb.c:457
457 tmpinp =3D in_pcblookup_local(pcbinfo, laddr,
(kgdb) list *0xffffffff807371d8
0xffffffff807371d8 is in in_pcb_lport (/usr/src/sys/netinet/in_pcb.c:457).
452 #endif
453 #if defined(INET) && defined(INET6)
454 else
455 #endif
456 #ifdef INET
457 tmpinp =3D in_pcblookup_local(pcbinfo, laddr,
458 lport, lookupflags, cred);
459 #endif
460 } while (tmpinp !=3D NULL);
461=09
(kgdb) up
#8 0xffffffff8073745a in in_pcbbind_setup (inp=3D0xfffffe016c2fb7a8, nam=
=3D0x0,=20
laddrp=3D0xffffff81aad4e900, lportp=3D0xffffff81aad4e90e, cred=3D0xffff=
fe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:615
615 error =3D in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
(kgdb) list *0xffffffff8073745a
0xffffffff8073745a is in in_pcbbind_setup (/usr/src/sys/netinet/in_pcb.c:61=
6).
611 }
612 if (*lportp !=3D 0)
613 lport =3D *lportp;
614 if (lport =3D=3D 0) {
615 error =3D in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
616 if (error !=3D 0)
617 return (error);
618=09
619 }
620 *laddrp =3D laddr.s_addr;
(kgdb) p nam
$2 =3D (struct sockaddr *) 0x0
(kgdb) root@orion:/usr/obj/usr/src/sys/ORION # ^D
Script done on Sat Apr 27 23:55:22 2013
--BFVE2HhgxTpCzM8t
Content-Type: application/pgp-signature
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.19 (FreeBSD)
iQEcBAEBCAAGBQJRfJ9wAAoJEFJPDDeguUajmG0IAJb6zfujMoxbu9AiYUI+XXjF
2TZtKhv2z0iBZP3o5N+kGge/xz4I21QeJyOpzNMZqftHunxgZrtfaKS5dJiXpt5E
CnDo9/8yze2u8JfmamAZUTRWMKwI/3PcPiYWRUDo2JKfJo8+ECh/qrcPPk2yBUFI
+DQSgfBcs63WTrP/fx+lGO12r18W7s+fSbM4XK88RFf0GLfpOq52XmVQFlr5uH76
mc2pzzCdcabtHRgiQ5ytX4qoaEluMDQQMZsMBjrGUtf41KI/kjYi2rI8W3zTV+eO
5hYlfQHb5/wWZnLMaZvJtuZyGtpJwbjGWad+CNOl260r5zlmGY/yuGKwwVbR364=
=lv8y
-----END PGP SIGNATURE-----
--BFVE2HhgxTpCzM8t--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20130428040256.GK1611>