Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 25 Jun 2008 14:09:37 +0000
From:      Chameeya Software Services Ltd. <chameeyass@hotmail.com>
To:        <remko@elvandar.org>
Cc:        freebsd-bugs@freebsd.org
Subject:   RE: kern/124670: large file operation on RAID cause many GEOM errors - crash
Message-ID:  <BLU121-W756225FD7CFB5FEDDF198D0A00@phx.gbl>
In-Reply-To: <fcafb04e5d6e2e6845adfd3dfda51840.squirrel@galain.elvandar.org>
References:  <200806171930.m5HJU45s014686@freefall.freebsd.org> <BLU121-W29537AD2A92063D65E7830D0AA0@phx.gbl>  <fcafb04e5d6e2e6845adfd3dfda51840.squirrel@galain.elvandar.org>

next in thread | previous in thread | raw e-mail | index | archive | help

I've got two more cores today which I tried to debug..though I'm not gettin=
g a lot of information from
them. This all seems a bit random..as if there are bad blocks on the hard-d=
rive and its just
randome whether the file hit them or not. but that's just speculation..coul=
d also be a software/hardware timing incompatibility.

[<code>]
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:=
 Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you ar=
e
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".

Unread portion of the kernel message buffer:
panic: vinvalbuf: dirty bufs
cpuid =3D 0
Uptime: 26m36s
Physical memory: 627 MB
Dumping 114 MB: 99 83 67 51 35 19 3

#0  doadump () at pcpu.h:195
195             __asm __volatile("movl %%fs:0,%0" : "=3Dr" (td));
(kgdb) backtrace
#0  doadump () at pcpu.h:195
#1  0xc0754457 in boot (howto=3D260) at /usr/src/sys/kern/kern_shutdown.c:4=
09
#2  0xc0754719 in panic (fmt=3DVariable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:563
#3  0xc07cb2c6 in bufobj_invalbuf (bo=3D0xc370cc70, flags=3D1, td=3D0xc3c3e=
210,=20
    slpflag=3D0, slptimeo=3D0) at /usr/src/sys/kern/vfs_subr.c:1075
#4  0xc07cb5f2 in vinvalbuf (vp=3D0xc370cbb0, flags=3D1, td=3D0xc3c3e210, s=
lpflag=3D0,=20
    slptimeo=3D0) at /usr/src/sys/kern/vfs_subr.c:1143
#5  0xc07cb6bf in vgonel (vp=3D0xc370cbb0) at /usr/src/sys/kern/vfs_subr.c:=
2514
#6  0xc07cb9a7 in vgone (vp=3D0xc370cbb0) at /usr/src/sys/kern/vfs_subr.c:2=
472
#7  0xc06e7c05 in devfs_delete (dm=3D0xc36771c0, de=3D0xc3672000, vp_locked=
=3D0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:265
#8  0xc06e80cd in devfs_populate_loop (dm=3D0xc36771c0, cleanup=3D0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:384
#9  0xc06e840e in devfs_populate (dm=3D0xc36771c0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:485
#10 0xc06ebe6e in devfs_lookup (ap=3D0xdaae19b8)
    at /usr/src/sys/fs/devfs/devfs_vnops.c:601
#11 0xc0a5f256 in VOP_LOOKUP_APV (vop=3D0xc0b71680, a=3D0xdaae19b8)
    at vnode_if.c:99
#12 0xc07c28f1 in lookup (ndp=3D0xdaae1b80) at vnode_if.h:57
#13 0xc07c35ff in namei (ndp=3D0xdaae1b80) at /usr/src/sys/kern/vfs_lookup.=
c:219
#14 0xc07d9fd7 in vn_open_cred (ndp=3D0xdaae1b80, flagp=3D0xdaae1c78, cmode=
=3D0,=20
    cred=3D0xc39ba200, fp=3D0xc3b3f990) at /usr/src/sys/kern/vfs_vnops.c:18=
8
---Type <return> to continue, or q <return> to quit---
#15 0xc07da2a3 in vn_open (ndp=3D0xdaae1b80, flagp=3D0xdaae1c78, cmode=3D0,=
=20
    fp=3D0xc3b3f990) at /usr/src/sys/kern/vfs_vnops.c:94
#16 0xc07d7f27 in kern_open (td=3D0xc3c3e210,=20
    path=3D0x2887e109 <Address 0x2887e109 out of bounds>, pathseg=3DUIO_USE=
RSPACE,=20
    flags=3D1, mode=3D0) at /usr/src/sys/kern/vfs_syscalls.c:1028
#17 0xc07d8490 in open (td=3D0xc3c3e210, uap=3D0xdaae1cfc)
    at /usr/src/sys/kern/vfs_syscalls.c:995
#18 0xc0a49635 in syscall (frame=3D0xdaae1d38)
    at /usr/src/sys/i386/i386/trap.c:1035
#19 0xc0a2fc70 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s=
:196
#20 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
[<CODE>]

and from the other core dump
[<CODE>]
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:=
 Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you ar=
e
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".

Unread portion of the kernel message buffer:
panic: vinvalbuf: dirty bufs
cpuid =3D 0
Uptime: 3h47m18s
Physical memory: 627 MB
Dumping 111 MB: 96 80 64 48 32 16

#0  doadump () at pcpu.h:195
195             __asm __volatile("movl %%fs:0,%0" : "=3Dr" (td));
(kgdb) backtrace
#0  doadump () at pcpu.h:195
#1  0xc0754457 in boot (howto=3D260) at /usr/src/sys/kern/kern_shutdown.c:4=
09
#2  0xc0754719 in panic (fmt=3DVariable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:563
#3  0xc07cb2c6 in bufobj_invalbuf (bo=3D0xc370cc70, flags=3D1, td=3D0xc39b7=
000,=20
    slpflag=3D0, slptimeo=3D0) at /usr/src/sys/kern/vfs_subr.c:1075
#4  0xc07cb5f2 in vinvalbuf (vp=3D0xc370cbb0, flags=3D1, td=3D0xc39b7000, s=
lpflag=3D0,=20
    slptimeo=3D0) at /usr/src/sys/kern/vfs_subr.c:1143
#5  0xc07cb6bf in vgonel (vp=3D0xc370cbb0) at /usr/src/sys/kern/vfs_subr.c:=
2514
#6  0xc07cb9a7 in vgone (vp=3D0xc370cbb0) at /usr/src/sys/kern/vfs_subr.c:2=
472
#7  0xc06e7c05 in devfs_delete (dm=3D0xc36771c0, de=3D0xc3672000, vp_locked=
=3D0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:265
#8  0xc06e80cd in devfs_populate_loop (dm=3D0xc36771c0, cleanup=3D0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:384
#9  0xc06e840e in devfs_populate (dm=3D0xc36771c0)
    at /usr/src/sys/fs/devfs/devfs_devs.c:485
#10 0xc06ebe6e in devfs_lookup (ap=3D0xdaa809b8)
    at /usr/src/sys/fs/devfs/devfs_vnops.c:601
#11 0xc0a5f256 in VOP_LOOKUP_APV (vop=3D0xc0b71680, a=3D0xdaa809b8)
    at vnode_if.c:99
#12 0xc07c28f1 in lookup (ndp=3D0xdaa80b80) at vnode_if.h:57
#13 0xc07c35ff in namei (ndp=3D0xdaa80b80) at /usr/src/sys/kern/vfs_lookup.=
c:219
#14 0xc07d9d32 in vn_open_cred (ndp=3D0xdaa80b80, flagp=3D0xdaa80c78, cmode=
=3D420,=20
    cred=3D0xc39b9a00, fp=3D0xc3936798) at /usr/src/sys/kern/vfs_vnops.c:13=
0
#15 0xc07da2a3 in vn_open (ndp=3D0xdaa80b80, flagp=3D0xdaa80c78, cmode=3D42=
0,=20
    fp=3D0xc3936798) at /usr/src/sys/kern/vfs_vnops.c:94
#16 0xc07d7f27 in kern_open (td=3D0xc39b7000,=20
    path=3D0x28202480 <Address 0x28202480 out of bounds>, pathseg=3DUIO_USE=
RSPACE,=20
    flags=3D1538, mode=3D438) at /usr/src/sys/kern/vfs_syscalls.c:1028
#17 0xc07d8490 in open (td=3D0xc39b7000, uap=3D0xdaa80cfc)
    at /usr/src/sys/kern/vfs_syscalls.c:995
#18 0xc0a49635 in syscall (frame=3D0xdaa80d38)
    at /usr/src/sys/i386/i386/trap.c:1035
---Type <return> to continue, or q <return> to quit---
#19 0xc0a2fc70 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s=
:196
#20 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb)=20

[</CODE>]

Note that I was running gmirror at this time. So this has nothing to do wit=
h gmirror.

I thought perhaps there was a specific problem with a specific folder. But =
after several tests
It seems quite random. I could not reproduce it consistently. yikes.

Salik.


_________________________________________________________________

http://clk.atdmt.com/UKM/go/msnnkmgl0010000009ukm/direct/01/=



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?BLU121-W756225FD7CFB5FEDDF198D0A00>