Date: Sun, 9 Mar 2014 12:56:48 -0400 From: Glen Barber <gjb@FreeBSD.org> To: freebsd-current@FreeBSD.org Subject: panic: vm_fault: fault on nofault entry Message-ID: <20140309165648.GF1776@glenbarber.us>
next in thread | raw e-mail | index | archive | help
--s5/bjXLgkIwAv6Hi
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
We are having regular panics on several machines in the cluster.
Below follows the script from the kgdb(1) session, hopefully providing
enough information. This machine runs 11.0-CURRENT #2 r262892, from
2 days ago.
It uses tmpfs(5) for the port build workspace. I have an unconfirmed
suspicion that use of sysutils/lsof is involved somehow, but cannot be
sure. (In my experience with panics with port building, removing lsof
=66rom the system did have an effect, but I may be going down the wrong
rabbit hole.)
Script started on Sun Mar 9 16:40:07 2014
root@redbuild01.nyi:/usr/obj/usr/src/sys/REDBUILD # sh
# kgdb ./kernel.debug /var/crash/vmcore.1
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
panic: vm_fault: fault on nofault entry, addr: fffffe035021a000
cpuid =3D 1
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe1839a54=
180
kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe1839a54230
panic() at panic+0x155/frame 0xfffffe1839a542b0
vm_fault_hold() at vm_fault_hold+0x1e7a/frame 0xfffffe1839a54500
vm_fault() at vm_fault+0x77/frame 0xfffffe1839a54540
trap_pfault() at trap_pfault+0x199/frame 0xfffffe1839a545e0
trap() at trap+0x4a0/frame 0xfffffe1839a547f0
calltrap() at calltrap+0x8/frame 0xfffffe1839a547f0
--- trap 0xc, rip =3D 0xffffffff80d97bab, rsp =3D 0xfffffe1839a548b0, rbp =
=3D 0xfffffe1839a54910 ---
copyout() at copyout+0x3b/frame 0xfffffe1839a54910
memrw() at memrw+0x19f/frame 0xfffffe1839a54950
giant_read() at giant_read+0xa4/frame 0xfffffe1839a54990
devfs_read_f() at devfs_read_f+0xeb/frame 0xfffffe1839a549f0
dofileread() at dofileread+0x95/frame 0xfffffe1839a54a40
kern_readv() at kern_readv+0x68/frame 0xfffffe1839a54a90
sys_read() at sys_read+0x63/frame 0xfffffe1839a54ae0
amd64_syscall() at amd64_syscall+0x3fb/frame 0xfffffe1839a54bf0
Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe1839a54bf0
--- syscall (3, FreeBSD ELF64, sys_read), rip =3D 0x800b8444a, rsp =3D 0x7f=
ffffffd088, rbp =3D 0x7fffffffd0d0 ---
KDB: enter: panic
Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
Reading symbols from /boot/kernel/ums.ko.symbols...done.
Loaded symbols for /boot/kernel/ums.ko.symbols
Reading symbols from /boot/kernel/tmpfs.ko.symbols...done.
Loaded symbols for /boot/kernel/tmpfs.ko.symbols
Reading symbols from /boot/kernel/nullfs.ko.symbols...done.
Loaded symbols for /boot/kernel/nullfs.ko.symbols
Reading symbols from /boot/kernel/linprocfs.ko.symbols...done.
Loaded symbols for /boot/kernel/linprocfs.ko.symbols
Reading symbols from /boot/kernel/linux.ko.symbols...done.
Loaded symbols for /boot/kernel/linux.ko.symbols
#0 doadump (textdump=3D-967130448) at pcpu.h:219
219 __asm("movq %%gs:%1,%0" : "=3Dr" (td)
(kgdb) bt
#0 doadump (textdump=3D-967130448) at pcpu.h:219
#1 0xffffffff8034a1a5 in db_fncall (dummy1=3D<value optimized out>,=20
dummy2=3D<value optimized out>, dummy3=3D<value optimized out>, dummy4=
=3D<value optimized out>)
at /usr/src/sys/ddb/db_command.c:578
#2 0xffffffff80349e8d in db_command (cmd_table=3D0x0) at /usr/src/sys/ddb/=
db_command.c:449
#3 0xffffffff80349c04 in db_command_loop () at /usr/src/sys/ddb/db_command=
=2Ec:502
#4 0xffffffff8034c660 in db_trap (type=3D<value optimized out>, code=3D0)
at /usr/src/sys/ddb/db_main.c:231
#5 0xffffffff80987ae9 in kdb_trap (type=3D3, code=3D0, tf=3D<value optimiz=
ed out>)
at /usr/src/sys/kern/subr_kdb.c:656
#6 0xffffffff80d999b9 in trap (frame=3D0xfffffe1839a54160)
at /usr/src/sys/amd64/amd64/trap.c:571
#7 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception=
=2ES:231
#8 0xffffffff8098724e in kdb_enter (why=3D0xffffffff8100f4ba "panic", msg=
=3D<value optimized out>)
at cpufunc.h:63
#9 0xffffffff80946a75 in panic (fmt=3D<value optimized out>)
at /usr/src/sys/kern/kern_shutdown.c:752
#10 0xffffffff80c0a1fa in vm_fault_hold (map=3D<value optimized out>,=20
vaddr=3D<value optimized out>, fault_type=3D<value optimized out>,=20
fault_flags=3D<value optimized out>, m_hold=3D<value optimized out>)
at /usr/src/sys/vm/vm_fault.c:272
#11 0xffffffff80c08337 in vm_fault (map=3D0xfffff80002000000, vaddr=3D<valu=
e optimized out>,=20
fault_type=3D1 '\001', fault_flags=3D128) at /usr/src/sys/vm/vm_fault.c=
:217
#12 0xffffffff80d9a1a9 in trap_pfault (frame=3D0xfffffe1839a54800, usermode=
=3D0)
at /usr/src/sys/amd64/amd64/trap.c:767
#13 0xffffffff80d999d0 in trap (frame=3D0xfffffe1839a54800)
at /usr/src/sys/amd64/amd64/trap.c:455
#14 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception=
=2ES:231
#15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:=
246
#16 0xffffffff8099c2f5 in uiomove_faultflag (cp=3D<value optimized out>,=20
n=3D<value optimized out>, uio=3D0xfffffe1839a54ab0, nofault=3D<value o=
ptimized out>)
at /usr/src/sys/kern/subr_uio.c:192
#17 0xffffffff80d8612f in memrw (dev=3D0xfffff8000dbd0400, uio=3D0xfffffe18=
39a54ab0,=20
flags=3D113246208) at /usr/src/sys/amd64/amd64/mem.c:101
#18 0xffffffff808ecf04 in giant_read (dev=3D0xfffff8000dbd0400, uio=3D0xfff=
ffe1839a54ab0, ioflag=3D0)
at /usr/src/sys/kern/kern_conf.c:442
#19 0xffffffff808185cb in devfs_read_f (fp=3D0xfffff80083439230, uio=3D0xff=
fffe1839a54ab0,=20
cred=3D<value optimized out>, flags=3D0, td=3D0xfffff80e4edb8490)
at /usr/src/sys/fs/devfs/devfs_vnops.c:1193
#20 0xffffffff809a15e5 in dofileread (td=3D0xfffff80e4edb8490, fd=3D4, fp=
=3D0xfffff80083439230,=20
auio=3D0xfffffe1839a54ab0, offset=3D<value optimized out>, flags=3D1172=
307968) at file.h:299
#21 0xffffffff809a1308 in kern_readv (td=3D0xfffff80e4edb8490, fd=3D4, auio=
=3D0xfffffe1839a54ab0)
at /usr/src/sys/kern/sys_generic.c:256
#22 0xffffffff809a1293 in sys_read (td=3D<value optimized out>, uap=3D<valu=
e optimized out>)
at /usr/src/sys/kern/sys_generic.c:171
#23 0xffffffff80d9a9fb in amd64_syscall (td=3D0xfffff80e4edb8490, traced=3D=
0) at subr_syscall.c:133
#24 0xffffffff80d7e9cb in Xfast_syscall () at /usr/src/sys/amd64/amd64/exce=
ption.S:390
#25 0x0000000800b8444a in ?? ()
Previous frame inner to this frame (corrupt stack?)
Current language: auto; currently minimal
(kgdb) frame 19
#19 0xffffffff808185cb in devfs_read_f (fp=3D0xfffff80083439230, uio=3D0xff=
fffe1839a54ab0,=20
cred=3D<value optimized out>, flags=3D0, td=3D0xfffff80e4edb8490)
at /usr/src/sys/fs/devfs/devfs_vnops.c:1193
1193 error =3D dsw->d_read(dev, uio, ioflag);
(kgdb) list
1188 ioflag =3D fp->f_flag & (O_NONBLOCK | O_DIRECT);
1189 if (ioflag & O_DIRECT)
1190 ioflag |=3D IO_DIRECT;
1191=09
1192 foffset_lock_uio(fp, uio, flags | FOF_NOLOCK);
1193 error =3D dsw->d_read(dev, uio, ioflag);
1194 if (uio->uio_resid !=3D resid || (error =3D=3D 0 && resid !=3D 0))
1195 vfs_timestamp(&dev->si_atime);
1196 td->td_fpop =3D fpop;
1197 dev_relthread(dev, ref);
(kgdb) down
#18 0xffffffff808ecf04 in giant_read (dev=3D0xfffff8000dbd0400, uio=3D0xfff=
ffe1839a54ab0, ioflag=3D0)
at /usr/src/sys/kern/kern_conf.c:442
442 retval =3D dsw->d_gianttrick->d_read(dev, uio, ioflag);
(kgdb) list
437=09
438 dsw =3D dev_refthread(dev, &ref);
439 if (dsw =3D=3D NULL)
440 return (ENXIO);
441 mtx_lock(&Giant);
442 retval =3D dsw->d_gianttrick->d_read(dev, uio, ioflag);
443 mtx_unlock(&Giant);
444 dev_relthread(dev, ref);
445 return (retval);
446 }
(kgdb) p *dev
$1 =3D {si_spare0 =3D 0x0, si_flags =3D 4, si_atime =3D {tv_sec =3D 1394286=
776, tv_nsec =3D 0},=20
si_ctime =3D {tv_sec =3D 1394236183, tv_nsec =3D 584945000}, si_mtime =3D=
{tv_sec =3D 1394236183,=20
tv_nsec =3D 584945000}, si_uid =3D 0, si_gid =3D 2, si_mode =3D 416, si=
_cred =3D 0x0, si_drv0 =3D 1,=20
si_refcount =3D 9, si_list =3D {le_next =3D 0xfffff8000dbd0600, le_prev =
=3D 0xffffffff8144db18},=20
si_clone =3D {le_next =3D 0x0, le_prev =3D 0x0}, si_children =3D {lh_firs=
t =3D 0x0}, si_siblings =3D {
le_next =3D 0x0, le_prev =3D 0x0}, si_parent =3D 0x0, si_mountpt =3D 0x=
0, si_drv1 =3D 0x0,=20
si_drv2 =3D 0x0, si_devsw =3D 0xffffffff8144da78, si_iosize_max =3D 65536=
, si_usecount =3D 1,=20
si_threadcount =3D 2, __si_u =3D {__sid_snapdata =3D 0x0},=20
si_name =3D "kmem", '\0' <repeats 59 times>}
(kgdb) p *uio
$2 =3D {uio_iov =3D 0xfffffe1839a54aa0, uio_iovcnt =3D 1, uio_offset =3D -2=
184830705664,=20
uio_resid =3D 113246208, uio_segflg =3D UIO_USERSPACE, uio_rw =3D UIO_REA=
D,=20
uio_td =3D 0xfffff80e4edb8490}
(kgdb) p *ioflag
Cannot access memory at address 0x0
(kgdb) p Giant
$3 =3D {lock_object =3D {lo_name =3D 0xffffffff8100e05a "Giant", lo_flags =
=3D 17498112, lo_data =3D 0,=20
lo_witness =3D 0x0}, mtx_lock =3D 18446735339069080720}
(kgdb) down
#17 0xffffffff80d8612f in memrw (dev=3D0xfffff8000dbd0400, uio=3D0xfffffe18=
39a54ab0,=20
flags=3D113246208) at /usr/src/sys/amd64/amd64/mem.c:101
101 error =3D uiomove((void *)PHYS_TO_DMAP(v), (int)c, uio);
(kgdb) list
96 if (dev2unit(dev) =3D=3D CDEV_MINOR_MEM) {
97 v =3D uio->uio_offset;
98 kmemphys:
99 o =3D v & PAGE_MASK;
100 c =3D min(uio->uio_resid, (u_int)(PAGE_SIZE - o));
101 error =3D uiomove((void *)PHYS_TO_DMAP(v), (int)c, uio);
102 continue;
103 }
104 else if (dev2unit(dev) =3D=3D CDEV_MINOR_KMEM) {
105 v =3D uio->uio_offset;
(kgdb) p *v
$4 =3D 0
(kgdb) p *c
$5 =3D 0
(kgdb) p *uio
$6 =3D {uio_iov =3D 0xfffffe1839a54aa0, uio_iovcnt =3D 1, uio_offset =3D -2=
184830705664,=20
uio_resid =3D 113246208, uio_segflg =3D UIO_USERSPACE, uio_rw =3D UIO_REA=
D,=20
uio_td =3D 0xfffff80e4edb8490}
(kgdb) down
#16 0xffffffff8099c2f5 in uiomove_faultflag (cp=3D<value optimized out>,=20
n=3D<value optimized out>, uio=3D0xfffffe1839a54ab0, nofault=3D<value o=
ptimized out>)
at /usr/src/sys/kern/subr_uio.c:192
192 error =3D copyout(cp, iov->iov_base, cnt);
(kgdb) list
187 switch (uio->uio_segflg) {
188=09
189 case UIO_USERSPACE:
190 maybe_yield();
191 if (uio->uio_rw =3D=3D UIO_READ)
192 error =3D copyout(cp, iov->iov_base, cnt);
193 else
194 error =3D copyin(iov->iov_base, cp, cnt);
195 if (error)
196 goto out;
(kgdb) p *cp
Attempt to dereference a generic pointer.
(kgdb) p cp
$7 =3D <value optimized out>
(kgdb) down
#15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:=
246
246 cld
Current language: auto; currently asm
(kgdb) list
241 xchgq %rdi,%rsi
242 /* bcopy(%rsi, %rdi, %rdx) */
243 movq %rdx,%rcx
244=09
245 shrq $3,%rcx
246 cld
247 rep
248 movsq
249 movb %dl,%cl
250 andb $7,%cl
(kgdb) down
#14 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception=
=2ES:231
231 call trap
(kgdb) list
226 #endif
227 .globl calltrap
228 .type calltrap,@function
229 calltrap:
230 movq %rsp,%rdi
231 call trap
232 MEXITCOUNT
233 jmp doreti /* Handle any pending ASTs */
234=09
235 /*
(kgdb) quit
Script done on Sun Mar 9 16:46:04 2014
Glen
--s5/bjXLgkIwAv6Hi
Content-Type: application/pgp-signature
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.22 (FreeBSD)
iQIcBAEBCAAGBQJTHJ1QAAoJELls3eqvi17QvuYQAK1+lQnlh/KNuObYFEPCE+H0
tFpx9PRRsywGXZn4IhBLkDjzI3sdPAET87vhf0qmFFjtcrR0H0Mhg/rpme855BlW
/kfcWNw0v4qpzcGB7Ua7hFTaNlUM4nQ7fT5CnBOPgkZAjOBlktE4EVKLwL26fX+C
BNh/DCwpGlHMbccIQXXuRWdV8YC8V+Rda8K5uwDvTEw10F29FMmNVSbVFzdVj0qI
MQsSDz4cPgbYPSkUVHbNXIO4wErcctpiSSI7WWXFXj7MBYAp212zjS9ONUlcVxas
rXmNK/BmPRlqN0F8R71yC4DgzrkbJIriMFwBRUHR+ptlktSZqWvLaaaTtkBObGjB
Q5X5Fc0BxAdED/jkMieom/MfWU4w5ijp8nMN+MCzI70n7xa7BsAwmu6zeeas4CJk
yUSbL9M0muWcduDdHCNmd90TPSqmwY04F6RNZNy38Ol+cx1ilHtqqSQ5jXyuD6T9
2/RrgehfMIjqqlYy38xAzgtGxHwKxlcSVIHZCdnC5XL+79cTbRdAyAB2XZlOJRoz
ZOkgQUom6rSH/2lHPBJsrV017qlwAxYA5/1z/kzIZqhjUJQ0fbblS701rRYaA+pX
3NLTIdwCVwFf/UjrDLzXv58gTxGYsd5t+hXtxPvZxBst1w2UqpohV3tDX9GZ/npf
vgGv5/3/d3YHYMRivpGz
=/ww3
-----END PGP SIGNATURE-----
--s5/bjXLgkIwAv6Hi--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20140309165648.GF1776>