Date: Fri, 5 Jun 2009 09:38:25 -0400 From: John Baldwin <jhb@freebsd.org> To: freebsd-hackers@freebsd.org Cc: Menshikov Konstantin <kostjn@peterhost.ru> Subject: Re: Inline function (difficult debug) Message-ID: <200906050938.25706.jhb@freebsd.org> In-Reply-To: <4A27C5E8.7020902@peterhost.ru> References: <4A2792B7.5010006@peterhost.ru> <200906040749.17206.jhb@freebsd.org> <4A27C5E8.7020902@peterhost.ru>
next in thread | previous in thread | raw e-mail | index | archive | help
On Thursday 04 June 2009 9:02:32 am Menshikov Konstantin wrote:
> John Baldwin wrote:
> > On Thursday 04 June 2009 5:24:07 am Menshikov Konstantin wrote:
> > =20
> >> Hi.
> >> My system FreeBSD 7.1 RELEASE periodically freeze.
> >>
> >> I`m compiling kernel with WITNESS and get backtrace.
> >>
> >> #0 doadump () at pcpu.h:195
> >> #1 0xffffffff801a899c in db_fncall (dummy1=3DVariable "dummy1" is not=
=20
> >> available.
> >> ) at /usr/src/sys/ddb/db_command.c:516
> >> #2 0xffffffff801a8ecf in db_command (last_cmdp=3D0xffffffff80825688,=20
> >> cmd_table=3D0x0, dopager=3D1)
> >> at /usr/src/sys/ddb/db_command.c:413
> >> #3 0xffffffff801a90e0 in db_command_loop () at=20
> >> /usr/src/sys/ddb/db_command.c:466
> >> #4 0xffffffff801aaa19 in db_trap (type=3DVariable "type" is not availa=
ble.
> >> ) at /usr/src/sys/ddb/db_main.c:228
> >> #5 0xffffffff803d5e55 in kdb_trap (type=3D3, code=3D0,=20
> >> tf=3D0xffffffffabf284c0) at /usr/src/sys/kern/subr_kdb.c:524
> >> #6 0xffffffff8059a0e5 in trap (frame=3D0xffffffffabf284c0) at=20
> >> /usr/src/sys/amd64/amd64/trap.c:538
> >> #7 0xffffffff8057f60e in calltrap () at=20
> >> /usr/src/sys/amd64/amd64/exception.S:209
> >> #8 0xffffffff803d602d in kdb_enter_why (why=3D0xffffffff80628137=20
> >> "witness", msg=3D0xa <Address 0xa out of bounds>)
> >> at cpufunc.h:63
> >> #9 0xffffffff803e70c3 in witness_checkorder (lock=3DVariable "lock" is=
not=20
> >> available.
> >> ) at /usr/src/sys/kern/subr_witness.c:1126
> >> #10 0xffffffff8039e341 in _mtx_lock_spin_flags (m=3D0xffffffff8084fad0=
,=20
> >> opts=3D0, file=3Ddwarf2_read_address: Corrupted DWARF expression.
> >> ) at /usr/src/sys/kern/kern_mutex.c:227
> >> #11 0xffffffff80319838 in sc_puts (scp=3D0xffffffff8084f980,=20
> >> buf=3D0xffffffffabf286a7 "a@\az\200=D0=AA=D0=AA=D0=AA=D0=AA", len=3D1)
> >> at /usr/src/sys/dev/syscons/syscons.c:2519
> >> #12 0xffffffff8031b3da in sc_cnputc (cd=3DVariable "cd" is not availab=
le.
> >> ) at /usr/src/sys/dev/syscons/syscons.c:1561
> >> #13 0xffffffff803f9d89 in cnputc (c=3D97)=20
at /usr/src/sys/kern/tty_cons.c:632
> >> #14 0xffffffff803d993b in putchar (c=3D97, arg=3DVariable "arg" is not=
=20
> >> available.
> >> ) at /usr/src/sys/kern/subr_prf.c:421
> >> #15 0xffffffff803d7f22 in kvprintf (fmt=3D0xffffffff80628b89 "cquiring=
=20
> >> duplicate lock of same type: \"%s\"\n",
> >> func=3D0xffffffff803d98d0 <putchar>, arg=3D0xffffffffabf28890, radix=
=3D10,=20
> >> ap=3DVariable "ap" is not available.
> >> ) at /usr/src/sys/kern/subr_prf.c:674
> >> #16 0xffffffff803d9384 in printf (fmt=3DVariable "fmt" is not availabl=
e.
> >> ) at /usr/src/sys/kern/subr_prf.c:314
> >> #17 0xffffffff803e705d in witness_checkorder (lock=3D0xffffffff808668f=
8,=20
> >> flags=3D0,
> >> file=3D0xffffffff80626cf8 "/usr/src/sys/kern/subr_sleepqueue.c", line=
=3D232)=20
> >> at /usr/src/sys/kern/subr_witness.c:948
> >> #18 0xffffffff8039e341 in _mtx_lock_spin_flags (m=3D0xffffffff808668f8=
,=20
> >> opts=3D0, file=3Ddwarf2_read_address: Corrupted DWARF expression.
> >> ) at /usr/src/sys/kern/kern_mutex.c:227
> >> #19 0xffffffff803b2271 in wakeup (ident=3D0xffffffff80850ac0) at=20
> >> /usr/src/sys/kern/kern_synch.c:341
> >> #20 0xffffffff803aed95 in tdsignal (p=3D0xffffff00017978f0,=20
> >> td=3D0xffffff000146a6e0, sig=3D14, ksi=3DVariable "ksi" is not availab=
le.
> >> )
> >> at /usr/src/sys/kern/kern_sig.c:2292
> >> #21 0xffffffff803ba2de in realitexpire (arg=3DVariable "arg" is not=20
available.
> >> ) at /usr/src/sys/kern/kern_time.c:669
> >> #22 0xffffffff803bbe1a in softclock (dummy=3DVariable "dummy" is not=20
> >> available.
> >> ) at /usr/src/sys/kern/kern_timeout.c:274
> >> #23 0xffffffff8038c120 in ithread_loop (arg=3D0xffffff00010ebc00) at=20
> >> /usr/src/sys/kern/kern_intr.c:1088
> >> #24 0xffffffff8038978a in fork_exit (callout=3D0xffffffff8038c040=20
> >> <ithread_loop>, arg=3D0xffffff00010ebc00,
> >> frame=3D0xffffffffabf28c80) at /usr/src/sys/kern/kern_fork.c:804
> >> #25 0xffffffff8057f9de in fork_trampoline () at=20
> >> /usr/src/sys/amd64/amd64/exception.S:455
> >>
> >> But file /usr/src/sys/kern/kern_sig.c:2292
> >> hasn`t run function wake up on string 2292
> >> I`m think, what compiler use inline function, therefore code and trace=
=20
> >> differ.
> >> Tell me please, how compile kernel without any inline function.
> >> =20
> >
> > makeoptions DEBUG=3D"-g -fno-inline"
> >
> > However. I think you already have enough information to debug this. Ca=
n=20
you=20
> > go to frame 17 and do an 'l'? I think the two things you want to print=
=20
out=20
> > are "i", "*lock", "*lock1", and "*lock2".
> >
> > =20
> Unfortunately, the kernel is not compiled with an option-fno-inline.
> cc -c -O2 -frename-registers -pipe -fno-strict-aliasing -std=3Dc99 -g=20
> -fno-inline -Wall -Wredundant-decls -Wnested-externs=20
> -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Winline=20
> -Wcast-qual -Wundef -Wno-pointer-sign -fformat-extensions -nostdinc =20
> -I. -I/usr/src/sys -I/usr/src/sys/contrib/altq -D_KERNEL=20
> -DHAVE_KERNEL_OPTION_HEADERS -include opt_global.h -fno-common=20
> -finline-limit=3D8000 --param inline-unit-growth=3D100 --param=20
> large-function-growth=3D1000 -fno-omit-frame-pointer -mcmodel=3Dkernel=20
> -mno-red-zone -mfpmath=3D387 -mno-sse -mno-sse2 -mno-mmx -mno-3dnow =20
> -msoft-float -fno-asynchronous-unwind-tables -ffreestanding -Werror =20
> /usr/src/sys/cam/cam_periph.c
> cc1: warnings being treated as errors
> /usr/src/sys/cam/cam_periph.c: In function 'scsi_extract_sense':
> /usr/src/sys/cam/scsi/scsi_all.h:1215: warning: function=20
> 'scsi_extract_sense' can never be inlined because it is suppressed using=
=20
> -fno-inline
>=20
> I`m try build with -O0 options.
>=20
> lock in frame 17
> (kgdb) print {struct mtx} 0xffffffff808668f8
> $4 =3D {lock_object =3D {lo_name =3D 0xffffffff806270eb "sleepq chain",=20
> lo_type =3D 0xffffffff806270eb "sleepq chain",
> lo_flags =3D 720896, lo_witness_data =3D {lod_list =3D {stqe_next =3D=
=20
> 0xffffffff808793e0}, lod_witness =3D 0xffffffff808793e0}},
> mtx_lock =3D 4, mtx_recurse =3D 0}
> next in list
> (kgdb) print {struct mtx} 0xffffffff808793e0
> $5 =3D {lock_object =3D {lo_name =3D 0xffffffff806270eb "sleepq chain",=20
> lo_type =3D 0xffffffff807e8860 "\031\006b\200=D0=AA=D0=AA=D0=AA=D0=AA\n",
> lo_flags =3D 2156368928, lo_witness_data =3D {lod_list =3D {stqe_next=
=3D=20
> 0xffffffff80879420},
> lod_witness =3D 0xffffffff80879420}}, mtx_lock =3D=20
> 18446744071571216096, mtx_recurse =3D 2153913763}
> (kgdb) print {struct mtx} 0xffffffff80879420
> next
> $6 =3D {lock_object =3D {lo_name =3D 0xffffffff806205df "process slock",=
=20
> lo_type =3D 0xffffffff807e8860 "\031\006b\200=D0=AA=D0=AA=D0=AA=D0=AA\n",
> lo_flags =3D 2156368992, lo_witness_data =3D {lod_list =3D {stqe_next=
=3D=20
> 0xffffffff80879460},
> lod_witness =3D 0xffffffff80879460}}, mtx_lock =3D=20
> 18446744071571216160, mtx_recurse =3D 2153889520}
Where did you get the pointer values from? I meant doing 'frame 17', 'p=20
i', 'p *lock', 'p *lock1', 'p *lock2'
=2D-=20
John Baldwin
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906050938.25706.jhb>