Date: Tue, 24 Sep 2013 15:14:34 +0300 From: Konstantin Belousov <kostikbel@gmail.com> To: Patrick Lamaiziere <patfbsd@davenulle.org> Cc: freebsd-stable@freebsd.org Subject: Re: Possible kqueue related issue on STABLE/RC. Message-ID: <20130924121434.GI41229@kib.kiev.ua> In-Reply-To: <20130924114738.60c700c9@mr129166> References: <alpine.BSF.2.00.1309111705460.89324@olgeni.olgeni> <20130911171913.GG41229@kib.kiev.ua> <alpine.BSF.2.00.1309112316160.2547@olgeni.olgeni> <20130912073643.GM41229@kib.kiev.ua> <20130920151705.33aae120@mr129166> <20130923153708.45c3be3d@mr129166> <20130923203141.GV41229@kib.kiev.ua> <20130924094427.0f4b902a@mr129166> <20130924082909.GH41229@kib.kiev.ua> <20130924114738.60c700c9@mr129166>
next in thread | previous in thread | raw e-mail | index | archive | help
--NqggDZQ8GcqffSaB Content-Type: text/plain; charset=koi8-r Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Tue, Sep 24, 2013 at 11:47:38AM +0200, Patrick Lamaiziere wrote: > Le Tue, 24 Sep 2013 11:29:09 +0300, > Konstantin Belousov <kostikbel@gmail.com> a ?crit : >=20 > Hello, >=20 > ... >=20 > > > > > Ok This has been mfced to 9.2-STABLE. But I still see this panic > > > > > with 9-2/STABLE of today (Revision=9A: 255811). This may be better > > > > > because before the box paniced within minutes and now within > > > > > hours (still using poudriere). > > > > >=20 > > > > > panic: > > > > > fault code =3D supervisor read data, page not present > > > > > instruction pointer =3D 0x20:0xffffffff808ebfcd > > > > > stack pointer =3D 0x28:0xffffff824c2e0630 > > > > > frame pointer =3D 0x28:0xffffff824c2e06a0 > > > > > code segment =3D base 0x0, limit 0xfffff, type 0x1b > > > > > =3D DPL 0, pres 1, long 1, def32 0, gran 1 > > > > > processor eflags =3D interrupt enabled, resume, IOPL =3D 0 > > > > > current process =3D 54243 (gvfsd-trash) > > > > > trap number =3D 12 > > > > > panic: page fault > > > > > cpuid =3D 2 > > > > > KDB: stack backtrace: > > > > > #0 0xffffffff80939ad6 at kdb_backtrace+0x66 > > > > > #1 0xffffffff808ffacd at panic+0x1cd > > > > > #2 0xffffffff80cdfbe9 at trap_fatal+0x289 > > > > > #3 0xffffffff80cdff4f at trap_pfault+0x20f > > > > > #4 0xffffffff80ce0504 at trap+0x344 > > > > > #5 0xffffffff80cc9b43 at calltrap+0x8 > > > > > #6 0xffffffff8099d043 at filt_vfsvnode+0xf3 > > > > > #7 0xffffffff808c4793 at kqueue_register+0x3e3 > > > > > #8 0xffffffff808c4de8 at kern_kevent+0x108 > > > > > #9 0xffffffff808c5950 at sys_kevent+0x90 > > > > > #10 0xffffffff80cdf3a8 at amd64_syscall+0x5d8 > > > > > #11 0xffffffff80cc9e27 at Xfast_syscall+0xf7 > > > > >=20 > > > > > Full core.txt :=20 > > > > > http://user.lamaiziere.net/patrick/public/vfs_vnode-core.txt.0 > > > >=20 > > > > For start, please load the core into kgdb and for > > > > frame 8 > > > > p *kn > > >=20 > > > (kgdb) frame 8 > > > #8 0xffffffff8099d043 in filt_vfsvnode (kn=3D0xfffffe0147a7f000, > > > hint=3D0) at /usr/src/sys/kern/vfs_subr.c:4600 > > > 4600 VI_LOCK(vp); > > > (kgdb) p *kn > > > $1 =3D {kn_link =3D {sle_next =3D 0x0}, kn_selnext =3D {sle_next =3D = 0x0},=20 > > > kn_knlist =3D 0x0, kn_tqe =3D {tqe_next =3D 0x0, tqe_prev =3D 0x0},= =20 > > > kn_kq =3D 0xfffffe01079a6200, kn_kevent =3D {ident =3D 62, filter = =3D -4,=20 > > > flags =3D 32784, fflags =3D 0, data =3D 0, udata =3D 0x0}, kn_sta= tus =3D > > > 24, kn_sfflags =3D 47, kn_sdata =3D 0, kn_ptr =3D {p_fp =3D > > > 0xfffffe016949e190, p_proc =3D 0xfffffe016949e190, p_aio =3D > > > 0xfffffe016949e190, p_lio =3D 0xfffffe016949e190}, kn_fop =3D > > > 0xffffffff812f0, kn_hook =3D 0xfffffe0119d0b1f8, kn_hookid =3D 0} > > From the kgdb, also please do > > p *(struct vnode *)0xfffffe0119d0b1f8 >=20 > With a kernel with debug info, this panic becomes mtx_lock() of > destroyed mutex > panic: mtx_lock() of destroyed mutex >=20 > http://user.lamaiziere.net/patrick/public/panic_mtx_lock.txt >=20 > @ /usr/src/sys/kern/vfs_subr.c:4600 cpuid =3D 2 > KDB: stack backtrace: > #0 0xffffffff80920286 at kdb_backtrace+0x66 > #1 0xffffffff808e738d at panic+0x1cd > #2 0xffffffff808d58d6 at _mtx_lock_flags+0x116 > #3 0xffffffff8098143b at filt_vfsvnode+0x3b > #4 0xffffffff808b213a at kqueue_register+0x4ca > #5 0xffffffff808b2688 at kern_kevent+0x108 > #6 0xffffffff808b3190 at sys_kevent+0x90 > #7 0xffffffff80cbd975 at amd64_syscall+0x2f5 > #8 0xffffffff80ca8557 at Xfast_syscall+0xf7 >=20 > (kgdb) frame 5 > #5 0xffffffff808b213a in kqueue_register (kq=3D0xfffffe00ddc98900, kev= =3D0xffffff824bb5f880, td=3D0xfffffe00b1e7f000, waitok=3D1) at /usr/src/sys= /kern/kern_event.c:1136 > 1136 event =3D kn->kn_fop->f_event(kn, 0); >=20 > (kgdb) p *kn > $1 =3D {kn_link =3D {sle_next =3D 0x0}, kn_selnext =3D {sle_next =3D 0xff= fffe011c232b00}, kn_knlist =3D 0x0, kn_tqe =3D {tqe_next =3D 0x0, tqe_prev = =3D 0x0}, kn_kq =3D 0xfffffe00ddc98900,=20 > kn_kevent =3D {ident =3D 62, filter =3D -4, flags =3D 32784, fflags =3D= 0, data =3D 0, udata =3D 0x0}, kn_status =3D 24, kn_sfflags =3D 47, kn_sda= ta =3D 0, kn_ptr =3D { > p_fp =3D 0xfffffe00ddd4d870, p_proc =3D 0xfffffe00ddd4d870, p_aio =3D= 0xfffffe00ddd4d870, p_lio =3D 0xfffffe00ddd4d870}, kn_fop =3D 0xffffffff81= 2fcca0,=20 > kn_hook =3D 0xfffffe02064a6000, kn_hookid =3D 0} >=20 > (kgdb) p *(struct vnode *)0xfffffe02064a6000 > $2 =3D {v_type =3D VBAD, v_tag =3D 0xffffffff80d89084 "none", v_op =3D 0x= 0, v_data =3D 0x0, v_mount =3D 0x0, v_nmntvnodes =3D {tqe_next =3D 0xfffffe= 020d3e6000,=20 > tqe_prev =3D 0xfffffe0086625a68}, v_un =3D {vu_mount =3D 0x0, vu_sock= et =3D 0x0, vu_cdev =3D 0x0, vu_fifoinfo =3D 0x0}, v_hashlist =3D {le_next = =3D 0x0,=20 > le_prev =3D 0xffffff8000de9698}, v_hash =3D 238022, v_cache_src =3D {= lh_first =3D 0x0}, v_cache_dst =3D {tqh_first =3D 0x0, tqh_last =3D 0xfffff= e02064a6060}, v_cache_dd =3D 0x0,=20 > v_cstart =3D 0, v_lasta =3D 0, v_lastw =3D 0, v_clen =3D 0, v_lock =3D = {lock_object =3D {lo_name =3D 0xffffffff80f56e48 "ufs", lo_flags =3D 918814= 72, lo_data =3D 0,=20 > lo_witness =3D 0xffffff80006c3400}, lk_lock =3D 1, lk_exslpfail =3D= 0, lk_timo =3D 51, lk_pri =3D 96, lk_stack =3D {depth =3D 12, pcs =3D {184= 46744071571296822,=20 > 18446744071573768556, 18446744071576111075, 18446744071606114523,= 18446744071576111075, 18446744071572113927, 18446744071572067653, 18446744= 071606111219,=20 > 18446744071572016126, 18446744071572018094, 18446744071575427445,= 18446744071575340375, 0, 0, 0, 0, 0, 0}}}, v_interlock =3D {lock_object = =3D { > lo_name =3D 0xffffffff80f6f8a3 "vnode interlock", lo_flags =3D 1690= 8288, lo_data =3D 0, lo_witness =3D 0xffffff80006a9600}, mtx_lock =3D 6}, v= _vnlock =3D 0xfffffe02064a6098,=20 > v_holdcnt =3D 0, v_usecount =3D 0, v_iflag =3D 128, v_vflag =3D 0, v_wr= itecount =3D 0, v_actfreelist =3D {tqe_next =3D 0xfffffe0086625a40, tqe_pre= v =3D 0xfffffe020d3e61a8},=20 > v_bufobj =3D {bo_mtx =3D {lock_object =3D {lo_name =3D 0xffffffff80f781= ec "bufobj interlock", lo_flags =3D 16908288, lo_data =3D 0, lo_witness =3D= 0xffffff80006b1680},=20 > mtx_lock =3D 6}, bo_clean =3D {bv_hd =3D {tqh_first =3D 0x0, tqh_la= st =3D 0xfffffe02064a61d8}, bv_root =3D 0x0, bv_cnt =3D 0}, bo_dirty =3D {b= v_hd =3D {tqh_first =3D 0x0,=20 > tqh_last =3D 0xfffffe02064a61f8}, bv_root =3D 0x0, bv_cnt =3D 0},= bo_numoutput =3D 0, bo_flag =3D 0, bo_ops =3D 0xffffffff812f9960, bo_bsize= =3D 32768, bo_object =3D 0x0,=20 > bo_synclist =3D {le_next =3D 0xfffffe001d797bf8, le_prev =3D 0xfffffe= 005f632240}, bo_private =3D 0xfffffe02064a6000, __bo_vnode =3D 0xfffffe0206= 4a6000},=20 > v_pollinfo =3D 0xfffffe00b15dc540, v_label =3D 0x0, v_lockf =3D 0x0, v_= rl =3D > {rl_waiters =3D {tqh_first =3D 0x0, tqh_last =3D 0xfffffe02064a6278}, > rl_currdep =3D 0x0}} >=20 >=20 > ----------- >=20 > > Please try the following. > >=20 > > diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c > > index aa165a0..5715f35 100644 > > --- a/sys/kern/vfs_subr.c > > +++ b/sys/kern/vfs_subr.c > > @@ -4421,10 +4421,14 @@ filt_vfsdetach(struct knote *kn) > > static int > > filt_vfsread(struct knote *kn, long hint) > > { > > - struct vnode *vp =3D (struct vnode *)kn->kn_hook; > > + struct vnode *vp; > > struct vattr va; > > int res; > > =20 > > + if ((kn->kn_status & KN_DETACHED) !=3D 0) > > + return (0); > > + vp =3D (struct vnode *)kn->kn_hook; > > + > > /* > > * filesystem is gone, so set the EOF flag and schedule > > * the knote for deletion. > > @@ -4450,8 +4454,11 @@ filt_vfsread(struct knote *kn, long hint) > > static int > > filt_vfswrite(struct knote *kn, long hint) > > { > > - struct vnode *vp =3D (struct vnode *)kn->kn_hook; > > + struct vnode *vp; > > =20 > > + if ((kn->kn_status & KN_DETACHED) !=3D 0) > > + return (0); > > + vp =3D (struct vnode *)kn->kn_hook; > > VI_LOCK(vp); > > =20 > > /* > > @@ -4469,9 +4476,12 @@ filt_vfswrite(struct knote *kn, long hint) > > static int > > filt_vfsvnode(struct knote *kn, long hint) > > { > > - struct vnode *vp =3D (struct vnode *)kn->kn_hook; > > + struct vnode *vp; > > int res; > > =20 > > + if ((kn->kn_status & KN_DETACHED) !=3D 0) > > + return (0); > > + vp =3D (struct vnode *)kn->kn_hook; > > VI_LOCK(vp); > > if (kn->kn_sfflags & hint) > > kn->kn_fflags |=3D hint; >=20 > With this I've got another panic 'vn_lock zero hold count' (it is an asse= rtion in vfs_vnops.c) : >=20 For this one I have another patch, written some time ago, http://people.freebsd.org/~kib/misc/vnode_filter.1.patch Please see below for combined patch, thank you for the testing and detailed reports. > http://user.lamaiziere.net/patrick/public/panic_zero_hold_count.txt >=20 > panic: vn_lock 0xfffffe018afd0000: zero hold count > cpuid =3D 4 > KDB: stack backtrace: > #0 0xffffffff80920286 at kdb_backtrace+0x66 > #1 0xffffffff808e738d at panic+0x1cd > #2 0xffffffff80994abd at _vn_lock+0xcd > #3 0xffffffff808aff57 at knlist_remove_kq+0x67 > #4 0xffffffff808b1617 at knote_fdclose+0x177 > #5 0xffffffff808ab1a9 at kern_close+0xe9 > #6 0xffffffff80cbd9b5 at amd64_syscall+0x2f5 > #7 0xffffffff80ca8597 at Xfast_syscall+0xf7 >=20 > (kgdb) #0 doadump (textdump=3D<value optimized out>) at pcpu.h:234 > #1 0xffffffff808e792e in kern_reboot (howto=3D260) > at /usr/src/sys/kern/kern_shutdown.c:449 > #2 0xffffffff808e7364 in panic (fmt=3D0x1 <Address 0x1 out of bounds>) > at /usr/src/sys/kern/kern_shutdown.c:637 > #3 0xffffffff80994abd in _vn_lock (vp=3D0xfffffe018afd0000, flags=3D0, f= ile=3D0x0,=20 > line=3D0) at /usr/src/sys/kern/vfs_vnops.c:1398 > #4 0xffffffff808aff57 in knlist_remove_kq (knl=3D0xfffffe0056b053b0,=20 > kn=3D0xfffffe018a9c0b80, knlislocked=3D0, kqislocked=3D0) > at /usr/src/sys/kern/kern_event.c:1843 > #5 0xffffffff808b1617 in knote_fdclose (td=3D0xfffffe0056c2a000, fd=3D62) > at /usr/src/sys/kern/kern_event.c:2065 > #6 0xffffffff808ab1a9 in kern_close (td=3D0xfffffe0056c2a000, fd=3D62) > at /usr/src/sys/kern/kern_descrip.c:1250 > #7 0xffffffff80cbd9b5 in amd64_syscall (td=3D0xfffffe0056c2a000, traced= =3D0) > at subr_syscall.c:135 > #8 0xffffffff80ca8597 in Xfast_syscall () > at /usr/src/sys/amd64/amd64/exception.S:391 > #9 0x00000008032f3bcc in ?? () > Previous frame inner to this frame (corrupt stack?) >=20 > the code of frame 3 is > (kgdb) frame 3 > #3 0xffffffff80994abd in _vn_lock (vp=3D0xfffffe018afd0000, flags=3D0, f= ile=3D0x0, line=3D0) at /usr/src/sys/kern/vfs_vnops.c:1398 > 1398 KASSERT(vp->v_holdcnt !=3D 0, > (kgdb) l > 1393=09 > 1394 VNASSERT((flags & LK_TYPE_MASK) !=3D 0, vp, > 1395 ("vn_lock called with no locktype.")); > 1396 do { > 1397 #ifdef DEBUG_VFS_LOCKS > 1398 KASSERT(vp->v_holdcnt !=3D 0, > 1399 ("vn_lock %p: zero hold count", vp)); > 1400 #endif > 1401 error =3D VOP_LOCK1(vp, flags, file, line); > 1402 flags &=3D ~LK_INTERLOCK; /* Interlock is always dropped. */ >=20 > the vp is =3D=20 > (kgdb) p *(struct vnode *) 0xfffffe018afd0000 > $2 =3D {v_type =3D VDIR, v_tag =3D 0xffffffff80f56e88 "ufs", v_op =3D 0xf= fffffff8132c800, v_data =3D 0xfffffe013a2a7348, v_mount =3D 0xfffffe0009c00= b70, v_nmntvnodes =3D { > tqe_next =3D 0xfffffe018ace3520, tqe_prev =3D 0xfffffe018afd02b8}, v_= un =3D {vu_mount =3D 0x0, vu_socket =3D 0x0, vu_cdev =3D 0x0, vu_fifoinfo = =3D 0x0}, v_hashlist =3D { > le_next =3D 0x0, le_prev =3D 0xffffff8000e29f00}, v_hash =3D 236846, = v_cache_src =3D {lh_first =3D 0x0}, v_cache_dst =3D {tqh_first =3D 0x0, tqh= _last =3D 0xfffffe018afd0060},=20 > v_cache_dd =3D 0x0, v_cstart =3D 0, v_lasta =3D 0, v_lastw =3D 0, v_cle= n =3D 0, v_lock =3D {lock_object =3D {lo_name =3D 0xffffffff80f56e88 "ufs",= lo_flags =3D 91947008, lo_data =3D 0,=20 > lo_witness =3D 0xffffff80006c3400}, lk_lock =3D 1, lk_exslpfail =3D= 0, lk_timo =3D 51, lk_pri =3D 96, lk_stack =3D {depth =3D 12, pcs =3D {184= 46744071571296822,=20 > 18446744071573768620, 18446744071576111139, 18446744071606114523,= 18446744071576111139, 18446744071572113991, 18446744071572067715, 18446744= 071606111219,=20 > 18446744071572016126, 18446744071572018094, 18446744071575427509,= 18446744071575340439, 0, 0, 0, 0, 0, 0}}}, v_interlock =3D {lock_object = =3D { > lo_name =3D 0xffffffff80f6f8e3 "vnode interlock", lo_flags =3D 1697= 3824, lo_data =3D 0, lo_witness =3D 0xffffff80006a9600}, mtx_lock =3D 4}, v= _vnlock =3D 0xfffffe018afd0098,=20 > v_holdcnt =3D 0, v_usecount =3D 0, v_iflag =3D 256, v_vflag =3D 0, v_wr= itecount =3D 0, v_actfreelist =3D {tqe_next =3D 0xfffffe018ace3cd0, tqe_pre= v =3D 0xfffffe00aefe6e78},=20 > v_bufobj =3D {bo_mtx =3D {lock_object =3D {lo_name =3D 0xffffffff80f782= 2c "bufobj interlock", lo_flags =3D 16973824, lo_data =3D 0, lo_witness =3D= 0xffffff80006b1680},=20 > mtx_lock =3D 4}, bo_clean =3D {bv_hd =3D {tqh_first =3D 0x0, tqh_la= st =3D 0xfffffe018afd01d8}, bv_root =3D 0x0, bv_cnt =3D 0}, bo_dirty =3D {b= v_hd =3D {tqh_first =3D 0x0,=20 > tqh_last =3D 0xfffffe018afd01f8}, bv_root =3D 0x0, bv_cnt =3D 0},= bo_numoutput =3D 0, bo_flag =3D 0, bo_ops =3D 0xffffffff812f99a0, bo_bsize= =3D 32768,=20 > bo_object =3D 0xfffffe01f8a6c570, bo_synclist =3D {le_next =3D 0xffff= fe018afd0448, le_prev =3D 0xfffffe00096d5ca0}, bo_private =3D 0xfffffe018af= d0000,=20 > __bo_vnode =3D 0xfffffe018afd0000}, v_pollinfo =3D 0xfffffe0056b05380= , v_label =3D 0x0, v_lockf =3D 0x0, v_rl =3D {rl_waiters =3D {tqh_first =3D= 0x0,=20 > tqh_last =3D 0xfffffe018afd0278}, rl_currdep =3D 0x0}} >=20 diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index 3cbc95f..d3013a0 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -4397,6 +4399,7 @@ vfs_kqfilter(struct vop_kqfilter_args *ap) v_addpollinfo(vp); if (vp->v_pollinfo =3D=3D NULL) return (ENOMEM); + vhold(vp); knl =3D &vp->v_pollinfo->vpi_selinfo.si_note; knlist_add(knl, kn, 0); =20 @@ -4412,17 +4415,23 @@ filt_vfsdetach(struct knote *kn) struct vnode *vp =3D (struct vnode *)kn->kn_hook; =20 KASSERT(vp->v_pollinfo !=3D NULL, ("Missing v_pollinfo")); + kn->kn_hook =3D NULL; knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); + vdrop(vp); } =20 /*ARGSUSED*/ static int filt_vfsread(struct knote *kn, long hint) { - struct vnode *vp =3D (struct vnode *)kn->kn_hook; + struct vnode *vp; struct vattr va; int res; =20 + if ((kn->kn_status & KN_DETACHED) !=3D 0) + return (0); + vp =3D (struct vnode *)kn->kn_hook; + /* * filesystem is gone, so set the EOF flag and schedule * the knote for deletion. @@ -4448,8 +4457,11 @@ filt_vfsread(struct knote *kn, long hint) static int filt_vfswrite(struct knote *kn, long hint) { - struct vnode *vp =3D (struct vnode *)kn->kn_hook; + struct vnode *vp; =20 + if ((kn->kn_status & KN_DETACHED) !=3D 0) + return (0); + vp =3D (struct vnode *)kn->kn_hook; VI_LOCK(vp); =20 /* @@ -4467,9 +4479,12 @@ filt_vfswrite(struct knote *kn, long hint) static int filt_vfsvnode(struct knote *kn, long hint) { - struct vnode *vp =3D (struct vnode *)kn->kn_hook; + struct vnode *vp; int res; =20 + if ((kn->kn_status & KN_DETACHED) !=3D 0) + return (0); + vp =3D (struct vnode *)kn->kn_hook; VI_LOCK(vp); if (kn->kn_sfflags & hint) kn->kn_fflags |=3D hint; --NqggDZQ8GcqffSaB Content-Type: application/pgp-signature -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.21 (FreeBSD) iQIcBAEBAgAGBQJSQYIpAAoJEJDCuSvBvK1Bfp0QAIwIe9R8UYOh6eDtw3M9CHpd z2VjMASCjHyEKPEUwN0AIDRvEsPxJI0Q4Mt7Q5XQ2eiqYV1QTrWPy2EvTJ1wKr5u Bgmzz7rJNd4x/+Ih51qJI4opFowT7vwP+V07F8R3fcrX9ISfRzbztbvMe4/oAMqB iLjvyfeXsBBVVirwNEyIzm/SlNKv79lHN57LQ4TQLSOUFAmPR6lniwZkJYbH8WWA QDuEJ/8aYiBaSv7o8rkwN7RH+YuA/YxM0y9Y9hQoMgUo6LoI6pMdH10AtMlwOJ4B Pcr1/uKojxaVGWaeR2UDC2cqtb3jFMiujZdlzAg9Oi3hYqy2iJ/ZWJENezuwCt+v VjAHnKWGOT4d9/VSR4wrbUIyX1MlC/n+E4BV0ZYxBmM5K2GGOU0aP9GxZZJYsVyo Xfkvywm7eWtCNQD57QJGLoR30O8orGjzNK2ef00JyfliW2pibhcb1AwlQgo5nUXc LCmyPLN5dvwFSBR8IhUsqJPKPYG8bYRpzWZJt4n+hJ8Tuybw6z2j+0ybjtx7bV13 D0OwomL8Fo7A4U/CoBjmTvHmZM95isKvdSkuwVIg7n6Zu8jzlk7aZ7Ua8IqfswDy 676ff04+8Bgaf8zo/iFuFZSG8HtMaxk5KQ2rSkU39W/nBknmk18a4wJ4wqxESmmX pTYIhuK7QaP9twgpJhEy =ze+r -----END PGP SIGNATURE----- --NqggDZQ8GcqffSaB--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20130924121434.GI41229>