Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 12 Mar 2019 02:35:16 +0000
From:      bugzilla-noreply@freebsd.org
To:        fs@FreeBSD.org
Subject:   [Bug 236220] ZFS vnode deadlock on zfs_mknode
Message-ID:  <bug-236220-3630-DH41ERbzuf@https.bugs.freebsd.org/bugzilla/>
In-Reply-To: <bug-236220-3630@https.bugs.freebsd.org/bugzilla/>
References:  <bug-236220-3630@https.bugs.freebsd.org/bugzilla/>

next in thread | previous in thread | raw e-mail | index | archive | help
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=3D236220

--- Comment #12 from ncrogers@gmail.com ---
(In reply to Andriy Gapon from comment #11)

It does seems like it may be a different issue, however definitely started
happening after enabling invariants and seems to take about as long as the
deadlock issue to manifest.

Here is the requested info:

(kgdb) fr 6
#6  0xffffffff803f094b in dbuf_create (dn=3D0xfffff8015f77a770,=20
    level=3D0 '\000', blkid=3D18446744073709551615,=20
    parent=3D0xfffff80296803c60, blkptr=3D0x0)
    at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:2540
2540            ASSERT(dn->dn_type !=3D DMU_OT_NONE);


(kgdb) print *dn
$2 =3D {dn_struct_rwlock =3D {lock_object =3D {
      lo_name =3D 0xffffffff814ada70 "dn->dn_struct_rwlock",=20
      lo_flags =3D 40960000, lo_data =3D 0, lo_witness =3D 0x0},=20
    sx_lock =3D 18446735277813228928}, dn_link =3D {
    list_next =3D 0xfffff8026c5973d8, list_prev =3D 0xfffff8018f9cbb48},=20
  dn_objset =3D 0xfffff8000b72c400, dn_object =3D 33262,=20
  dn_dbuf =3D 0xfffff80296803c60, dn_handle =3D 0xfffff802edef9520,=20
  dn_phys =3D 0xfffffe01c6fd1c00, dn_type =3D DMU_OT_NONE, dn_bonuslen =3D =
168,=20
  dn_bonustype =3D 44 ',', dn_nblkptr =3D 1 '\001', dn_checksum =3D 0 '\000=
',=20
  dn_compress =3D 0 '\000', dn_nlevels =3D 1 '\001',=20
  dn_indblkshift =3D 17 '\021', dn_datablkshift =3D 0 '\000',=20
  dn_moved =3D 0 '\000', dn_datablkszsec =3D 7, dn_datablksz =3D 3584,=20
  dn_maxblkid =3D 0, dn_next_type =3D "\000\000\000",=20
  dn_num_slots =3D 1 '\001', dn_next_nblkptr =3D "\000\000\000",=20
  dn_next_nlevels =3D "\000\000\000", dn_next_indblkshift =3D "\000\000\000=
",=20
  dn_next_bonustype =3D "\000\000\000", dn_rm_spillblk =3D "\000\000\000",=
=20
  dn_next_bonuslen =3D {0, 0, 0, 0}, dn_next_blksz =3D {0, 0, 0, 0},=20
  dn_dbufs_count =3D 0, dn_dirty_link =3D {{list_next =3D 0x0,=20
      list_prev =3D 0x0}, {list_next =3D 0xfffff803bbf774e0,=20
      list_prev =3D 0xfffff8034a965830}, {list_next =3D 0x0,=20
      list_prev =3D 0x0}, {list_next =3D 0x0, list_prev =3D 0x0}}, dn_mtx =
=3D {
    lock_object =3D {lo_name =3D 0xffffffff81488d05 "dn->dn_mtx",=20
      lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =
=3D 1},=20
  dn_dirty_records =3D {{list_size =3D 344, list_offset =3D 0, list_head =
=3D {
        list_next =3D 0xfffff8015f77a890, list_prev =3D 0xfffff8015f77a890}=
},=20
    {list_size =3D 344, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff8015f77a8b0, list_prev =3D 0xfffff8015f77a8b0}=
},=20
    {list_size =3D 344, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff8015f77a8d0, list_prev =3D 0xfffff8015f77a8d0}=
},=20
    {list_size =3D 344, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff8015f77a8f0,=20
        list_prev =3D 0xfffff8015f77a8f0}}}, dn_free_ranges =3D {0x0, 0x0,=
=20
    0x0, 0x0}, dn_allocated_txg =3D 0, dn_free_txg =3D 0,=20
  dn_assigned_txg =3D 0, dn_notxholds =3D {
    cv_description =3D 0xffffffff814ba0a4 "dn->dn_notxholds",=20
    cv_waiters =3D 0}, dn_dirtyctx =3D DN_DIRTY_OPEN,=20
  dn_dirtyctx_firstset =3D 0xfffff8038681d070
"\336\300\255\336\336\300\255\336\360G\305\201\377\377\377\377\336\300\255\=
336\336\300\255\336\360G\305\201\377\377\377\377\336\300\255\336\336\300\25=
5\336\360G\305\201\377\377\377\377\336\300\255\336\336\300\255\336\360G\305=
\201\377\377\377\377\336\300\255\336\336\300\255\336\360G\305\201\377\377\3=
77\377\336\300\255\336\336\300\255\336\360G\305\201\377\377\377\377\336\300=
\255\336\336\300\255\336\360G\305\201\377\377\377\377\336\300\255\336\336\3=
00\255\336\360G\305\201\377\377\377\377\336\300\255\336\336\300\255\336\360=
G\305\201\377\377\377\377\336\300\255\336\336\300\255\336\360G\305\201\377\=
377\377\377\336\300\255\336\336\300\255\336\360G\305\201\377\377\377\377\33=
6\300\255\336\336\300\255\336\3--Type
<RET> for more, q to quit, c to continue without paging--
60G\305\201\377\377\377\377\336\300\255\336\336\300\255\336"...,=20
  dn_tx_holds =3D {rc_mtx =3D {lock_object =3D {
        lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", lo_flags =3D 577830912=
,=20
        lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rc_tracked =3D =
0,=20
    rc_list =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff8015f77a990, list_prev =3D 0xfffff8015f77a990}=
},=20
    rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff8015f77a9b0, list_prev =3D 0xfffff8015f77a9b0}=
},=20
    rc_count =3D 0, rc_removed_count =3D 0}, dn_holds =3D {rc_mtx =3D {
      lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
        lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0},=20
      sx_lock =3D 1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40,=20
      list_offset =3D 0, list_head =3D {list_next =3D 0xfffff8015f77aa08,=20
        list_prev =3D 0xfffff8015f77aa08}}, rc_removed =3D {list_size =3D 4=
0,=20
      list_offset =3D 0, list_head =3D {list_next =3D 0xfffff8015f77aa28,=20
        list_prev =3D 0xfffff8015f77aa28}}, rc_count =3D 2,=20
    rc_removed_count =3D 0}, dn_dbufs_mtx =3D {lock_object =3D {
      lo_name =3D 0xffffffff814404f1 "dn->dn_dbufs_mtx",=20
      lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =
=3D 1},=20
  dn_dbufs =3D {avl_root =3D 0x0,=20
    avl_compar =3D 0xffffffff80414810 <dbuf_compare>, avl_offset =3D 288,=20
    avl_numnodes =3D 0, avl_size =3D 352}, dn_bonus =3D 0x0, dn_have_spill =
=3D 0,=20
  dn_zio =3D 0xfffff8024a314000, dn_oldused =3D 0, dn_oldflags =3D 0,=20
  dn_olduid =3D 0, dn_oldgid =3D 0, dn_newuid =3D 70, dn_newgid =3D 70,=20
  dn_id_flags =3D 13, dn_zfetch =3D {zf_rwlock =3D {lock_object =3D {
        lo_name =3D 0xffffffff813ec741 "zf->zf_rwlock",=20
        lo_flags =3D 40960000, lo_data =3D 0, lo_witness =3D 0x0},=20
      sx_lock =3D 1}, zf_stream =3D {list_size =3D 80, list_offset =3D 64,=
=20
      list_head =3D {list_next =3D 0xfffff8015f77ab10,=20
        list_prev =3D 0xfffff8015f77ab10}}, zf_dnode =3D 0xfffff8015f77a770=
}}
(kgdb) print *parent
$3 =3D {db =3D {db_object =3D 0, db_offset =3D 17022976, db_size =3D 16384,=
=20
    db_data =3D 0xfffffe01c6fd0000}, db_objset =3D 0xfffff8000b72c400,=20
  db_dnode_handle =3D 0xfffff8000b72c420, db_parent =3D 0xfffff800743a9000,=
=20
  db_hash_next =3D 0x0, db_blkid =3D 1039, db_blkptr =3D 0xfffffe017b400780=
,=20
  db_level =3D 0 '\000', db_mtx =3D {lock_object =3D {
      lo_name =3D 0xffffffff81427def "db->db_mtx", lo_flags =3D 577830912,=
=20
      lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, db_state =3D DB_C=
ACHED,=20
  db_holds =3D {rc_mtx =3D {lock_object =3D {
        lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", lo_flags =3D 577830912=
,=20
        lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rc_tracked =3D =
0,=20
    rc_list =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff80296803d18, list_prev =3D 0xfffff80296803d18}=
},=20
    rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
        list_next =3D 0xfffff80296803d38, list_prev =3D 0xfffff80296803d38}=
},=20
    rc_count =3D 19, rc_removed_count =3D 0}, db_buf =3D 0xfffff80225c1de00=
,=20
  db_changed =3D {cv_description =3D 0xffffffff8138c10b "db->db_changed",=20
    cv_waiters =3D 0}, db_data_pending =3D 0xfffff80449928000,=20
  db_last_dirty =3D 0xfffff80449928000, db_link =3D {avl_child =3D {
      0xfffff802abcc6ee0, 0xfffff8030efbfc20},=20
    avl_pcb =3D 18446735290778607909}, db_cache_link =3D {list_next =3D 0x0=
,=20
    list_prev =3D 0x0}, db_caching_status =3D DB_NO_CACHE,=20
  db_user =3D 0xfffff802edef9000, db_user_immediate_evict =3D 0 '\000',=20
  db_freed_in_flight =3D 0 '\000', db_pending_evict =3D 0 '\000',=20
  db_dirtycnt =3D 1 '\001'}


(kgdb) fr 9
#9  0xffffffff804cde29 in zfs_zget (zfsvfs=3D0xfffffe008ea28000, obj_num=3D=
33262,
zpp=3D0xfffffe00b77c56c0)
    at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c:=
1137
1137            err =3D sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db);


(kgdb) print *zfsvfs
$4 =3D {z_vfs =3D 0xfffff800133fd000, z_parent =3D 0xfffffe008ea28000, z_os=
 =3D
0xfffff8000b72c400, z_root =3D 4, z_rootvnode =3D 0xfffff8001340eb40,=20
  z_rootvnodelock =3D {lock_object =3D {lo_name =3D 0xffffffff814594a9 "zfs=
 root
vnode lock", lo_flags =3D 50528256, lo_data =3D 0, lo_witness =3D 0x0},=20
    rm_writecpus =3D {__bits =3D {0, 0, 0, 0}}, rm_activeReaders =3D {lh_fi=
rst =3D
0x0}, _rm_lock =3D {_rm_wlock_object =3D {
        lo_name =3D 0xffffffff814594a9 "zfs root vnode lock", lo_flags =3D
16842752, lo_data =3D 0, lo_witness =3D 0x0}, _rm_lock_mtx =3D {lock_object=
 =3D {
          lo_name =3D 0xffffffff814594a9 "zfs root vnode lock", lo_flags =3D
16842752, lo_data =3D 0, lo_witness =3D 0x0}, mtx_lock =3D 0}, _rm_lock_sx =
=3D {
        lock_object =3D {lo_name =3D 0xffffffff814594a9 "zfs root vnode loc=
k",
lo_flags =3D 16842752, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 0}}}=
,=20
  z_unlinkedobj =3D 3, z_max_blksz =3D 16384, z_fuid_obj =3D 0, z_fuid_size=
 =3D 0,
z_fuid_idx =3D {avl_root =3D 0x0, avl_compar =3D 0x0, avl_offset =3D 0,=20
    avl_numnodes =3D 0, avl_size =3D 0}, z_fuid_domain =3D {avl_root =3D 0x=
0,
avl_compar =3D 0x0, avl_offset =3D 0, avl_numnodes =3D 0, avl_size =3D 0},=
=20
  z_fuid_lock =3D {lock_object =3D {lo_name =3D 0xffffffff81496098
"zfsvfs->z_fuid_lock", lo_flags =3D 40960000, lo_data =3D 0, lo_witness =3D=
 0x0},
sx_lock =3D 1},=20
  z_fuid_loaded =3D 0, z_fuid_dirty =3D 0, z_fuid_replay =3D 0x0, z_log =3D
0xfffff80003b09800, z_acl_mode =3D 0, z_acl_inherit =3D 4,=20
  z_case =3D ZFS_CASE_SENSITIVE, z_utf8 =3D 0, z_norm =3D 0, z_atime =3D 0,=
 z_unmounted
=3D 0, z_teardown_lock =3D {locks =3D {{rr_lock =3D {lock_object =3D {
            lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock", lo_flags =3D 577=
830912,
lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rr_cv =3D {
          cv_description =3D 0xffffffff8141c850 "rrl->rr_cv", cv_waiters =
=3D 0},
rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =3D {lock_object =3D {
              lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", lo_flags =3D 577=
830912,
lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rc_tracked =3D 0,=20
          rc_list =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {=
list_next =3D
0xfffffe008ea281c0, list_prev =3D 0xfffffe008ea281c0}}, rc_removed =3D {
            list_size =3D 40, list_offset =3D 0, list_head =3D {list_next =
=3D
0xfffffe008ea281e0, list_prev =3D 0xfffffe008ea281e0}}, rc_count =3D 1,=20
          rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {lock_o=
bject =3D
{lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", lo_flags =3D 577830912,=20
              lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rc_tracke=
d =3D 0,
rc_list =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
              list_next =3D 0xfffffe008ea28238, list_prev =3D 0xfffffe008ea=
28238}},
rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {
              list_next =3D 0xfffffe008ea28258, list_prev =3D 0xfffffe008ea=
28258}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_writer_wanted =3D 0,=20
        rr_track_all =3D 0}, {rr_lock =3D {lock_object =3D {lo_name =3D
0xffffffff813bcb70 "rrl->rr_lock", lo_flags =3D 577830912, lo_data =3D 0,=20
            lo_witness =3D 0x0}, sx_lock =3D 1}, rr_cv =3D {cv_description =
=3D
0xffffffff8141c850 "rrl->rr_cv", cv_waiters =3D 0}, rr_writer =3D 0x0,=20
        rr_anon_rcount =3D {rc_mtx =3D {lock_object =3D {lo_name =3D 0xffff=
ffff814893e6
"rc->rc_mtx", lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0},=20
            sx_lock =3D 1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40,
list_offset =3D 0, list_head =3D {list_next =3D 0xfffffe008ea282f0,=20
              list_prev =3D 0xfffffe008ea282f0}}, rc_removed =3D {list_size=
 =3D 40,
list_offset =3D 0, list_head =3D {list_next =3D 0xfffffe008ea28310,=20
              list_prev =3D 0xfffffe008ea28310}}, rc_count =3D 0, rc_remove=
d_count
=3D 0}, rr_linked_rcount =3D {rc_mtx =3D {lock_object =3D {
              lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", lo_flags =3D 577=
830912,
lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1}, rc_tracked =3D 0,=20
          rc_list =3D {list_size =3D 40, list_offset =3D 0, list_head =3D {=
list_next =3D
0xfffffe008ea28368, list_prev =3D 0xfffffe008ea28368}}, rc_removed =3D {
            list_size =3D 40, list_offset =3D 0, list_head =3D {list_next =
=3D
0xfffffe008ea28388, list_prev =3D 0xfffffe008ea28388}}, rc_count =3D 0,=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28420, list_prev =3D
0xfffffe008ea28420}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28440, list_prev =3D 0xfffffe008ea=
28440}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28498, list_prev =3D 0xfffffe008ea28498}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea284b8, list_prev =3D 0xfffffe008ea284b8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28550, list_prev =3D
0xfffffe008ea28550}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
--Type <RET> for more, q to quit, c to continue without paging--
              list_next =3D 0xfffffe008ea28570, list_prev =3D 0xfffffe008ea=
28570}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea285c8, list_prev =3D 0xfffffe008ea285c8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea285e8, list_prev =3D 0xfffffe008ea285e8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28680, list_prev =3D
0xfffffe008ea28680}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea286a0, list_prev =3D 0xfffffe008ea=
286a0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea286f8, list_prev =3D 0xfffffe008ea286f8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28718, list_prev =3D 0xfffffe008ea28718}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea287b0, list_prev =3D
0xfffffe008ea287b0}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea287d0, list_prev =3D 0xfffffe008ea=
287d0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28828, list_prev =3D 0xfffffe008ea28828}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28848, list_prev =3D 0xfffffe008ea28848}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea288e0, list_prev =3D
0xfffffe008ea288e0}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28900, list_prev =3D 0xfffffe008ea=
28900}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28958, list_prev =3D 0xfffffe008ea28958}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28978, list_prev =3D 0xfffffe008ea28978}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28a10, list_prev =3D
0xfffffe008ea28a10}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28a30, list_prev =3D 0xfffffe008ea=
28a30}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28a88, list_prev =3D 0xfffffe008ea28a88}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28aa8, list_prev =3D 0xfffffe008ea28aa8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
--Type <RET> for more, q to quit, c to continue without paging--
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28b40, list_prev =3D
0xfffffe008ea28b40}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28b60, list_prev =3D 0xfffffe008ea=
28b60}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28bb8, list_prev =3D 0xfffffe008ea28bb8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28bd8, list_prev =3D 0xfffffe008ea28bd8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28c70, list_prev =3D
0xfffffe008ea28c70}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28c90, list_prev =3D 0xfffffe008ea=
28c90}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28ce8, list_prev =3D 0xfffffe008ea28ce8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28d08, list_prev =3D 0xfffffe008ea28d08}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28da0, list_prev =3D
0xfffffe008ea28da0}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28dc0, list_prev =3D 0xfffffe008ea=
28dc0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28e18, list_prev =3D 0xfffffe008ea28e18}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28e38, list_prev =3D 0xfffffe008ea28e38}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea28ed0, list_prev =3D
0xfffffe008ea28ed0}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea28ef0, list_prev =3D 0xfffffe008ea=
28ef0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea28f48, list_prev =3D 0xfffffe008ea28f48}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea28f68, list_prev =3D 0xfffffe008ea28f68}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea29000, list_prev =3D
0xfffffe008ea29000}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea29020, list_prev =3D 0xfffffe008ea=
29020}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea29078, list_prev =3D 0xfffffe008ea29078}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea29098, list_prev =3D 0xfffffe008ea29098}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
--Type <RET> for more, q to quit, c to continue without paging--
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea29130, list_prev =3D
0xfffffe008ea29130}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea29150, list_prev =3D 0xfffffe008ea=
29150}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea291a8, list_prev =3D 0xfffffe008ea291a8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea291c8, list_prev =3D 0xfffffe008ea291c8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea29260, list_prev =3D
0xfffffe008ea29260}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea29280, list_prev =3D 0xfffffe008ea=
29280}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea292d8, list_prev =3D 0xfffffe008ea292d8}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea292f8, list_prev =3D 0xfffffe008ea292f8}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea29390, list_prev =3D
0xfffffe008ea29390}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea293b0, list_prev =3D 0xfffffe008ea=
293b0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea29408, list_prev =3D 0xfffffe008ea29408}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea29428, list_prev =3D 0xfffffe008ea29428}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0},
{rr_lock =3D {lock_object =3D {lo_name =3D 0xffffffff813bcb70 "rrl->rr_lock=
",=20
            lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_=
lock =3D 1},
rr_cv =3D {cv_description =3D 0xffffffff8141c850 "rrl->rr_cv",=20
          cv_waiters =3D 0}, rr_writer =3D 0x0, rr_anon_rcount =3D {rc_mtx =
=3D
{lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx",=20
              lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, s=
x_lock =3D
1}, rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D 0,=20
            list_head =3D {list_next =3D 0xfffffe008ea294c0, list_prev =3D
0xfffffe008ea294c0}}, rc_removed =3D {list_size =3D 40, list_offset =3D 0, =
list_head
=3D {
              list_next =3D 0xfffffe008ea294e0, list_prev =3D 0xfffffe008ea=
294e0}},
rc_count =3D 0, rc_removed_count =3D 0}, rr_linked_rcount =3D {rc_mtx =3D {
            lock_object =3D {lo_name =3D 0xffffffff814893e6 "rc->rc_mtx", l=
o_flags
=3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
          rc_tracked =3D 0, rc_list =3D {list_size =3D 40, list_offset =3D =
0, list_head
=3D {list_next =3D 0xfffffe008ea29538, list_prev =3D 0xfffffe008ea29538}},=
=20
          rc_removed =3D {list_size =3D 40, list_offset =3D 0, list_head =
=3D {list_next
=3D 0xfffffe008ea29558, list_prev =3D 0xfffffe008ea29558}}, rc_count =3D 0,=
=20
          rc_removed_count =3D 0}, rr_writer_wanted =3D 0, rr_track_all =3D=
 0}}},
z_teardown_inactive_lock =3D {lock_object =3D {
      lo_name =3D 0xffffffff81441592 "zfsvfs->z_teardown_inactive_lock", lo=
_flags
=3D 40960000, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=20
  z_all_znodes =3D {list_size =3D 272, list_offset =3D 240, list_head =3D {=
list_next =3D
0xfffff8000ba58420, list_prev =3D 0xfffff8006add8ca0}}, z_znodes_lock =3D {
    lock_object =3D {lo_name =3D 0xffffffff81489faa "zfsvfs->z_znodes_lock",
lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock =3D 1},=
=20
  z_ctldir =3D 0xfffff8000ba05400, z_show_ctldir =3D 0, z_issnap =3D 0, z_v=
scan =3D 0,
z_use_fuids =3D 1, z_replay =3D 0, z_use_sa =3D 1, z_use_namecache =3D 1,=20
  z_version =3D 5, z_shares_dir =3D 7, z_lock =3D {lock_object =3D {lo_name=
 =3D
0xffffffff8138ce8d "zfsvfs->z_lock", lo_flags =3D 577830912, lo_data =3D 0,=
=20
      lo_witness =3D 0x0}, sx_lock =3D 1}, z_userquota_obj =3D 0, z_groupqu=
ota_obj =3D
0, z_replay_eof =3D 0, z_attr_table =3D 0xfffff80013351b00, z_hold_mtx =3D =
{{
      lock_object =3D {lo_name =3D 0xffffffff81429501 "zfsvfs->z_hold_mtx[i=
]",
lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0},=20
--Type <RET> for more, q to quit, c to continue without paging--
      sx_lock =3D 1} <repeats 46 times>, {lock_object =3D {lo_name =3D
0xffffffff81429501 "zfsvfs->z_hold_mtx[i]", lo_flags =3D 577830912, lo_data=
 =3D 0,=20
        lo_witness =3D 0x0}, sx_lock =3D 18446735277813228928}, {lock_objec=
t =3D
{lo_name =3D 0xffffffff81429501 "zfsvfs->z_hold_mtx[i]",=20
        lo_flags =3D 577830912, lo_data =3D 0, lo_witness =3D 0x0}, sx_lock=
 =3D 1}
<repeats 17 times>}, z_unlinked_drain_task =3D {ta_link =3D {stqe_next =3D =
0x0},=20
    ta_pending =3D 0, ta_priority =3D 0, ta_func =3D 0xffffffff804bdcd0
<zfsvfs_task_unlinked_drain>, ta_context =3D 0xfffffe008ea28000}}


(kgdb) print *zfsvfs->z_vfs
$5 =3D {mnt_mtx =3D {lock_object =3D {lo_name =3D 0xffffffff81393043 "struc=
t mount
mtx", lo_flags =3D 16973824, lo_data =3D 0, lo_witness =3D 0x0}, mtx_lock =
=3D 0},=20
  mnt_gen =3D 1, mnt_list =3D {tqe_next =3D 0xfffff8001343a000, tqe_prev =3D
0xfffff800133fe028}, mnt_op =3D 0xffffffff81c64dc8 <zfs_vfsops>,=20
  mnt_vfc =3D 0xffffffff81c64f18 <zfs_vfsconf>, mnt_vnodecovered =3D
0xfffff8001340ed20, mnt_syncer =3D 0xfffff8001340e960, mnt_ref =3D 896,=20
  mnt_nvnodelist =3D {tqh_first =3D 0xfffff8001340eb40, tqh_last =3D
0xfffff80137ee83e0}, mnt_nvnodelistsize =3D 896, mnt_writeopcount =3D 0,=20
  mnt_kern_flag =3D 1073742272, mnt_flag =3D 268439568, mnt_opt =3D
0xfffff8000ba57e10, mnt_optnew =3D 0x0, mnt_maxsymlinklen =3D 0, mnt_stat =
=3D {
    f_version =3D 538182936, f_type =3D 222, f_flags =3D 268439568, f_bsize=
 =3D 512,
f_iosize =3D 16384, f_blocks =3D 404148032, f_bfree =3D 390589344,=20
    f_bavail =3D 390589344, f_files =3D 390593281, f_ffree =3D 390589344,
f_syncwrites =3D 0, f_asyncwrites =3D 0, f_syncreads =3D 0, f_asyncreads =
=3D 0,=20
    f_spare =3D {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, f_namemax =3D 255, f_owner =
=3D 0,
f_fsid =3D {val =3D {-901290623, -1775448610}},=20
    f_charspare =3D '\000' <repeats 79 times>, f_fstypename =3D "zfs", '\00=
0'
<repeats 12 times>,=20
    f_mntfromname =3D "zroot/space/pgsql", '\000' <repeats 1006 times>,
f_mntonname =3D "/space/pgsql", '\000' <repeats 1011 times>},=20
  mnt_cred =3D 0xfffff800133ef900, mnt_data =3D 0xfffffe008ea28000, mnt_tim=
e =3D 0,
mnt_iosize_max =3D 65536, mnt_export =3D 0x0, mnt_label =3D 0x0,=20
  mnt_hashseed =3D 127686091, mnt_lockref =3D 0, mnt_secondary_writes =3D 0,
mnt_secondary_accwrites =3D 0, mnt_susp_owner =3D 0x0, mnt_gjprovider =3D 0=
x0,=20
  mnt_listmtx =3D {lock_object =3D {lo_name =3D 0xffffffff8142f7e7 "struct =
mount
vlist mtx", lo_flags =3D 16973824, lo_data =3D 0, lo_witness =3D 0x0},=20
    mtx_lock =3D 0}, mnt_activevnodelist =3D {tqh_first =3D 0xfffff802882eb=
b40,
tqh_last =3D 0xfffff8001340ec00}, mnt_activevnodelistsize =3D 438,=20
  mnt_tmpfreevnodelist =3D {tqh_first =3D 0xfffff800244071e0, tqh_last =3D
0xfffff803c87a82a0}, mnt_tmpfreevnodelistsize =3D 11, mnt_explock =3D {
    lock_object =3D {lo_name =3D 0xffffffff81469cce "explock", lo_flags =3D
108199936, lo_data =3D 0, lo_witness =3D 0x0}, lk_lock =3D 1, lk_exslpfail =
=3D 0,=20
    lk_timo =3D 0, lk_pri =3D 96}, mnt_upper_link =3D {tqe_next =3D 0x0, tq=
e_prev =3D
0x0}, mnt_uppers =3D {tqh_first =3D 0x0, tqh_last =3D 0xfffff800133fdaa8}}
(kgdb)=20


courtland# zdb -dddd zroot/space/pgsql 33262
Dataset zroot/space/pgsql [ZPL], ID 184, cr_txg 37, 6.37G, 3905 objects, ro=
otbp
DVA[0]=3D<0:95f64f000:1000> DVA[1]=3D<0:11e4fbc000:1000> [L0 DMU objset] fl=
etcher4
uncompressed LE contiguous unique double size=3D800L/800P birth=3D925624L/9=
25624P
fill=3D3905 cksum=3Dd3563dce4:eed0e01cca3:a72f616c79c87:5770ae867335855

    Object  lvl   iblk   dblk  dsize  dnsize lsize   %full  type
zdb: dmu_bonus_hold(33262) failed, errno 2

--=20
You are receiving this mail because:
You are the assignee for the bug.=



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?bug-236220-3630-DH41ERbzuf>